You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/05/08 21:14:47 UTC
[01/50] [abbrv] lucene-solr:jira/solr-11779: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11779 d778367b6 -> b5fa66538
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e263ae30
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e263ae30
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e263ae30
Branch: refs/heads/jira/solr-11779
Commit: e263ae30ed011315f377097e08f2bb6ca1d35594
Parents: 43f0c67 e3a9817
Author: Karl Wright <Da...@gmail.com>
Authored: Fri Apr 27 08:39:40 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Fri Apr 27 08:39:40 2018 -0400
----------------------------------------------------------------------
.../apache/solr/search/join/BJQParserTest.java | 62 +++++++-------------
1 file changed, 22 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
[16/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-8998: uniqueBlock()
aggreagation for singlevalue string fields in json.facet
Posted by ab...@apache.org.
SOLR-8998: uniqueBlock() aggreagation for singlevalue string fields in json.facet
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ee7b52f4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ee7b52f4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ee7b52f4
Branch: refs/heads/jira/solr-11779
Commit: ee7b52f4c6fe55f0d07ce8228c246b61d1f2b5fb
Parents: d92b891
Author: Mikhail Khludnev <mk...@apache.org>
Authored: Tue May 1 20:19:15 2018 +0300
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Tue May 1 20:19:15 2018 +0300
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../apache/solr/search/ValueSourceParser.java | 8 ++
.../solr/search/facet/UniqueBlockAgg.java | 91 ++++++++++++++++++++
.../search/facet/UniqueSinglevaluedSlotAcc.java | 16 ++--
.../apache/solr/search/QueryEqualityTest.java | 1 +
.../facet/TestJsonFacetsWithNestedObjects.java | 26 ++++++
.../search/join/BlockJoinFacetDistribTest.java | 77 +++++++++++++++--
7 files changed, 208 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee7b52f4/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ea166fb..f245002 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -105,6 +105,9 @@ New Features
* SOLR-11924: Added the ability to listen to changes in the set of active collections in a cloud
in the ZkStateReader, through the CloudCollectionsListener. (Houston Putman, Dennis Gove)
+* SOLR-8998: introducing uniqueBlock(_root_) aggregation as faster alternative to unique(_root_) for counting
+ child value facets in parents via json.facet on block index (Dr Oleg Savrasov, Mikhail Khludnev)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee7b52f4/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
index 0e26bf8..683cf4a 100644
--- a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
+++ b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
@@ -64,6 +64,7 @@ import org.apache.solr.search.facet.StddevAgg;
import org.apache.solr.search.facet.SumAgg;
import org.apache.solr.search.facet.SumsqAgg;
import org.apache.solr.search.facet.UniqueAgg;
+import org.apache.solr.search.facet.UniqueBlockAgg;
import org.apache.solr.search.facet.VarianceAgg;
import org.apache.solr.search.function.CollapseScoreFunction;
import org.apache.solr.search.function.ConcatStringFunction;
@@ -964,6 +965,13 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
}
});
+ addParser("agg_uniqueBlock", new ValueSourceParser() {
+ @Override
+ public ValueSource parse(FunctionQParser fp) throws SyntaxError {
+ return new UniqueBlockAgg(fp.parseArg());
+ }
+ });
+
addParser("agg_hll", new ValueSourceParser() {
@Override
public ValueSource parse(FunctionQParser fp) throws SyntaxError {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee7b52f4/solr/core/src/java/org/apache/solr/search/facet/UniqueBlockAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueBlockAgg.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueBlockAgg.java
new file mode 100644
index 0000000..c2bfec7
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueBlockAgg.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.facet;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.solr.schema.SchemaField;
+
+public class UniqueBlockAgg extends UniqueAgg {
+
+ private static final class UniqueBlockSlotAcc extends UniqueSinglevaluedSlotAcc {
+
+ private int lastSeenValuesPerSlot[];
+
+ private UniqueBlockSlotAcc(FacetContext fcontext, SchemaField field, int numSlots)
+ throws IOException { //
+ super(fcontext, field, /*numSlots suppressing inherited accumulator */0, null);
+ counts = new int[numSlots];
+ lastSeenValuesPerSlot = new int[numSlots];
+ Arrays.fill(lastSeenValuesPerSlot, Integer.MIN_VALUE);
+ }
+
+ @Override
+ protected void collectOrdToSlot(int slotNum, int ord) {
+ if (lastSeenValuesPerSlot[slotNum]!=ord) {
+ counts[slotNum]+=1;
+ lastSeenValuesPerSlot[slotNum] = ord;
+ }
+ }
+
+ @Override
+ public void calcCounts() {
+ // noop already done
+ }
+
+ @Override
+ public void reset() throws IOException {
+ Arrays.fill(counts, 0);
+ Arrays.fill(lastSeenValuesPerSlot, Integer.MIN_VALUE);
+ }
+
+ @Override
+ public Object getValue(int slot) throws IOException {
+ return counts[slot];
+ }
+ }
+
+ private final static String uniqueBlock = "uniqueBlock";
+
+ public UniqueBlockAgg(String field) {
+ super(field);
+ name= uniqueBlock;
+ }
+
+ @Override
+ public SlotAcc createSlotAcc(FacetContext fcontext, int numDocs, int numSlots) throws IOException {
+ final String fieldName = getArg();
+ SchemaField sf = fcontext.qcontext.searcher().getSchema().getField(fieldName);
+ if (sf.multiValued() || sf.getType().multiValuedFieldCache()) {
+ throw new IllegalArgumentException(uniqueBlock+"("+fieldName+
+ ") doesn't allow multivalue fields, got " + sf);
+ } else {
+ if (sf.getType().getNumberType() != null) {
+ throw new IllegalArgumentException(uniqueBlock+"("+fieldName+
+ ") not yet support numbers " + sf);
+ } else {
+ return new UniqueBlockSlotAcc(fcontext, sf, numSlots);
+ }
+ }
+ }
+
+ @Override
+ public FacetMerger createFacetMerger(Object prototype) {
+ return new FacetLongMerger() ;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee7b52f4/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
index 9efa1e9..9a1b51e 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java
@@ -81,12 +81,16 @@ class UniqueSinglevaluedSlotAcc extends UniqueSlotAcc {
int segOrd = subDv.ordValue();
int ord = toGlobal==null ? segOrd : (int)toGlobal.get(segOrd);
- FixedBitSet bits = arr[slotNum];
- if (bits == null) {
- bits = new FixedBitSet(nTerms);
- arr[slotNum] = bits;
- }
- bits.set(ord);
+ collectOrdToSlot(slotNum, ord);
}
}
+
+ protected void collectOrdToSlot(int slotNum, int ord) {
+ FixedBitSet bits = arr[slotNum];
+ if (bits == null) {
+ bits = new FixedBitSet(nTerms);
+ arr[slotNum] = bits;
+ }
+ bits.set(ord);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee7b52f4/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
index 6112303..d562076 100644
--- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
+++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
@@ -1162,6 +1162,7 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
assertFuncEquals("agg_sum(foo_i)", "agg_sum(foo_i)");
assertFuncEquals("agg_count()", "agg_count()");
assertFuncEquals("agg_unique(foo_i)", "agg_unique(foo_i)");
+ assertFuncEquals("agg_uniqueBlock(foo_i)", "agg_uniqueBlock(foo_i)");
assertFuncEquals("agg_hll(foo_i)", "agg_hll(foo_i)");
assertFuncEquals("agg_sumsq(foo_i)", "agg_sumsq(foo_i)");
assertFuncEquals("agg_percentile(foo_i,50)", "agg_percentile(foo_i,50)");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee7b52f4/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetsWithNestedObjects.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetsWithNestedObjects.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetsWithNestedObjects.java
index 5a638ac..cb8b71a 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetsWithNestedObjects.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacetsWithNestedObjects.java
@@ -334,4 +334,30 @@ public class TestJsonFacetsWithNestedObjects extends SolrTestCaseHS{
" ]}}"
);
}
+
+ public void testUniqueBlock() throws Exception {
+ final Client client = Client.localClient();
+ ModifiableSolrParams p = params("rows","0");
+ client.testJQ(params(p, "q", "{!parent tag=top which=type_s:book v=$childquery}"
+ , "childquery", "comment_t:*"
+ , "fl", "id", "fl" , "title_t"
+ , "json.facet", "{" +
+ " types: {" +
+ " domain: { blockChildren:\"type_s:book\"" +
+ " }," +
+ " type:terms," +
+ " field:type_s,"
+ + " limit:-1," +
+ " facet: {" +
+ " in_books: \"uniqueBlock(_root_)\" }"+//}}," +
+ " }" +
+ "}" )
+
+ , "response=={numFound:2,start:0,docs:[]}"
+ , "facets=={ count:2," +
+ "types:{" +
+ " buckets:[ {val:review, count:5, in_books:2} ]}" +
+ "}"
+ );
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee7b52f4/solr/core/src/test/org/apache/solr/search/join/BlockJoinFacetDistribTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/join/BlockJoinFacetDistribTest.java b/solr/core/src/test/org/apache/solr/search/join/BlockJoinFacetDistribTest.java
index 123ce97..c9d63c0 100644
--- a/solr/core/src/test/org/apache/solr/search/join/BlockJoinFacetDistribTest.java
+++ b/solr/core/src/test/org/apache/solr/search/join/BlockJoinFacetDistribTest.java
@@ -43,6 +43,7 @@ import org.junit.Test;
public class BlockJoinFacetDistribTest extends SolrCloudTestCase{
+ private static final int defFacetLimit = 10;
private static final String collection = "facetcollection";
@BeforeClass
@@ -74,7 +75,8 @@ public class BlockJoinFacetDistribTest extends SolrCloudTestCase{
"fuchsia", "light","dark","green","grey","don't","know","any","more" );
final static List<String> sizes = Arrays.asList("s","m","l","xl","xxl","xml","xxxl","3","4","5","6","petite","maxi");
- @Test
+ @SuppressWarnings("unchecked")
+ @Test
public void testBJQFacetComponent() throws Exception {
assert ! colors.removeAll(sizes): "there is no colors in sizes";
@@ -126,12 +128,20 @@ public class BlockJoinFacetDistribTest extends SolrCloudTestCase{
if (!parentDocs.isEmpty()) {
indexDocs(parentDocs);
}
- cluster.getSolrClient().commit(collection);
-
+ if (random().nextBoolean()) {
+ cluster.getSolrClient().commit(collection);
+ } else {
+ cluster.getSolrClient().optimize(collection);
+ }
// to parent query
- final String childQueryClause = "COLOR_s:("+(matchingColors.toString().replaceAll("[,\\[\\]]", " "))+")";
+ final String matchingColorsCommaSep = matchingColors.toString().replaceAll("[ \\[\\]]", "");
+ final String childQueryClause = "{!terms f=COLOR_s}" + matchingColorsCommaSep;
final boolean oldFacetsEnabled = random().nextBoolean();
- QueryResponse results = query("q", "{!parent which=\"type_s:parent\"}"+childQueryClause,
+ final boolean limitJsonSizes = random().nextBoolean();
+ final boolean limitJsonColors = random().nextBoolean();
+
+ QueryResponse results = query("q", "{!parent which=\"type_s:parent\" v=$matchingColors}",//+childQueryClause,
+ "matchingColors", childQueryClause,
"facet", oldFacetsEnabled ? "true":"false", // try to enforce multiple phases
oldFacetsEnabled ? "facet.field" : "ignore" , "BRAND_s",
oldFacetsEnabled&&usually() ? "facet.limit" : "ignore" , "1",
@@ -141,7 +151,18 @@ public class BlockJoinFacetDistribTest extends SolrCloudTestCase{
"child.facet.field", "COLOR_s",
"child.facet.field", "SIZE_s",
"distrib.singlePass", random().nextBoolean() ? "true":"false",
- "rows", random().nextBoolean() ? "0":"10"
+ "rows", random().nextBoolean() ? "0":"10",
+ "json.facet","{ "
+ + "children:{ type: query, query:\"*:*\", domain:{"
+ +"blockChildren:\"type_s:parent\", filter:{param:matchingColors}"
+ + "}, facet:{ colors:{ type:field, field:COLOR_s,"
+ + (limitJsonColors ? "":" limit:-1,")
+ + " facet:{ inprods:\"uniqueBlock(_root_)\"}}, "
+ + "sizes:{type:field, field:SIZE_s, "
+ + (limitJsonSizes ? "" : "limit:-1,")
+ + " facet:{inprods:\"uniqueBlock(_root_)\"}}"
+ + "}"
+ + "}}", "debugQuery","true"//, "shards", "shard1"
);
NamedList<Object> resultsResponse = results.getResponse();
assertNotNull(resultsResponse);
@@ -155,9 +176,49 @@ public class BlockJoinFacetDistribTest extends SolrCloudTestCase{
parentIdsByAttrValue.get(c.getName()).size(), c.getCount());
}
}
-
+
assertEquals(msg , parentIdsByAttrValue.size(),color_s.getValueCount() + size_s.getValueCount());
- //System.out.println(parentIdsByAttrValue);
+
+ final List<NamedList<Object>> jsonSizes = (List<NamedList<Object>>)
+ get(resultsResponse, "facets", "children", "sizes", "buckets");
+ final List<NamedList<Object>> jsonColors = (List<NamedList<Object>>)
+ get(resultsResponse, "facets", "children", "colors", "buckets");
+
+ if (limitJsonColors) {
+ assertTrue(""+jsonColors, jsonColors.size()<=defFacetLimit);
+ }
+
+ if (limitJsonSizes) {
+ assertTrue(""+jsonSizes, jsonSizes.size()<=defFacetLimit);
+ }
+
+ for (List<NamedList<Object>> vals : new List[] { jsonSizes,jsonColors}) {
+ int i=0;
+ for(NamedList<Object> tuples: vals) {
+ String val = (String) get(tuples,"val");
+ Number count = (Number) get(tuples,"inprods");
+ if (((vals==jsonSizes && limitJsonSizes) || // vals close to the limit are not exact
+ (vals==jsonColors && limitJsonColors)) && i>=defFacetLimit/2) {
+ assertTrue(i+ "th "+tuples+". "+vals,
+ parentIdsByAttrValue.get(val).size()>= count.intValue() &&
+ count.intValue()>0);
+ } else {
+ assertEquals(tuples+". "+vals,
+ parentIdsByAttrValue.get(val).size(),count.intValue());
+ }
+ i++;
+ }
+ }
+ if (!limitJsonColors && !limitJsonSizes) {
+ assertEquals(""+jsonSizes+jsonColors, parentIdsByAttrValue.size(),jsonSizes.size() + jsonColors.size());
+ }
+ }
+
+ private static Object get(Object nvList, String ... segments) {
+ for(String segment: segments) {
+ nvList = ((NamedList<Object>) nvList).get(segment);
+ }
+ return nvList;
}
private QueryResponse query(String ... arg) throws SolrServerException, IOException {
[35/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8295: Remove
useless liveDocsSharedPending flag.
Posted by ab...@apache.org.
LUCENE-8295: Remove useless liveDocsSharedPending flag.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ad0ad2ec
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ad0ad2ec
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ad0ad2ec
Branch: refs/heads/jira/solr-11779
Commit: ad0ad2ec891b7c122ddf5b0385a9e4b984e78b38
Parents: 3a6f531
Author: Adrien Grand <jp...@gmail.com>
Authored: Fri May 4 14:38:42 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Fri May 4 14:38:42 2018 +0200
----------------------------------------------------------------------
.../src/java/org/apache/lucene/index/ReadersAndUpdates.java | 9 ---------
1 file changed, 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ad0ad2ec/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index b33ac4b..a02d26a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -88,10 +88,6 @@ final class ReadersAndUpdates {
final AtomicLong ramBytesUsed = new AtomicLong();
- // if set to true the pending deletes must be marked as shared next time the reader is
- // returned from #getReader()
- private boolean liveDocsSharedPending = false;
-
ReadersAndUpdates(int indexCreatedVersionMajor, SegmentCommitInfo info,
PendingDeletes pendingDeletes) {
this.info = info;
@@ -196,8 +192,6 @@ final class ReadersAndUpdates {
// We steal returned ref:
reader = new SegmentReader(info, indexCreatedVersionMajor, context);
pendingDeletes.onNewReader(reader, info);
- } else if (liveDocsSharedPending) {
- markAsShared();
}
// Ref for caller
@@ -224,7 +218,6 @@ final class ReadersAndUpdates {
} finally {
reader = null;
}
- liveDocsSharedPending = false;
}
decRef();
@@ -678,7 +671,6 @@ final class ReadersAndUpdates {
private void swapNewReaderWithLatestLiveDocs() throws IOException {
reader = createNewReaderWithLatestLiveDocs(reader);
- liveDocsSharedPending = true;
}
synchronized void setIsMerging() {
@@ -764,7 +756,6 @@ final class ReadersAndUpdates {
private final void markAsShared() {
assert Thread.holdsLock(this);
- liveDocsSharedPending = false;
pendingDeletes.liveDocsShared(); // this is not costly we can just call it even if it's already marked as shared
}
[24/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-8998: ref guide
update.
Posted by ab...@apache.org.
SOLR-8998: ref guide update.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/df713fc7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/df713fc7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/df713fc7
Branch: refs/heads/jira/solr-11779
Commit: df713fc70009733afed84484298326b15f963d15
Parents: 46ecb73
Author: Mikhail Khludnev <mk...@apache.org>
Authored: Wed May 2 18:23:08 2018 +0300
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Wed May 2 18:23:15 2018 +0300
----------------------------------------------------------------------
solr/solr-ref-guide/src/blockjoin-faceting.adoc | 2 +
solr/solr-ref-guide/src/json-facet-api.adoc | 69 +++++++++++++++++++-
solr/solr-ref-guide/src/other-parsers.adoc | 2 +-
3 files changed, 70 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df713fc7/solr/solr-ref-guide/src/blockjoin-faceting.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/blockjoin-faceting.adoc b/solr/solr-ref-guide/src/blockjoin-faceting.adoc
index 41030d5..7e2408b 100644
--- a/solr/solr-ref-guide/src/blockjoin-faceting.adoc
+++ b/solr/solr-ref-guide/src/blockjoin-faceting.adoc
@@ -20,6 +20,8 @@ BlockJoin facets allow you to aggregate children facet counts by their parents.
It is a common requirement that if a parent document has several children documents, all of them need to increment facet value count only once. This functionality is provided by `BlockJoinDocSetFacetComponent`, and `BlockJoinFacetComponent` just an alias for compatibility.
+CAUTION: This functionality is considered deprecated. Users are encouraged to use `uniqueBlock(\_root_)` aggregation under terms facet in <<json-facet-api.adoc#Blockjoinfacetexample,JSON Facet API>>.
+
CAUTION: This component is considered experimental, and must be explicitly enabled for a request handler in `solrconfig.xml`, in the same way as any other <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#requesthandlers-and-searchcomponents-in-solrconfig,search component>>.
This example shows how you could add this search components to `solrconfig.xml` and define it in request handler:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df713fc7/solr/solr-ref-guide/src/json-facet-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/json-facet-api.adoc b/solr/solr-ref-guide/src/json-facet-api.adoc
index 90e197e..e71c8bd 100644
--- a/solr/solr-ref-guide/src/json-facet-api.adoc
+++ b/solr/solr-ref-guide/src/json-facet-api.adoc
@@ -59,7 +59,7 @@ The response to the facet request above will start with documents matching the r
[[BucketingFacetExample]]
=== Bucketing Facet Example
-Here's an example of a bucketing facet, that partitions documents into bucket based on the `cat` field (short for category), and returns the top 5 buckets:
+Here's an example of a bucketing facet, that partitions documents into bucket based on the `cat` field (short for category), and returns the top 3 buckets:
[source,bash]
----
@@ -342,7 +342,8 @@ Aggregation functions, also called *facet functions, analytic functions,* or **m
|avg |avg(popularity) |average of numeric values
|min |min(salary) |minimum value
|max |max(mul(price,popularity)) |maximum value
-|unique |unique(author) |number of unique values
+|unique |unique(author) |number of unique values of the given field. Beyond 100 values it yields not exact estimate
+|uniqueBlock |uniqueBlock(\_root_) |same as above with smaller footprint strictly requires <<uploading-data-with-index-handlers.adoc#nested-child-documents, block index>>. The given field is expected to be unique across blocks, now only singlevalued string fields are supported, docValues are recommended.
|hll |hll(author) |distributed cardinality estimate via hyper-log-log algorithm
|percentile |percentile(salary,50,75,99,99.9) |Percentile estimates via t-digest algorithm. When sorting by this metric, the first percentile listed is used as the sort value.
|sumsq |sumsq(rent) |sum of squares of field or function
@@ -449,6 +450,70 @@ And the response will look something like:
By default "top authors" is defined by simple document count descending, but we could use our aggregation functions to sort by more interesting metrics.
+
+[[BlockJoinFacets]]
+== Block Join Facets
+
+Block Join Facets facets allow bucketing <<uploading-data-with-index-handlers.adoc#nested-child-documents, child documents>> as attributes of their parents.
+
+[[Blockjoinfacetexample]]
+=== Block Join Facet example
+
+Suppose we have products with multiple SKUs, and we want to count products for each color.
+
+[source,java]
+----
+{
+ "id": "1", "type": "product", "name": "Solr T-Shirt",
+ "_childDocuments_": [
+ { "id": "11", "type": "SKU", "color": "Red", "size": "L" },
+ { "id": "12", "type": "SKU", "color": "Blue", "size": "L" },
+ { "id": "13", "type": "SKU", "color": "Red", "size": "M" },
+ { "id": "14", "type": "SKU", "color": "Blue", "size": "S" }
+ ]
+ }
+
+----
+
+For *SKU domain* we can request
+
+[source,java]
+----
+ color: {
+ type: terms,
+ field: color,
+ limit: -1,
+ facet: {
+ productsCount: "uniqueBlock(_root_)"
+ }
+ }
+
+
+----
+
+and get
+
+[source,java]
+----
+
+ [...]
+ color:{
+ buckets:[
+ {
+ val:Red, count:2, productsCount:1
+ },
+ {
+ val:Blue, count:2, productsCount:1
+ }
+ ]
+ }
+----
+
+Please notice that `\_root_` is an internal field added by Lucene to each child document to reference on parent one.
+Aggregation `uniqueBlock(\_root_)` is functionally equivalent to `unique(\_root_)`, but is optimized for nested documents block structure.
+It's recommended to define `limit: -1` for `uniqueBlock` calculation, like in above example,
+since default value of `limit` parameter is `10`, while `uniqueBlock` is supposed to be much faster with `-1`.
+
[[References]]
== References
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df713fc7/solr/solr-ref-guide/src/other-parsers.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/other-parsers.adoc b/solr/solr-ref-guide/src/other-parsers.adoc
index 1b82883..bcd28eb 100644
--- a/solr/solr-ref-guide/src/other-parsers.adoc
+++ b/solr/solr-ref-guide/src/other-parsers.adoc
@@ -24,7 +24,7 @@ Many of these parsers are expressed the same way as <<local-parameters-in-querie
== Block Join Query Parsers
-There are two query parsers that support block joins. These parsers allow indexing and searching for relational content that has been <<uploading-data-with-index-handlers.adoc#uploading-data-with-index-handlers,indexed as nested documents>>.
+There are two query parsers that support block joins. These parsers allow indexing and searching for relational content that has been <<uploading-data-with-index-handlers.adoc#nested-child-documents, indexed as nested documents>>.
The example usage of the query parsers below assumes these two documents and each of their child documents have been indexed:
[04/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8280: Use a
combination of strategies to work around the fact that no strategy can be
used in all situations.
Posted by ab...@apache.org.
LUCENE-8280: Use a combination of strategies to work around the fact that no strategy can be used in all situations.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c94a83fd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c94a83fd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c94a83fd
Branch: refs/heads/jira/solr-11779
Commit: c94a83fd283bb7d081e7b3c1578e863a6c58363a
Parents: e263ae30
Author: Karl Wright <Da...@gmail.com>
Authored: Sat Apr 28 09:06:44 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Sat Apr 28 09:06:44 2018 -0400
----------------------------------------------------------------------
.../spatial3d/geom/GeoComplexPolygon.java | 559 ++++++++++++++++---
.../lucene/spatial3d/geom/GeoPolygonTest.java | 27 +-
2 files changed, 502 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c94a83fd/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
index 873c3b9..4b82897 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
@@ -19,6 +19,8 @@ package org.apache.lucene.spatial3d.geom;
import java.util.Arrays;
import java.util.List;
import java.util.ArrayList;
+import java.util.Set;
+import java.util.HashSet;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
@@ -323,30 +325,21 @@ class GeoComplexPolygon extends GeoBasePolygon {
//System.out.println(" Using XZ plane alone");
final CountingEdgeIterator crossingEdgeIterator = createLinearCrossingEdgeIterator(testPoint, testPointFixedYPlane, testPointFixedYAbovePlane, testPointFixedYBelowPlane, x, y, z);
// Traverse our way from the test point to the check point. Use the y tree because that's fixed.
- if (!yTree.traverse(crossingEdgeIterator, testPoint.y)) {
- // Endpoint is on edge
- return true;
- }
- return ((crossingEdgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet;
+ yTree.traverse(crossingEdgeIterator, testPoint.y);
+ return crossingEdgeIterator.isOnEdge() || (((crossingEdgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet);
} else if (testPointFixedXAbovePlane != null && testPointFixedXBelowPlane != null && testPointFixedXPlane.evaluateIsZero(x, y, z)) {
// Use the YZ plane exclusively.
//System.out.println(" Using YZ plane alone");
final CountingEdgeIterator crossingEdgeIterator = createLinearCrossingEdgeIterator(testPoint, testPointFixedXPlane, testPointFixedXAbovePlane, testPointFixedXBelowPlane, x, y, z);
// Traverse our way from the test point to the check point. Use the x tree because that's fixed.
- if (!xTree.traverse(crossingEdgeIterator, testPoint.x)) {
- // Endpoint is on edge
- return true;
- }
- return ((crossingEdgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet;
+ xTree.traverse(crossingEdgeIterator, testPoint.x);
+ return crossingEdgeIterator.isOnEdge() || (((crossingEdgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet);
} else if (testPointFixedZAbovePlane != null && testPointFixedZBelowPlane != null && testPointFixedZPlane.evaluateIsZero(x, y, z)) {
//System.out.println(" Using XY plane alone");
final CountingEdgeIterator crossingEdgeIterator = createLinearCrossingEdgeIterator(testPoint, testPointFixedZPlane, testPointFixedZAbovePlane, testPointFixedZBelowPlane, x, y, z);
// Traverse our way from the test point to the check point. Use the z tree because that's fixed.
- if (!zTree.traverse(crossingEdgeIterator, testPoint.z)) {
- // Endpoint is on edge
- return true;
- }
- return ((crossingEdgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet;
+ zTree.traverse(crossingEdgeIterator, testPoint.z);
+ return crossingEdgeIterator.isOnEdge() || (((crossingEdgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet);
} else {
//System.out.println(" Using two planes");
// This is the expensive part!!
@@ -618,40 +611,49 @@ class GeoComplexPolygon extends GeoBasePolygon {
assert bestDistance > 0.0 : "Best distance should not be zero unless on single plane";
assert bestDistance < Double.POSITIVE_INFINITY : "Couldn't find an intersection point of any kind";
- // First, we'll determine if the intersection point is in set or not
- //System.out.println(" Finding whether "+intersectionPoint+" is in-set, based on travel from "+testPoint+" along "+firstLegPlane+" (value="+firstLegValue+")");
- final boolean intersectionPointInSet;
- final CountingEdgeIterator testPointEdgeIterator = createLinearCrossingEdgeIterator(testPoint,
- firstLegPlane, firstLegAbovePlane, firstLegBelowPlane,
- intersectionPoint.x, intersectionPoint.y, intersectionPoint.z);
- // Traverse our way from the test point to the check point. Use the z tree because that's fixed.
- if (!firstLegTree.traverse(testPointEdgeIterator, firstLegValue)) {
- // Endpoint is on edge
- //System.out.println(" Landed on edge -- in-set");
- intersectionPointInSet = true;
- } else {
- intersectionPointInSet = ((testPointEdgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet;
- }
-
- //System.out.println(" Intersection point in-set? "+intersectionPointInSet);
-
- // Now do the final leg
- //System.out.println(" Finding whether ["+x+","+y+","+z+"] is in-set, based on travel from "+intersectionPoint+" along "+secondLegPlane+" (value="+secondLegValue+")");
- final boolean rval;
- final CountingEdgeIterator travelEdgeIterator = createLinearCrossingEdgeIterator(intersectionPoint,
- secondLegPlane, secondLegAbovePlane, secondLegBelowPlane,
- x, y, z);
- // Traverse our way from the test point to the check point.
- if (!secondLegTree.traverse(travelEdgeIterator, secondLegValue)) {
- // Endpoint is on edge
- //System.out.println(" Landed on edge -- in-set");
- rval = true;
- } else {
- rval = ((travelEdgeIterator.getCrossingCount() & 1) == 0)?intersectionPointInSet:!intersectionPointInSet;
+ // First, try with two individual legs. If that doesn't work, try the DualCrossingIterator.
+ try {
+ // First, we'll determine if the intersection point is in set or not
+ //System.out.println(" Finding whether "+intersectionPoint+" is in-set, based on travel from "+testPoint+" along "+firstLegPlane+" (value="+firstLegValue+")");
+ final CountingEdgeIterator testPointEdgeIterator = createLinearCrossingEdgeIterator(testPoint,
+ firstLegPlane, firstLegAbovePlane, firstLegBelowPlane,
+ intersectionPoint.x, intersectionPoint.y, intersectionPoint.z);
+ // Traverse our way from the test point to the check point. Use the z tree because that's fixed.
+ firstLegTree.traverse(testPointEdgeIterator, firstLegValue);
+ final boolean intersectionPointOnEdge = testPointEdgeIterator.isOnEdge();
+ // If the intersection point is on the edge, we cannot use this combination of legs, since it's not logically possible to compute in-set or out-of-set
+ // with such a starting point.
+ if (intersectionPointOnEdge) {
+ throw new IllegalArgumentException("Intersection point landed on an edge -- illegal path");
+ }
+ final boolean intersectionPointInSet = intersectionPointOnEdge || (((testPointEdgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet);
+
+ //System.out.println(" Intersection point in-set? "+intersectionPointInSet+" On edge? "+intersectionPointOnEdge);
+
+ // Now do the final leg
+ //System.out.println(" Finding whether ["+x+","+y+","+z+"] is in-set, based on travel from "+intersectionPoint+" along "+secondLegPlane+" (value="+secondLegValue+")");
+ final CountingEdgeIterator travelEdgeIterator = createLinearCrossingEdgeIterator(intersectionPoint,
+ secondLegPlane, secondLegAbovePlane, secondLegBelowPlane,
+ x, y, z);
+ // Traverse our way from the test point to the check point.
+ secondLegTree.traverse(travelEdgeIterator, secondLegValue);
+ final boolean rval = travelEdgeIterator.isOnEdge() || (((travelEdgeIterator.getCrossingCount() & 1) == 0)?intersectionPointInSet:!intersectionPointInSet);
+
+ //System.out.println(" Check point in set? "+rval);
+ return rval;
+ } catch (IllegalArgumentException e) {
+ // Intersection point apparently was on edge, so try another strategy
+ final CountingEdgeIterator edgeIterator = new DualCrossingEdgeIterator(testPoint,
+ firstLegPlane, firstLegAbovePlane, firstLegBelowPlane,
+ secondLegPlane, secondLegAbovePlane, secondLegBelowPlane,
+ x, y, z, intersectionPoint);
+ firstLegTree.traverse(edgeIterator, firstLegValue);
+ if (edgeIterator.isOnEdge()) {
+ return true;
+ }
+ secondLegTree.traverse(edgeIterator, secondLegValue);
+ return edgeIterator.isOnEdge() || (((edgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet);
}
-
- //System.out.println(" Check point in set? "+rval);
- return rval;
}
}
@@ -764,7 +766,8 @@ class GeoComplexPolygon extends GeoBasePolygon {
/** Create a linear crossing edge iterator with the appropriate cutoff planes given the geometry.
*/
private CountingEdgeIterator createLinearCrossingEdgeIterator(final GeoPoint testPoint,
- final Plane plane, final Plane abovePlane, final Plane belowPlane, final double thePointX, final double thePointY, final double thePointZ) {
+ final Plane plane, final Plane abovePlane, final Plane belowPlane,
+ final double thePointX, final double thePointY, final double thePointZ) {
// If thePoint and testPoint are parallel, we won't be able to determine sidedness of the bounding planes. So detect that case, and build the iterator differently if we find it.
// This didn't work; not sure why not:
//if (testPoint.isParallel(thePointX, thePointY, thePointZ)) {
@@ -844,6 +847,12 @@ class GeoComplexPolygon extends GeoBasePolygon {
* @return the number of edges that were crossed.
*/
public int getCrossingCount();
+
+ /**
+ * @return true if the endpoint was on an edge.
+ */
+ public boolean isOnEdge();
+
}
/**
@@ -1155,11 +1164,13 @@ class GeoComplexPolygon extends GeoBasePolygon {
private final double thePointY;
private final double thePointZ;
+ private boolean onEdge = false;
private int aboveCrossingCount = 0;
private int belowCrossingCount = 0;
public FullLinearCrossingEdgeIterator(final GeoPoint testPoint,
- final Plane plane, final Plane abovePlane, final Plane belowPlane, final double thePointX, final double thePointY, final double thePointZ) {
+ final Plane plane, final Plane abovePlane, final Plane belowPlane,
+ final double thePointX, final double thePointY, final double thePointZ) {
assert plane.evaluateIsZero(thePointX, thePointY, thePointZ) : "Check point is not on travel plane";
assert plane.evaluateIsZero(testPoint) : "Test point is not on travel plane";
this.testPoint = testPoint;
@@ -1180,19 +1191,21 @@ class GeoComplexPolygon extends GeoBasePolygon {
@Override
public int getCrossingCount() {
- if (aboveCrossingCount < belowCrossingCount) {
- return aboveCrossingCount;
- } else {
- return belowCrossingCount;
- }
+ return Math.min(aboveCrossingCount, belowCrossingCount);
}
-
+
+ @Override
+ public boolean isOnEdge() {
+ return onEdge;
+ }
+
@Override
public boolean matches(final Edge edge) {
//System.out.println(" Edge ["+edge.startPoint+" --> "+edge.endPoint+"] potentially crosses travel plane "+plane);
// Early exit if the point is on the edge.
if (edge.isWithin(thePointX, thePointY, thePointZ)) {
//System.out.println(" Point is on the edge; in-set");
+ onEdge = true;
return false;
}
@@ -1206,11 +1219,14 @@ class GeoComplexPolygon extends GeoBasePolygon {
}
}
- //System.out.println(" Edge ["+edge.startPoint+" --> "+edge.endPoint+"] intersects travel plane "+plane);
+ //System.out.println(" Edge intersects travel plane "+plane);
// Determine crossings of this edge against all inside/outside planes. There's no further need to look at the actual travel plane itself.
- aboveCrossingCount += countCrossings(edge, abovePlane, bound);
- belowCrossingCount += countCrossings(edge, belowPlane, bound);
+ final int aboveCrossings = countCrossings(edge, abovePlane, bound);
+ aboveCrossingCount += aboveCrossings;
+ final int belowCrossings = countCrossings(edge, belowPlane, bound);
+ belowCrossingCount += belowCrossings;
+ //System.out.println(" Above crossings = "+aboveCrossings+"; below crossings = "+belowCrossings);
return true;
}
@@ -1264,11 +1280,13 @@ class GeoComplexPolygon extends GeoBasePolygon {
private final double thePointY;
private final double thePointZ;
+ private boolean onEdge = false;
private int aboveCrossingCount = 0;
private int belowCrossingCount = 0;
public SectorLinearCrossingEdgeIterator(final GeoPoint testPoint,
- final Plane plane, final Plane abovePlane, final Plane belowPlane, final double thePointX, final double thePointY, final double thePointZ) {
+ final Plane plane, final Plane abovePlane, final Plane belowPlane,
+ final double thePointX, final double thePointY, final double thePointZ) {
assert plane.evaluateIsZero(thePointX, thePointY, thePointZ) : "Check point is not on travel plane";
assert plane.evaluateIsZero(testPoint) : "Test point is not on travel plane";
this.testPoint = testPoint;
@@ -1293,11 +1311,12 @@ class GeoComplexPolygon extends GeoBasePolygon {
@Override
public int getCrossingCount() {
- if (aboveCrossingCount < belowCrossingCount) {
- return aboveCrossingCount;
- } else {
- return belowCrossingCount;
- }
+ return Math.min(aboveCrossingCount, belowCrossingCount);
+ }
+
+ @Override
+ public boolean isOnEdge() {
+ return onEdge;
}
@Override
@@ -1307,6 +1326,7 @@ class GeoComplexPolygon extends GeoBasePolygon {
if (edge.isWithin(thePointX, thePointY, thePointZ)) {
// The point is on the edge. This means it's "in-set" by definition, so abort.
//System.out.println(" Point is on the edge; in-set");
+ onEdge = true;
return false;
}
@@ -1349,11 +1369,16 @@ class GeoComplexPolygon extends GeoBasePolygon {
//System.out.println(" There were intersection points!");
}
- //System.out.println(" Edge intersects travel plane "+plane);
-
+ //System.out.println(" Edge intersects travel plane");
+
// Determine crossings of this edge against all inside/outside planes. There's no further need to look at the actual travel plane itself.
- aboveCrossingCount += countCrossings(edge, abovePlane, bound1, bound2);
- belowCrossingCount += countCrossings(edge, belowPlane, bound1, bound2);
+ //System.out.println(" Getting above crossings...");
+ final int aboveCrossings = countCrossings(edge, abovePlane, bound1, bound2);
+ aboveCrossingCount += aboveCrossings;
+ //System.out.println(" Getting below crossings...");
+ final int belowCrossings = countCrossings(edge, belowPlane, bound1, bound2);
+ belowCrossingCount += belowCrossings;
+ //System.out.println(" Above crossings = "+aboveCrossings+"; below crossings = "+belowCrossings);
return true;
}
@@ -1368,10 +1393,15 @@ class GeoComplexPolygon extends GeoBasePolygon {
if (intersections != null) {
for (final GeoPoint intersection : intersections) {
if (edge.startPlane.strictlyWithin(intersection) && edge.endPlane.strictlyWithin(intersection)) {
+ //System.out.println(" Envelope intersection point = "+intersection);
// It's unique, so assess it
- crossings += edgeCrossesEnvelope(edge.plane, intersection, envelopePlane)?1:0;
+ final int counter = edgeCrossesEnvelope(edge.plane, intersection, envelopePlane)?1:0;
+ //System.out.println(" Edge crosses envelope "+counter+" times");
+ crossings += counter;
}
}
+ } else {
+ //System.out.println(" Intersections = null");
}
return crossings;
}
@@ -1391,7 +1421,396 @@ class GeoComplexPolygon extends GeoBasePolygon {
}
}
+
+
+ /** Count the number of verifiable edge crossings for a dual-leg journey.
+ */
+ private class DualCrossingEdgeIterator implements CountingEdgeIterator {
+
+ // This is a hash of which edges we've already looked at and tallied, so we don't repeat ourselves.
+ // It is lazily initialized since most transitions cross no edges at all.
+ private Set<Edge> seenEdges = null;
+
+ private final GeoPoint testPoint;
+ private final Plane testPointPlane;
+ private final Plane testPointAbovePlane;
+ private final Plane testPointBelowPlane;
+ private final Plane travelPlane;
+ private final Plane travelAbovePlane;
+ private final Plane travelBelowPlane;
+ private final double thePointX;
+ private final double thePointY;
+ private final double thePointZ;
+
+ private final GeoPoint intersectionPoint;
+
+ private final SidedPlane testPointCutoffPlane;
+ private final SidedPlane checkPointCutoffPlane;
+ private final SidedPlane testPointOtherCutoffPlane;
+ private final SidedPlane checkPointOtherCutoffPlane;
+
+ // These are computed on an as-needed basis
+
+ private boolean computedInsideOutside = false;
+ private Plane testPointInsidePlane;
+ private Plane testPointOutsidePlane;
+ private Plane travelInsidePlane;
+ private Plane travelOutsidePlane;
+ private SidedPlane insideTestPointCutoffPlane;
+ private SidedPlane insideTravelCutoffPlane;
+ private SidedPlane outsideTestPointCutoffPlane;
+ private SidedPlane outsideTravelCutoffPlane;
+
+ // The counters
+ private boolean onEdge = false;
+ private int innerCrossingCount = 0;
+ private int outerCrossingCount = 0;
+
+ public DualCrossingEdgeIterator(final GeoPoint testPoint,
+ final Plane testPointPlane, final Plane testPointAbovePlane, final Plane testPointBelowPlane,
+ final Plane travelPlane, final Plane travelAbovePlane, final Plane travelBelowPlane,
+ final double thePointX, final double thePointY, final double thePointZ, final GeoPoint intersectionPoint) {
+ this.testPoint = testPoint;
+ this.testPointPlane = testPointPlane;
+ this.testPointAbovePlane = testPointAbovePlane;
+ this.testPointBelowPlane = testPointBelowPlane;
+ this.travelPlane = travelPlane;
+ this.travelAbovePlane = travelAbovePlane;
+ this.travelBelowPlane = travelBelowPlane;
+ this.thePointX = thePointX;
+ this.thePointY = thePointY;
+ this.thePointZ = thePointZ;
+ this.intersectionPoint = intersectionPoint;
+
+ //System.out.println("Intersection point = "+intersectionPoint);
+ //System.out.println("TestPoint plane: "+testPoint+" -> "+intersectionPoint);
+ //System.out.println("Travel plane: ["+thePointX+","+thePointY+","+thePointZ+"] -> "+intersectionPoint);
+
+ assert travelPlane.evaluateIsZero(intersectionPoint) : "intersection point must be on travel plane";
+ assert testPointPlane.evaluateIsZero(intersectionPoint) : "intersection point must be on test point plane";
+
+ //System.out.println("Test point distance to intersection point: "+intersectionPoint.linearDistance(testPoint));
+ //System.out.println("Check point distance to intersection point: "+intersectionPoint.linearDistance(thePointX, thePointY, thePointZ));
+
+ assert !testPoint.isNumericallyIdentical(intersectionPoint) : "test point is the same as intersection point";
+ assert !intersectionPoint.isNumericallyIdentical(thePointX, thePointY, thePointZ) : "check point is same as intersection point";
+
+ /*
+ final SidedPlane bound1Plane = new SidedPlane(thePointX, thePointY, thePointZ, plane, testPoint);
+ final SidedPlane bound2Plane = new SidedPlane(testPoint, plane, thePointX, thePointY, thePointZ);
+ if (bound1Plane.isNumericallyIdentical(bound2Plane)) {
+ throw new IllegalArgumentException("Sector iterator unreliable when bounds planes are numerically identical");
+ }
+ */
+
+ final SidedPlane testPointBound1 = new SidedPlane(intersectionPoint, testPointPlane, testPoint);
+ final SidedPlane testPointBound2 = new SidedPlane(testPoint, testPointPlane, intersectionPoint);
+ if (testPointBound1.isNumericallyIdentical(testPointBound2)) {
+ throw new IllegalArgumentException("Dual iterator unreliable when bounds planes are numerically identical");
+ }
+ this.testPointCutoffPlane = testPointBound1;
+ this.testPointOtherCutoffPlane = testPointBound2;
+
+ final SidedPlane checkPointBound1 = new SidedPlane(intersectionPoint, travelPlane, thePointX, thePointY, thePointZ);
+ final SidedPlane checkPointBound2 = new SidedPlane(thePointX, thePointY, thePointZ, travelPlane, intersectionPoint);
+ if (checkPointBound1.isNumericallyIdentical(checkPointBound2)) {
+ throw new IllegalArgumentException("Dual iterator unreliable when bounds planes are numerically identical");
+ }
+ this.checkPointCutoffPlane = checkPointBound1;
+ this.checkPointOtherCutoffPlane = checkPointBound2;
+
+ // Sanity check
+ assert testPointCutoffPlane.isWithin(intersectionPoint) : "intersection must be within testPointCutoffPlane";
+ assert testPointOtherCutoffPlane.isWithin(intersectionPoint) : "intersection must be within testPointOtherCutoffPlane";
+ assert checkPointCutoffPlane.isWithin(intersectionPoint) : "intersection must be within checkPointCutoffPlane";
+ assert checkPointOtherCutoffPlane.isWithin(intersectionPoint) : "intersection must be within checkPointOtherCutoffPlane";
+
+ }
+
+ protected void computeInsideOutside() {
+ if (!computedInsideOutside) {
+ // Convert travel plane to a sided plane
+ final Membership intersectionBound1 = new SidedPlane(testPoint, travelPlane, travelPlane.D);
+ // Convert testPoint plane to a sided plane
+ final Membership intersectionBound2 = new SidedPlane(thePointX, thePointY, thePointZ, testPointPlane, testPointPlane.D);
+
+ assert intersectionBound1.isWithin(intersectionPoint) : "intersection must be within intersectionBound1";
+ assert intersectionBound2.isWithin(intersectionPoint) : "intersection must be within intersectionBound2";
+
+ // Figure out which of the above/below planes are inside vs. outside. To do this,
+ // we look for the point that is within the bounds of the testPointPlane and travelPlane. The two sides that intersected there are the inside
+ // borders.
+ // Each of these can generate two solutions. We need to refine them to generate only one somehow -- the one in the same area of the world as intersectionPoint.
+ // Since the travel/testpoint planes have one fixed coordinate, and that is represented by the plane's D value, it should be possible to choose based on the
+ // point's coordinates.
+ final GeoPoint[] aboveAbove = travelAbovePlane.findIntersections(planetModel, testPointAbovePlane, intersectionBound1, intersectionBound2);
+ assert aboveAbove != null : "Above + above should not be coplanar";
+ final GeoPoint[] aboveBelow = travelAbovePlane.findIntersections(planetModel, testPointBelowPlane, intersectionBound1, intersectionBound2);
+ assert aboveBelow != null : "Above + below should not be coplanar";
+ final GeoPoint[] belowBelow = travelBelowPlane.findIntersections(planetModel, testPointBelowPlane, intersectionBound1, intersectionBound2);
+ assert belowBelow != null : "Below + below should not be coplanar";
+ final GeoPoint[] belowAbove = travelBelowPlane.findIntersections(planetModel, testPointAbovePlane, intersectionBound1, intersectionBound2);
+ assert belowAbove != null : "Below + above should not be coplanar";
+
+ assert ((aboveAbove.length > 0)?1:0) + ((aboveBelow.length > 0)?1:0) + ((belowBelow.length > 0)?1:0) + ((belowAbove.length > 0)?1:0) == 1 : "Can be exactly one inside point, instead was: aa="+aboveAbove.length+" ab=" + aboveBelow.length+" bb="+ belowBelow.length+" ba=" + belowAbove.length;
+
+ final GeoPoint[] insideInsidePoints;
+ if (aboveAbove.length > 0) {
+ travelInsidePlane = travelAbovePlane;
+ testPointInsidePlane = testPointAbovePlane;
+ travelOutsidePlane = travelBelowPlane;
+ testPointOutsidePlane = testPointBelowPlane;
+ insideInsidePoints = aboveAbove;
+ } else if (aboveBelow.length > 0) {
+ travelInsidePlane = travelAbovePlane;
+ testPointInsidePlane = testPointBelowPlane;
+ travelOutsidePlane = travelBelowPlane;
+ testPointOutsidePlane = testPointAbovePlane;
+ insideInsidePoints = aboveBelow;
+ } else if (belowBelow.length > 0) {
+ travelInsidePlane = travelBelowPlane;
+ testPointInsidePlane = testPointBelowPlane;
+ travelOutsidePlane = travelAbovePlane;
+ testPointOutsidePlane = testPointAbovePlane;
+ insideInsidePoints = belowBelow;
+ } else if (belowAbove.length > 0) {
+ travelInsidePlane = travelBelowPlane;
+ testPointInsidePlane = testPointAbovePlane;
+ travelOutsidePlane = travelAbovePlane;
+ testPointOutsidePlane = testPointBelowPlane;
+ insideInsidePoints = belowAbove;
+ } else {
+ throw new IllegalStateException("Can't find traversal intersection among: "+travelAbovePlane+", "+testPointAbovePlane+", "+travelBelowPlane+", "+testPointBelowPlane);
+ }
+
+ // Get the inside-inside intersection point
+ // Picking which point, out of two, that corresponds to the already-selected intersectionPoint, is tricky, but it must be done.
+ // We expect the choice to be within a small delta of the intersection point in 2 of the dimensions, but not the third
+ final GeoPoint insideInsidePoint = pickProximate(insideInsidePoints);
+
+ // Get the outside-outside intersection point
+ //System.out.println("Computing outside-outside intersection");
+ final GeoPoint[] outsideOutsidePoints = testPointOutsidePlane.findIntersections(planetModel, travelOutsidePlane); //these don't add anything: , checkPointCutoffPlane, testPointCutoffPlane);
+ final GeoPoint outsideOutsidePoint = pickProximate(outsideOutsidePoints);
+
+ insideTravelCutoffPlane = new SidedPlane(thePointX, thePointY, thePointZ, travelInsidePlane, insideInsidePoint);
+ outsideTravelCutoffPlane = new SidedPlane(thePointX, thePointY, thePointZ, travelInsidePlane, outsideOutsidePoint);
+ insideTestPointCutoffPlane = new SidedPlane(testPoint, testPointInsidePlane, insideInsidePoint);
+ outsideTestPointCutoffPlane = new SidedPlane(testPoint, testPointOutsidePlane, outsideOutsidePoint);
+
+ /*
+ System.out.println("insideTravelCutoffPlane = "+insideTravelCutoffPlane);
+ System.out.println("outsideTravelCutoffPlane = "+outsideTravelCutoffPlane);
+ System.out.println("insideTestPointCutoffPlane = "+insideTestPointCutoffPlane);
+ System.out.println("outsideTestPointCutoffPlane = "+outsideTestPointCutoffPlane);
+ */
+
+ computedInsideOutside = true;
+ }
+ }
+
+ private GeoPoint pickProximate(final GeoPoint[] points) {
+ if (points.length == 0) {
+ throw new IllegalArgumentException("No off-plane intersection points were found; can't compute traversal");
+ } else if (points.length == 1) {
+ return points[0];
+ } else {
+ final double p1dist = computeSquaredDistance(points[0], intersectionPoint);
+ final double p2dist = computeSquaredDistance(points[1], intersectionPoint);
+ if (p1dist < p2dist) {
+ return points[0];
+ } else if (p2dist < p1dist) {
+ return points[1];
+ } else {
+ throw new IllegalArgumentException("Neither off-plane intersection point matched intersection point; intersection = "+intersectionPoint+"; offplane choice 0: "+points[0]+"; offplane choice 1: "+points[1]);
+ }
+ }
+ }
+
+ @Override
+ public int getCrossingCount() {
+ // Doesn't return the actual crossing count -- just gets the even/odd part right
+ return Math.min(innerCrossingCount, outerCrossingCount);
+ }
+
+ @Override
+ public boolean isOnEdge() {
+ return onEdge;
+ }
+
+ @Override
+ public boolean matches(final Edge edge) {
+ // Early exit if the point is on the edge, in which case we accidentally discovered the answer.
+ if (edge.isWithin(thePointX, thePointY, thePointZ)) {
+ onEdge = true;
+ return false;
+ }
+
+ // All edges that touch the travel planes get assessed the same. So, for each intersecting edge on both legs:
+ // (1) If the edge contains the intersection point, we analyze it on only one leg. For the other leg, we do nothing.
+ // (2) We compute the crossings of the edge with ALL FOUR inner and outer bounding planes.
+ // (3) We add the numbers of each kind of crossing to the total for that class of crossing (innerTotal and outerTotal).
+ // (4) When done all edges tallied in this way, we take min(innerTotal, outerTotal) and assume that is the number of crossings.
+ //
+ // Q: What if we see the same edge in both traversals?
+ // A: We should really evaluate it only in one. Keep a hash of the edges we've looked at already and don't process edges twice.
+
+ // Every edge should be looked at only once.
+ if (seenEdges != null && seenEdges.contains(edge)) {
+ return true;
+ }
+ if (seenEdges == null) {
+ seenEdges = new HashSet<>();
+ }
+ seenEdges.add(edge);
+
+ // We've never seen this edge before. Evaluate it in the context of inner and outer planes.
+ computeInsideOutside();
+
+ /*
+ System.out.println("\nThe following edges should intersect the travel/testpoint planes:");
+ Edge thisEdge = edge;
+ while (true) {
+ final GeoPoint[] travelCrossings = travelPlane.findIntersections(planetModel, thisEdge.plane, checkPointCutoffPlane, checkPointOtherCutoffPlane, thisEdge.startPlane, thisEdge.endPlane);
+ if (travelCrossings == null || travelCrossings.length > 0) {
+ System.out.println("Travel plane: "+thisEdge.startPoint+" -> "+thisEdge.endPoint);
+ }
+ final GeoPoint[] testPointCrossings = testPointPlane.findIntersections(planetModel, thisEdge.plane, testPointCutoffPlane, testPointOtherCutoffPlane, thisEdge.startPlane, thisEdge.endPlane);
+ if (testPointCrossings == null || testPointCrossings.length > 0) {
+ System.out.println("Test point plane: "+thisEdge.startPoint+" -> "+thisEdge.endPoint);
+ }
+ thisEdge = thisEdge.next;
+ if (thisEdge == edge) {
+ break;
+ }
+ }
+ */
+
+ //System.out.println("");
+ //System.out.println("Considering edge "+(edge.startPoint)+" -> "+(edge.endPoint));
+
+ // Some edges are going to be given to us even when there's no real intersection, so do that as a sanity check, first.
+ final GeoPoint[] travelCrossings = travelPlane.findIntersections(planetModel, edge.plane, checkPointCutoffPlane, checkPointOtherCutoffPlane, edge.startPlane, edge.endPlane);
+ if (travelCrossings != null && travelCrossings.length == 0) {
+ //System.out.println(" No intersections with travel plane...");
+ final GeoPoint[] testPointCrossings = testPointPlane.findIntersections(planetModel, edge.plane, testPointCutoffPlane, testPointOtherCutoffPlane, edge.startPlane, edge.endPlane);
+ if (testPointCrossings != null && testPointCrossings.length == 0) {
+ // As a last resort, see if the edge endpoints are on either plane. This is sometimes necessary because the
+ // intersection computation logic might not detect near-miss edges otherwise.
+ //System.out.println(" No intersections with testpoint plane...");
+ if (!travelPlane.evaluateIsZero(edge.startPoint) && !travelPlane.evaluateIsZero(edge.endPoint) &&
+ !testPointPlane.evaluateIsZero(edge.startPoint) && !testPointPlane.evaluateIsZero(edge.endPoint)) {
+ return true;
+ } else {
+ //System.out.println(" Startpoint/travelPlane="+travelPlane.evaluate(edge.startPoint)+" Startpoint/testPointPlane="+testPointPlane.evaluate(edge.startPoint));
+ //System.out.println(" Endpoint/travelPlane="+travelPlane.evaluate(edge.endPoint)+" Endpoint/testPointPlane="+testPointPlane.evaluate(edge.endPoint));
+ }
+ } else {
+ //System.out.println(" Intersection found with testPoint plane...");
+ }
+ } else {
+ //System.out.println(" Intersection found with travel plane...");
+ }
+
+ //System.out.println(" Edge intersects travel or testPoint plane");
+ /*
+ System.out.println(
+ " start point travel dist="+travelPlane.evaluate(edge.startPoint)+"; end point travel dist="+travelPlane.evaluate(edge.endPoint));
+ System.out.println(
+ " start point travel above dist="+travelAbovePlane.evaluate(edge.startPoint)+"; end point travel above dist="+travelAbovePlane.evaluate(edge.endPoint));
+ System.out.println(
+ " start point travel below dist="+travelBelowPlane.evaluate(edge.startPoint)+"; end point travel below dist="+travelBelowPlane.evaluate(edge.endPoint));
+ System.out.println(
+ " start point testpoint dist="+testPointPlane.evaluate(edge.startPoint)+"; end point testpoint dist="+testPointPlane.evaluate(edge.endPoint));
+ System.out.println(
+ " start point testpoint above dist="+testPointAbovePlane.evaluate(edge.startPoint)+"; end point testpoint above dist="+testPointAbovePlane.evaluate(edge.endPoint));
+ System.out.println(
+ " start point testpoint below dist="+testPointBelowPlane.evaluate(edge.startPoint)+"; end point testpoint below dist="+testPointBelowPlane.evaluate(edge.endPoint));
+ */
+
+ // Determine crossings of this edge against all inside/outside planes. There's no further need to look at the actual travel plane itself.
+ //System.out.println(" Assessing inner crossings...");
+ innerCrossingCount += countCrossings(edge, travelInsidePlane, checkPointCutoffPlane, insideTravelCutoffPlane, testPointInsidePlane, testPointCutoffPlane, insideTestPointCutoffPlane);
+ //System.out.println(" Assessing outer crossings...");
+ outerCrossingCount += countCrossings(edge, travelOutsidePlane, checkPointCutoffPlane, outsideTravelCutoffPlane, testPointOutsidePlane, testPointCutoffPlane, outsideTestPointCutoffPlane);
+ /*
+ final GeoPoint[] travelInnerCrossings = computeCrossings(travelInsidePlane, edge, checkPointCutoffPlane, insideTravelCutoffPlane);
+ final GeoPoint[] travelOuterCrossings = computeCrossings(travelOutsidePlane, edge, checkPointCutoffPlane, outsideTravelCutoffPlane);
+ final GeoPoint[] testPointInnerCrossings = computeCrossings(testPointInsidePlane, edge, testPointCutoffPlane, insideTestPointCutoffPlane);
+ final GeoPoint[] testPointOuterCrossings = computeCrossings(testPointOutsidePlane, edge, testPointCutoffPlane, outsideTestPointCutoffPlane);
+ */
+
+ return true;
+ }
+
+ /** Find the intersections with a pair of envelope planes, and assess those intersections for duplication and for
+ * whether they truly describe crossings.
+ */
+ private int countCrossings(final Edge edge,
+ final Plane travelEnvelopePlane, final Membership travelEnvelopeBound1, final Membership travelEnvelopeBound2,
+ final Plane testPointEnvelopePlane, final Membership testPointEnvelopeBound1, final Membership testPointEnvelopeBound2) {
+ final GeoPoint[] travelIntersections = edge.plane.findIntersections(planetModel, travelEnvelopePlane, travelEnvelopeBound1, travelEnvelopeBound2);
+ final GeoPoint[] testPointIntersections = edge.plane.findIntersections(planetModel, testPointEnvelopePlane, testPointEnvelopeBound1, testPointEnvelopeBound2);
+ int crossings = 0;
+ if (travelIntersections != null) {
+ for (final GeoPoint intersection : travelIntersections) {
+ if (edge.startPlane.strictlyWithin(intersection) && edge.endPlane.strictlyWithin(intersection)) {
+ // Make sure it's not a dup
+ boolean notDup = true;
+ if (testPointIntersections != null) {
+ for (final GeoPoint otherIntersection : testPointIntersections) {
+ if (edge.startPlane.strictlyWithin(otherIntersection) && edge.endPlane.strictlyWithin(otherIntersection) && intersection.isNumericallyIdentical(otherIntersection)) {
+ //System.out.println(" Points "+intersection+" and "+otherIntersection+" are duplicates");
+ notDup = false;
+ break;
+ }
+ }
+ }
+ if (!notDup) {
+ continue;
+ }
+ // It's unique, so assess it
+ //System.out.println(" Assessing travel envelope intersection point "+intersection+", travelPlane distance="+travelPlane.evaluate(intersection)+"...");
+ crossings += edgeCrossesEnvelope(edge.plane, intersection, travelEnvelopePlane)?1:0;
+ }
+ }
+ }
+ if (testPointIntersections != null) {
+ for (final GeoPoint intersection : testPointIntersections) {
+ if (edge.startPlane.strictlyWithin(intersection) && edge.endPlane.strictlyWithin(intersection)) {
+ // It's unique, so assess it
+ //System.out.println(" Assessing testpoint envelope intersection point "+intersection+", testPointPlane distance="+testPointPlane.evaluate(intersection)+"...");
+ crossings += edgeCrossesEnvelope(edge.plane, intersection, testPointEnvelopePlane)?1:0;
+ }
+ }
+ }
+ return crossings;
+ }
+
+ /** Return true if the edge crosses the envelope plane, given the envelope intersection point.
+ */
+ private boolean edgeCrossesEnvelope(final Plane edgePlane, final GeoPoint intersectionPoint, final Plane envelopePlane) {
+ final GeoPoint[] adjoiningPoints = findAdjoiningPoints(edgePlane, intersectionPoint, envelopePlane);
+ if (adjoiningPoints == null) {
+ // Couldn't find good adjoining points, so just assume there is a crossing.
+ return true;
+ }
+ int withinCount = 0;
+ for (final GeoPoint adjoining : adjoiningPoints) {
+ if ((travelPlane.evaluateIsZero(adjoining) && checkPointCutoffPlane.isWithin(adjoining) && checkPointOtherCutoffPlane.isWithin(adjoining)) ||
+ (testPointPlane.evaluateIsZero(adjoining) && testPointCutoffPlane.isWithin(adjoining) && testPointOtherCutoffPlane.isWithin(adjoining))) {
+ //System.out.println(" Adjoining point "+adjoining+" (intersection dist = "+intersectionPoint.linearDistance(adjoining)+") is within");
+ withinCount++;
+ } else {
+ //System.out.println(" Adjoining point "+adjoining+" (intersection dist = "+intersectionPoint.linearDistance(adjoining)+"; travelPlane dist="+travelPlane.evaluate(adjoining)+"; testPointPlane dist="+testPointPlane.evaluate(adjoining)+") is not within");
+ }
+ }
+ return (withinCount & 1) != 0;
+ }
+
+ }
+
/** This is the amount we go, roughly, in both directions, to find adjoining points to test. If we go too far,
* we might miss a transition, but if we go too little, we might not see it either due to numerical issues.
*/
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c94a83fd/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
index 3e59143..54c11ce 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
@@ -1219,25 +1219,20 @@ shape:
*/
final GeoPoint unquantized = new GeoPoint(PlanetModel.WGS84, -3.1780051348770987E-74, -3.032608859187692);
- final GeoPoint quantized = new GeoPoint(-0.9951793580415914, -0.10888987641797832, -2.3309121299774915E-10);
-
- final GeoPoint negativeX = new GeoPoint(PlanetModel.WGS84, 0.0, Math.PI);
- final GeoPoint negativeY = new GeoPoint(PlanetModel.WGS84, 0.0, -Math.PI * 0.5);
+ //final GeoPoint quantized = new GeoPoint(-0.9951793580415914, -0.10888987641797832, -2.3309121299774915E-10);
// Construct a standard polygon first to see what that does. This winds up being a large polygon under the covers.
GeoPolygon standard = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, pd);
- // This shows y < 0 hemisphere is all in-set
- //assertTrue(standard.isWithin(negativeY));
// This should be in-set too, but isn't!!
- assertTrue(standard.isWithin(negativeX));
+ assertTrue(standard.isWithin(PlanetModel.WGS84.MIN_X_POLE));
final XYZBounds standardBounds = new XYZBounds();
standard.getBounds(standardBounds);
final XYZSolid standardSolid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84, standardBounds);
// If within shape, should be within bounds
- assertTrue(standard.isWithin(quantized)?standardSolid.isWithin(quantized):true);
+ //assertTrue(standard.isWithin(quantized)?standardSolid.isWithin(quantized):true);
assertTrue(standard.isWithin(unquantized)?standardSolid.isWithin(unquantized):true);
}
@@ -1778,7 +1773,7 @@ shape:
}
@Test
- @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-8280")
+ //@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-8280")
public void testLUCENE8280() {
/*
[junit4] 1> unquantized=[lat=0.16367268756896675, lon=-3.141592653589793([X=-0.9876510422569805, Y=-1.2095236875745584E-16, Z=0.16311061810965483])]
@@ -1800,21 +1795,25 @@ shape:
points.add(new GeoPoint(PlanetModel.WGS84, -0.40516490647074055, 2.4457272005608357E-47));
points.add(new GeoPoint(PlanetModel.WGS84, 2.4457272005608357E-47, -0.6244585784444767));
final GeoPolygonFactory.PolygonDescription description = new GeoPolygonFactory.PolygonDescription(points);
- //final GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, description);
+ // I think this polygon may cross itself around lat=-0.91, lon=0. If so, this is an invalid test.
final GeoPolygon largePolygon = GeoPolygonFactory.makeLargeGeoPolygon(PlanetModel.WGS84, Collections.singletonList(description));
final GeoPoint point = new GeoPoint(PlanetModel.WGS84, 0.16367268756896675, -3.141592653589793);
-
- // Both are complex polygons, so this is going to pass.
- //assertTrue(polygon.isWithin(point) == largePolygon.isWithin(point));
+ assertFalse(largePolygon.isWithin(point));
+ /* Confirmed that bounds is OK
final XYZBounds xyzBounds = new XYZBounds();
largePolygon.getBounds(xyzBounds);
+
+ System.out.println("North pole is within? "+largePolygon.isWithin(PlanetModel.WGS84.NORTH_POLE));
+ System.out.println("South pole is within? "+largePolygon.isWithin(PlanetModel.WGS84.SOUTH_POLE));
+
final XYZSolid xyzSolid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84, xyzBounds);
// Failure is due either to bounds computation or multiple points having their in-set status wrongly assessed.
// Probably it is the former because there are more than a dozen points that otherwise fail to be correct.
assertTrue(largePolygon.isWithin(point)?xyzSolid.isWithin(point):true);
-
+ */
+
}
}
[06/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8244: Do not leak
open file descriptors in SearcherTaxonomyManager's refresh on exception (Mike
McCandless)
Posted by ab...@apache.org.
LUCENE-8244: Do not leak open file descriptors in SearcherTaxonomyManager's refresh on exception (Mike McCandless)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4fba55c8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4fba55c8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4fba55c8
Branch: refs/heads/jira/solr-11779
Commit: 4fba55c864298848a07c2971738947473484006e
Parents: 400449f
Author: Mike McCandless <mi...@apache.org>
Authored: Sat Apr 28 09:42:30 2018 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sat Apr 28 09:43:24 2018 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 +++-
.../facet/taxonomy/SearcherTaxonomyManager.java | 12 ++++++++-
.../taxonomy/TestSearcherTaxonomyManager.java | 26 ++++++++++++++++++++
3 files changed, 41 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fba55c8/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b141e45..4d902fe 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -173,6 +173,9 @@ Bug Fixes
the directory has pending deletes files even if the directory is filtered or
a FileSwitchDirectory (Simon Willnauer, Robert Muir)
+* LUCENE-8244: Do not leak open file descriptors in SearcherTaxonomyManager's
+ refresh on exception (Mike McCandless)
+
Other
* LUCENE-8228: removed obsolete IndexDeletionPolicy clone() requirements from
@@ -200,7 +203,7 @@ Build
Documentation
* LUCENE-8238: Improve WordDelimiterFilter and WordDelimiterGraphFilter javadocs
-xo (Mike Sokolov via Mike McCandless)
+ (Mike Sokolov via Mike McCandless)
======================= Lucene 7.3.0 =======================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fba55c8/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java
index 0a81dfb..e618aa5 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java
@@ -134,7 +134,17 @@ public class SearcherTaxonomyManager extends ReferenceManager<SearcherTaxonomyMa
if (newReader == null) {
return null;
} else {
- DirectoryTaxonomyReader tr = TaxonomyReader.openIfChanged(ref.taxonomyReader);
+ DirectoryTaxonomyReader tr;
+ try {
+ tr = TaxonomyReader.openIfChanged(ref.taxonomyReader);
+ } catch (Throwable t1) {
+ try {
+ IOUtils.close(newReader);
+ } catch (Throwable t2) {
+ t2.addSuppressed(t2);
+ }
+ throw t1;
+ }
if (tr == null) {
ref.taxonomyReader.incRef();
tr = ref.taxonomyReader;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fba55c8/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java
index a1e0344..d9a3f8e 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java
@@ -33,8 +33,10 @@ import org.apache.lucene.facet.FacetsCollector;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.taxonomy.SearcherTaxonomyManager.SearcherAndTaxonomy;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.index.IndexNotFoundException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.ReferenceManager;
@@ -321,4 +323,28 @@ public class TestSearcherTaxonomyManager extends FacetTestCase {
IOUtils.close(mgr, tw, taxoDir, indexDir);
}
+ public void testExceptionDuringRefresh() throws Exception {
+
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
+ IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
+ DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
+ w.commit();
+ tw.commit();
+
+ SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
+
+ tw.addCategory(new FacetLabel("a", "b"));
+ w.addDocument(new Document());
+
+ tw.commit();
+ w.commit();
+
+ // intentionally corrupt the taxo index:
+ SegmentInfos infos = SegmentInfos.readLatestCommit(taxoDir);
+ taxoDir.deleteFile(infos.getSegmentsFileName());
+ expectThrows(IndexNotFoundException.class, mgr::maybeRefreshBlocking);
+ IOUtils.close(w, tw, mgr, indexDir, taxoDir);
+ }
}
[49/50] [abbrv] lucene-solr:jira/solr-11779: Merge branch 'master'
into jira/solr-11779
Posted by ab...@apache.org.
Merge branch 'master' into jira/solr-11779
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c3337599
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c3337599
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c3337599
Branch: refs/heads/jira/solr-11779
Commit: c33375991b89867be3b256c4e402243581ba60e2
Parents: d778367 81f6112
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Mon May 7 21:02:35 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Mon May 7 21:02:35 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 23 +-
.../miscellaneous/WordDelimiterFilter.java | 13 +-
.../miscellaneous/WordDelimiterGraphFilter.java | 18 +-
.../miscellaneous/TestWordDelimiterFilter.java | 43 +-
.../TestWordDelimiterGraphFilter.java | 32 +
...ache.lucene.analysis.util.TokenFilterFactory | 1 +
.../codecs/blockterms/BlockTermsReader.java | 7 +-
.../blocktreeords/OrdsIntersectTermsEnum.java | 5 +-
.../blocktreeords/OrdsSegmentTermsEnum.java | 5 +-
.../bloom/BloomFilteringPostingsFormat.java | 7 +-
.../codecs/memory/DirectPostingsFormat.java | 9 +-
.../lucene/codecs/memory/FSTOrdTermsReader.java | 7 +-
.../lucene/codecs/memory/FSTTermsReader.java | 5 +-
.../codecs/memory/MemoryDocValuesProducer.java | 7 +-
.../codecs/memory/MemoryPostingsFormat.java | 7 +-
.../simpletext/SimpleTextFieldsReader.java | 5 +-
.../simpletext/SimpleTextTermVectorsReader.java | 7 +-
.../codecs/CompetitiveFreqNormAccumulator.java | 163 ----
.../codecs/CompetitiveImpactAccumulator.java | 127 ++++
.../lucene/codecs/PostingsReaderBase.java | 3 +-
.../codecs/blocktree/IntersectTermsEnum.java | 5 +-
.../codecs/blocktree/SegmentTermsEnum.java | 5 +-
.../CompressingTermVectorsReader.java | 6 +-
.../codecs/lucene50/Lucene50PostingsReader.java | 39 +-
.../codecs/lucene50/Lucene50PostingsWriter.java | 4 +-
.../lucene50/Lucene50ScoreSkipReader.java | 141 ++--
.../codecs/lucene50/Lucene50SkipReader.java | 3 +-
.../codecs/lucene50/Lucene50SkipWriter.java | 36 +-
.../lucene70/Lucene70DocValuesProducer.java | 3 +-
.../apache/lucene/document/FeatureQuery.java | 8 +-
.../index/BinaryDocValuesFieldUpdates.java | 174 ++---
.../apache/lucene/index/BufferedUpdates.java | 62 +-
.../org/apache/lucene/index/CheckIndex.java | 186 +++--
.../lucene/index/DocValuesFieldUpdates.java | 295 +++++--
.../apache/lucene/index/DocValuesUpdate.java | 92 ++-
.../apache/lucene/index/DocumentsWriter.java | 41 +-
.../index/DocumentsWriterDeleteQueue.java | 7 +-
.../lucene/index/DocumentsWriterFlushQueue.java | 180 ++---
.../lucene/index/DocumentsWriterPerThread.java | 5 +-
.../apache/lucene/index/FilterLeafReader.java | 5 +-
.../apache/lucene/index/FilteredTermsEnum.java | 5 +-
.../org/apache/lucene/index/FreqProxFields.java | 3 +-
.../lucene/index/FrozenBufferedUpdates.java | 147 ++--
.../java/org/apache/lucene/index/Impact.java | 61 ++
.../java/org/apache/lucene/index/Impacts.java | 54 ++
.../org/apache/lucene/index/ImpactsEnum.java | 39 +-
.../org/apache/lucene/index/IndexWriter.java | 400 ++++++----
.../org/apache/lucene/index/MergePolicy.java | 2 +
.../org/apache/lucene/index/MultiTermsEnum.java | 5 +-
.../index/NumericDocValuesFieldUpdates.java | 155 +---
.../org/apache/lucene/index/PendingDeletes.java | 18 +-
.../apache/lucene/index/PendingSoftDeletes.java | 43 +-
.../apache/lucene/index/ReadersAndUpdates.java | 362 ++++-----
.../apache/lucene/index/SlowImpactsEnum.java | 39 +-
.../lucene/index/SortedDocValuesTermsEnum.java | 3 +-
.../index/SortedSetDocValuesTermsEnum.java | 3 +-
.../java/org/apache/lucene/index/TermsEnum.java | 7 +-
.../search/BlockMaxConjunctionScorer.java | 3 +
.../lucene/search/DisjunctionMaxScorer.java | 1 +
.../lucene/search/DisjunctionSumScorer.java | 1 +
.../apache/lucene/search/FuzzyTermsEnum.java | 5 +-
.../org/apache/lucene/search/MaxScoreCache.java | 138 ++++
.../org/apache/lucene/search/ReqExclScorer.java | 5 +
.../apache/lucene/search/ReqOptSumScorer.java | 7 +
.../java/org/apache/lucene/search/Scorer.java | 4 +-
.../org/apache/lucene/search/TermScorer.java | 40 +-
.../org/apache/lucene/search/WANDScorer.java | 1 +
.../java/org/apache/lucene/store/Directory.java | 7 +
.../org/apache/lucene/store/FSDirectory.java | 16 +-
.../lucene/store/FileSwitchDirectory.java | 5 +
.../apache/lucene/store/FilterDirectory.java | 4 +
.../TestCompetitiveFreqNormAccumulator.java | 44 +-
.../lucene50/TestBlockPostingsFormat.java | 57 +-
.../org/apache/lucene/index/TestCodecs.java | 3 +-
.../lucene/index/TestDocValuesFieldUpdates.java | 72 ++
.../lucene/index/TestFilterLeafReader.java | 22 +
.../TestIndexWriterOutOfFileDescriptors.java | 5 +
.../lucene/index/TestMixedDocValuesUpdates.java | 154 +++-
.../lucene/index/TestPendingSoftDeletes.java | 62 +-
.../TestSoftDeletesRetentionMergePolicy.java | 102 ++-
.../lucene/search/TestSearcherManager.java | 3 +-
.../lucene/store/TestFilterDirectory.java | 2 +-
.../facet/taxonomy/SearcherTaxonomyManager.java | 12 +-
.../taxonomy/TestSearcherTaxonomyManager.java | 26 +
.../directory/TestDirectoryTaxonomyWriter.java | 61 +-
.../search/uhighlight/FieldOffsetStrategy.java | 2 +-
lucene/ivy-versions.properties | 2 +-
.../apache/lucene/index/memory/MemoryIndex.java | 5 +-
.../lucene/replicator/nrt/ReplicaNode.java | 5 +-
.../idversion/IDVersionPostingsReader.java | 5 +-
.../idversion/IDVersionSegmentTermsEnum.java | 5 +-
.../search/intervals/IntervalFunction.java | 4 +-
.../search/intervals/TestIntervalQuery.java | 12 +
.../lucene/search/intervals/TestIntervals.java | 25 +-
.../spatial3d/geom/GeoComplexPolygon.java | 762 +++++++++++++------
.../lucene/spatial3d/geom/GeoPolygonTest.java | 146 +++-
.../spatial3d/geom/RandomGeoPolygonTest.java | 249 +++---
.../codecs/ramonly/RAMOnlyPostingsFormat.java | 5 +-
.../lucene/index/AssertingLeafReader.java | 55 +-
.../lucene/index/RandomPostingsTester.java | 241 +++---
.../apache/lucene/search/AssertingScorer.java | 5 +-
.../org/apache/lucene/search/CheckHits.java | 2 +-
.../lucene/store/BaseLockFactoryTestCase.java | 18 +-
.../dependencies/InterpolatedProperties.java | 127 +++-
solr/CHANGES.txt | 37 +
solr/bin-test/test_auth.sh | 40 +
solr/bin-test/utils/assert.sh | 30 +-
.../prometheus-exporter/bin/solr-exporter.cmd | 8 +-
.../org/apache/solr/cloud/SyncStrategy.java | 2 +-
.../api/collections/DeleteCollectionCmd.java | 30 +-
.../OverseerCollectionMessageHandler.java | 9 +-
.../org/apache/solr/core/BlobRepository.java | 16 +-
.../java/org/apache/solr/core/SolrConfig.java | 45 +-
.../org/apache/solr/handler/IndexFetcher.java | 42 +-
.../apache/solr/handler/ReplicationHandler.java | 6 +-
.../admin/AutoscalingHistoryHandler.java | 2 +-
.../solr/handler/admin/CollectionsHandler.java | 4 +-
.../solr/handler/admin/CoreAdminHandler.java | 6 +
.../apache/solr/logging/MDCLoggingContext.java | 8 +-
.../reporters/solr/SolrClusterReporter.java | 2 +-
.../reporters/solr/SolrShardReporter.java | 2 +-
.../org/apache/solr/query/SolrRangeQuery.java | 5 +-
.../apache/solr/search/ValueSourceParser.java | 8 +
.../solr/search/facet/UniqueBlockAgg.java | 91 +++
.../search/facet/UniqueSinglevaluedSlotAcc.java | 16 +-
.../apache/solr/search/join/FiltersQParser.java | 38 +-
.../solr/security/PKIAuthenticationPlugin.java | 9 +-
.../org/apache/solr/servlet/HttpSolrCall.java | 6 +-
.../apache/solr/servlet/LoadAdminUiServlet.java | 14 +-
.../solr/servlet/ServletInputStreamWrapper.java | 2 +-
.../servlet/ServletOutputStreamWrapper.java | 2 +-
.../apache/solr/servlet/SolrDispatchFilter.java | 141 ++--
.../apache/solr/servlet/SolrRequestParsers.java | 8 +-
.../solr/spelling/SpellCheckCollator.java | 4 +-
.../apache/solr/uninverting/DocTermOrds.java | 5 +-
.../org/apache/solr/update/CommitTracker.java | 66 +-
.../solr/update/DefaultSolrCoreState.java | 5 +-
.../solr/update/DirectUpdateHandler2.java | 45 +-
.../java/org/apache/solr/update/PeerSync.java | 2 +-
.../apache/solr/update/SolrCmdDistributor.java | 8 +-
.../solr/update/StreamingSolrClients.java | 2 +-
.../org/apache/solr/update/TransactionLog.java | 7 +
.../java/org/apache/solr/update/UpdateLog.java | 7 +
.../apache/solr/update/UpdateShardHandler.java | 68 +-
.../processor/DistributedUpdateProcessor.java | 2 +-
.../IgnoreLargeDocumentProcessorFactory.java | 174 +++++
.../src/java/org/apache/solr/util/SolrCLI.java | 15 +
.../org/apache/solr/util/SystemIdResolver.java | 14 +-
.../conf/bad-solrconfig-no-autocommit-tag.xml | 52 ++
.../solr/collection1/conf/solrconfig-tlog.xml | 4 +
.../solr/cloud/DocValuesNotIndexedTest.java | 1 +
.../solr/cloud/LIRRollingUpdatesTest.java | 3 +
.../cloud/LeaderElectionIntegrationTest.java | 1 +
.../apache/solr/cloud/OverseerRolesTest.java | 1 +
.../apache/solr/cloud/SolrCloudExampleTest.java | 1 +
.../collections/TestHdfsCloudBackupRestore.java | 2 +
.../sim/SimClusterStateProvider.java | 3 +
.../test/org/apache/solr/core/TestConfig.java | 36 +
.../solr/handler/TestReplicationHandler.java | 1 +
.../apache/solr/schema/TestCloudSchemaless.java | 1 +
.../apache/solr/search/QueryEqualityTest.java | 1 +
.../apache/solr/search/TestStressRecovery.java | 1 +
.../facet/TestJsonFacetsWithNestedObjects.java | 26 +
.../apache/solr/search/join/BJQParserTest.java | 61 +-
.../search/join/BlockJoinFacetDistribTest.java | 77 +-
.../spelling/WordBreakSolrSpellCheckerTest.java | 14 +-
.../solr/update/MaxSizeAutoCommitTest.java | 398 ++++++++++
.../apache/solr/update/TransactionLogTest.java | 9 +-
...IgnoreLargeDocumentProcessorFactoryTest.java | 102 +++
.../apache/solr/util/TestSystemIdResolver.java | 41 +-
.../licenses/jackson-annotations-2.5.4.jar.sha1 | 1 -
.../licenses/jackson-annotations-2.9.5.jar.sha1 | 1 +
solr/licenses/jackson-core-2.5.4.jar.sha1 | 1 -
solr/licenses/jackson-core-2.9.5.jar.sha1 | 1 +
solr/licenses/jackson-databind-2.5.4.jar.sha1 | 1 -
solr/licenses/jackson-databind-2.9.5.jar.sha1 | 1 +
.../jackson-dataformat-smile-2.5.4.jar.sha1 | 1 -
.../jackson-dataformat-smile-2.9.5.jar.sha1 | 1 +
solr/solr-ref-guide/src/blockjoin-faceting.adoc | 3 +-
solr/solr-ref-guide/src/json-facet-api.adoc | 71 +-
solr/solr-ref-guide/src/other-parsers.adoc | 2 +-
.../src/update-request-processors.adoc | 2 +
.../org/apache/solr/client/solrj/io/Lang.java | 11 +-
.../client/solrj/io/eval/CanberraEvaluator.java | 49 ++
.../solrj/io/eval/ChebyshevEvaluator.java | 49 ++
.../client/solrj/io/eval/DistanceEvaluator.java | 118 ++-
.../solrj/io/eval/EarthMoversEvaluator.java | 49 ++
.../solrj/io/eval/EuclideanEvaluator.java | 49 ++
.../solr/client/solrj/io/eval/FFTEvaluator.java | 73 ++
.../client/solrj/io/eval/IFFTEvaluator.java | 71 ++
.../solr/client/solrj/io/eval/KnnEvaluator.java | 45 +-
.../solrj/io/eval/ManhattanEvaluator.java | 49 ++
.../client/solrj/io/eval/OnesEvaluator.java | 47 ++
.../client/solrj/io/eval/ZerosEvaluator.java | 47 ++
.../apache/solr/client/solrj/io/TestLang.java | 3 +-
.../solrj/io/stream/MathExpressionTest.java | 131 +++-
196 files changed, 5958 insertions(+), 2482 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3337599/lucene/ivy-versions.properties
----------------------------------------------------------------------
[25/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8142: Fix
QueryUtils to only call getMaxScore when it is legal to do so.
Posted by ab...@apache.org.
LUCENE-8142: Fix QueryUtils to only call getMaxScore when it is legal to do so.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/67c13bbe
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/67c13bbe
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/67c13bbe
Branch: refs/heads/jira/solr-11779
Commit: 67c13bbe2ebdab23c8ff316f8f0805529146a63d
Parents: 82e7cb2
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed May 2 17:41:26 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed May 2 17:42:18 2018 +0200
----------------------------------------------------------------------
.../src/java/org/apache/lucene/search/AssertingScorer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/67c13bbe/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
index 952687f..a8c32cf 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
@@ -99,7 +99,7 @@ public class AssertingScorer extends Scorer {
assert iterating() : state;
final float score = in.score();
assert !Float.isNaN(score) : "NaN score for in="+in;
- assert score <= getMaxScore(DocIdSetIterator.NO_MORE_DOCS);
+ assert lastShallowTarget == -1 || score <= getMaxScore(docID());
assert Float.compare(score, 0f) >= 0 : score;
return score;
}
[03/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-11734: Add ones and
zeros Stream Evaluators
Posted by ab...@apache.org.
SOLR-11734: Add ones and zeros Stream Evaluators
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2c487947
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2c487947
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2c487947
Branch: refs/heads/jira/solr-11779
Commit: 2c487947e8cd9e6583d746540451cba4e8157e45
Parents: 43c086a
Author: Joel Bernstein <jb...@apache.org>
Authored: Fri Apr 27 13:44:12 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Fri Apr 27 13:44:12 2018 -0400
----------------------------------------------------------------------
.../org/apache/solr/client/solrj/io/Lang.java | 4 +-
.../client/solrj/io/eval/OnesEvaluator.java | 47 ++++++++++++++++++++
.../client/solrj/io/eval/ZerosEvaluator.java | 47 ++++++++++++++++++++
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 45 +++++++++++++++++++
5 files changed, 143 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2c487947/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
index f0862db..941c1a1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
@@ -242,8 +242,10 @@ public class Lang {
.withFunctionName("earthMovers", EarthMoversEvaluator.class)
.withFunctionName("euclidean", EuclideanEvaluator.class)
.withFunctionName("chebyshev", ChebyshevEvaluator.class)
+ .withFunctionName("ones", OnesEvaluator.class)
+ .withFunctionName("zeros", ZerosEvaluator.class)
- // Boolean Stream Evaluators
+ // Boolean Stream Evaluators
.withFunctionName("and", AndEvaluator.class)
.withFunctionName("eor", ExclusiveOrEvaluator.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2c487947/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OnesEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OnesEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OnesEvaluator.java
new file mode 100644
index 0000000..bef53fd
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OnesEvaluator.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class OnesEvaluator extends RecursiveNumericEvaluator implements OneValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public OnesEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+
+ if(1 != containedEvaluators.size()){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting exactly 1 value but found %d",expression,containedEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Object doWork(Object value){
+ int size = ((Number)value).intValue();
+ List<Number> ones = new ArrayList();
+ for(int i=0; i<size; i++) {
+ ones.add(1);
+ }
+ return ones;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2c487947/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ZerosEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ZerosEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ZerosEvaluator.java
new file mode 100644
index 0000000..c379185
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/ZerosEvaluator.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class ZerosEvaluator extends RecursiveNumericEvaluator implements OneValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public ZerosEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+
+ if(1 != containedEvaluators.size()){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting exactly 1 value but found %d",expression,containedEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Object doWork(Object value){
+ int size = ((Number)value).intValue();
+ List<Number> ones = new ArrayList();
+ for(int i=0; i<size; i++) {
+ ones.add(0);
+ }
+ return ones;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2c487947/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
index 5f1e4f0..1e824da 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
@@ -69,7 +69,7 @@ public class TestLang extends LuceneTestCase {
TemporalEvaluatorDayOfQuarter.FUNCTION_NAME, "abs", "add", "div", "mult", "sub", "log", "pow",
"mod", "ceil", "floor", "sin", "asin", "sinh", "cos", "acos", "cosh", "tan", "atan", "tanh", "round", "sqrt",
"cbrt", "coalesce", "uuid", "if", "convert", "valueAt", "memset", "fft", "ifft", "euclidean","manhattan",
- "earthMovers", "canberra", "chebyshev"};
+ "earthMovers", "canberra", "chebyshev", "ones", "zeros"};
@Test
public void testLang() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2c487947/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index d63b922..94d5334 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -995,6 +995,51 @@ public class MathExpressionTest extends SolrCloudTestCase {
assertTrue(out.get(5).doubleValue() == 500.23D);
}
+
+ @Test
+ public void testOnes() throws Exception {
+ String cexpr = "ones(6)";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> out = (List<Number>)tuples.get(0).get("return-value");
+ assertEquals(out.size(), 6);
+ assertEquals(out.get(0).intValue(), 1);
+ assertEquals(out.get(1).intValue(), 1);
+ assertEquals(out.get(2).intValue(), 1);
+ assertEquals(out.get(3).intValue(), 1);
+ assertEquals(out.get(4).intValue(), 1);
+ assertEquals(out.get(5).intValue(), 1);
+ }
+
+ @Test
+ public void testZeros() throws Exception {
+ String cexpr = "zeros(6)";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> out = (List<Number>)tuples.get(0).get("return-value");
+ assertEquals(out.size(), 6);
+ assertEquals(out.get(0).intValue(), 0);
+ assertEquals(out.get(1).intValue(), 0);
+ assertEquals(out.get(2).intValue(), 0);
+ assertEquals(out.get(3).intValue(), 0);
+ assertEquals(out.get(4).intValue(), 0);
+ assertEquals(out.get(5).intValue(), 0);
+ }
+
@Test
public void testMatrix() throws Exception {
String cexpr = "let(echo=true," +
[17/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12202: Fix errors
in solr-exporter.cmd
Posted by ab...@apache.org.
SOLR-12202: Fix errors in solr-exporter.cmd
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ee2198d6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ee2198d6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ee2198d6
Branch: refs/heads/jira/solr-11779
Commit: ee2198d6bd12bed1b75ac7abbd0e99c80d5557af
Parents: ee7b52f
Author: koji <ko...@apache.org>
Authored: Wed May 2 13:59:13 2018 +0900
Committer: koji <ko...@apache.org>
Committed: Wed May 2 13:59:13 2018 +0900
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
solr/contrib/prometheus-exporter/bin/solr-exporter.cmd | 8 ++++----
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee2198d6/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f245002..a3a6c37 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -200,6 +200,8 @@ Bug Fixes
* SOLR-12284: WordBreakSolrSpellchecker will no longer add parenthesis in collations when breaking words in
non-boolean queries. (James Dyer)
+* SOLR-12202: Fix errors in solr-exporter.cmd. (Minoru Osuka via koji)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ee2198d6/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
----------------------------------------------------------------------
diff --git a/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd b/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
index 8fa4f60..31b8825 100644
--- a/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
+++ b/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
@@ -52,14 +52,14 @@ goto Win9xApp
:Win9xGetScriptDir
set SAVEDIR=%CD%
%0\
-cd %0\..\..
+cd %0\..\..
set BASEDIR=%CD%
cd %SAVEDIR%
set SAVE_DIR=
goto repoSetup
:WinNTGetScriptDir
-set BASEDIR=%~dp0\..
+set BASEDIR=%~dp0..
:repoSetup
@@ -68,14 +68,14 @@ if "%JAVACMD%"=="" set JAVACMD=java
if "%REPO%"=="" set REPO=%BASEDIR%\lib
-set CLASSPATH="%CLASSPATH%";"%REPO%\*;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\solr-core-*;%BASEDIR%\..\..\dist\solr-solrj-*;%BASEDIR%\..\..\dist\solr-prometheus-exporter-*;%BASEDIR%\lib\*"
+set CLASSPATH=%REPO%\*;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\*;%BASEDIR%\lucene-libs\*
set EXTRA_JVM_ARGUMENTS=-Xmx512m -Dlog4j.configurationFile=file:%BASEDIR%/conf/log4j2.xml
goto endInit
@REM Reaching here means variables are defined and arguments have been captured
:endInit
-%JAVACMD% %JAVA_OPTS% %EXTRA_JVM_ARGUMENTS% -classpath %CLASSPATH_PREFIX%;%CLASSPATH% -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" com.github.mosuka.solr.prometheus.exporter.SolrExporter %CMD_LINE_ARGS%
+%JAVACMD% %JAVA_OPTS% %EXTRA_JVM_ARGUMENTS% -classpath "%CLASSPATH_PREFIX%;%CLASSPATH%" -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" org.apache.solr.prometheus.exporter.SolrExporter %CMD_LINE_ARGS%
if ERRORLEVEL 1 goto error
goto end
[50/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-11779: Add unit
tests.
Posted by ab...@apache.org.
SOLR-11779: Add unit tests.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b5fa6653
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b5fa6653
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b5fa6653
Branch: refs/heads/jira/solr-11779
Commit: b5fa665387f6d7d8ab079f4cbe703e21f3465840
Parents: c333759
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue May 8 23:14:22 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue May 8 23:14:22 2018 +0200
----------------------------------------------------------------------
.../org/apache/solr/core/CoreContainer.java | 23 +-
.../solr/handler/admin/MetricsHandler.java | 19 +-
.../handler/admin/MetricsHistoryHandler.java | 71 +++++-
.../apache/solr/metrics/rrd/SolrRrdBackend.java | 4 +
.../solr/metrics/rrd/SolrRrdBackendFactory.java | 58 ++++-
.../solr/handler/admin/MetricsHandlerTest.java | 8 +-
.../admin/MetricsHistoryHandlerTest.java | 26 ++
.../metrics/rrd/SolrRrdBackendFactoryTest.java | 244 +++++++++++++++++++
.../apache/solr/common/params/CommonParams.java | 1 +
9 files changed, 421 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 9caadd2..b40fc32 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -25,6 +25,7 @@ import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH
import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.HEALTH_CHECK_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.METRICS_HISTORY_PATH;
import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
import static org.apache.solr.common.params.CommonParams.ZK_PATH;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
@@ -56,6 +57,7 @@ import org.apache.http.client.CredentialsProvider;
import org.apache.http.config.Lookup;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.store.Directory;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.client.solrj.impl.SolrHttpClientContextBuilder;
@@ -87,6 +89,7 @@ import org.apache.solr.handler.admin.HealthCheckHandler;
import org.apache.solr.handler.admin.InfoHandler;
import org.apache.solr.handler.admin.MetricsCollectorHandler;
import org.apache.solr.handler.admin.MetricsHandler;
+import org.apache.solr.handler.admin.MetricsHistoryHandler;
import org.apache.solr.handler.admin.SecurityConfHandler;
import org.apache.solr.handler.admin.SecurityConfHandlerLocal;
import org.apache.solr.handler.admin.SecurityConfHandlerZk;
@@ -190,6 +193,8 @@ public class CoreContainer {
protected MetricsHandler metricsHandler;
+ protected MetricsHistoryHandler metricsHistoryHandler;
+
protected MetricsCollectorHandler metricsCollectorHandler;
protected AutoscalingHistoryHandler autoscalingHistoryHandler;
@@ -540,7 +545,20 @@ public class CoreContainer {
infoHandler = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class);
coreAdminHandler = createHandler(CORES_HANDLER_PATH, cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
configSetsHandler = createHandler(CONFIGSETS_HANDLER_PATH, cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class);
- metricsHandler = createHandler(METRICS_PATH, MetricsHandler.class.getName(), MetricsHandler.class);
+
+ // metricsHistoryHandler uses metricsHandler, so create it first
+ metricsHandler = new MetricsHandler(metricManager);
+ containerHandlers.put(METRICS_PATH, metricsHandler);
+ metricsHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, METRICS_PATH);
+
+ if (isZooKeeperAware()) {
+ metricsHistoryHandler = new MetricsHistoryHandler(metricManager, metricsHandler,
+ new CloudSolrClient.Builder(Collections.singletonList(getZkController().getZkServerAddress()), Optional.empty())
+ .withHttpClient(updateShardHandler.getDefaultHttpClient()).build(), getZkController().getSolrCloudManager());
+ containerHandlers.put(METRICS_HISTORY_PATH, metricsHistoryHandler);
+ metricsHistoryHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, METRICS_HISTORY_PATH);
+ }
+
autoscalingHistoryHandler = createHandler(AUTOSCALING_HISTORY_PATH, AutoscalingHistoryHandler.class.getName(), AutoscalingHistoryHandler.class);
metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(), MetricsCollectorHandler.class);
// may want to add some configuration here in the future
@@ -765,6 +783,9 @@ public class CoreContainer {
} catch (Exception e) {
log.warn("Error removing live node. Continuing to close CoreContainer", e);
}
+ if (metricsHistoryHandler != null) {
+ IOUtils.closeQuietly(metricsHistoryHandler.getSolrClient());
+ }
if (metricManager != null) {
metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoBean.Group.cluster));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index 3c8a152..6f1c5e0 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -54,7 +54,6 @@ import org.apache.solr.util.stats.MetricUtils;
* Request handler to return metrics
*/
public class MetricsHandler extends RequestHandlerBase implements PermissionNameProvider {
- final CoreContainer container;
final SolrMetricManager metricManager;
public static final String COMPACT_PARAM = "compact";
@@ -71,13 +70,11 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
private static final Pattern KEY_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));
public MetricsHandler() {
- this.container = null;
this.metricManager = null;
}
- public MetricsHandler(CoreContainer container) {
- this.container = container;
- this.metricManager = this.container.getMetricManager();
+ public MetricsHandler(SolrMetricManager metricManager) {
+ this.metricManager = metricManager;
}
@Override
@@ -87,8 +84,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
- if (container == null) {
- throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Core container instance not initialized");
+ if (metricManager == null) {
+ throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "SolrMetricManager instance not initialized");
}
handleRequest(req.getParams(), (k, v) -> rsp.add(k, v));
@@ -237,7 +234,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
public Set<String> parseRegistries(String[] groupStr, String[] registryStr) {
if ((groupStr == null || groupStr.length == 0) && (registryStr == null || registryStr.length == 0)) {
// return all registries
- return container.getMetricManager().registryNames();
+ return metricManager.registryNames();
}
boolean allRegistries = false;
Set<String> initialPrefixes = Collections.emptySet();
@@ -253,7 +250,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
initialPrefixes.add(SolrMetricManager.overridableRegistryName(s.trim()));
}
if (allRegistries) {
- return container.getMetricManager().registryNames();
+ return metricManager.registryNames();
}
}
}
@@ -272,12 +269,12 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
initialPrefixes.add(SolrMetricManager.overridableRegistryName(s.trim()));
}
if (allRegistries) {
- return container.getMetricManager().registryNames();
+ return metricManager.registryNames();
}
}
}
Set<String> validRegistries = new HashSet<>();
- for (String r : container.getMetricManager().registryNames()) {
+ for (String r : metricManager.registryNames()) {
for (String prefix : initialPrefixes) {
if (r.startsWith(prefix)) {
validRegistries.add(r);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
index 5db09dd..87f3c2b 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
@@ -16,7 +16,9 @@
*/
package org.apache.solr.handler.admin;
+import java.io.Closeable;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -34,7 +36,10 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
@@ -55,11 +60,14 @@ import org.rrd4j.core.RrdBackendFactory;
import org.rrd4j.core.RrdDb;
import org.rrd4j.core.RrdDef;
import org.rrd4j.core.Sample;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
*/
-public class MetricsHistoryHandler extends RequestHandlerBase implements PermissionNameProvider {
+public class MetricsHistoryHandler extends RequestHandlerBase implements PermissionNameProvider, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final Set<String> DEFAULT_CORE_COUNTERS = new HashSet<String>() {{
add("QUERY./select.requests");
@@ -79,28 +87,32 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
}};
public static final int DEFAULT_COLLECT_PERIOD = 60;
- public static final String URI_PREFIX = "solr:///";
+ public static final String URI_PREFIX = "solr://";
private final SolrRrdBackendFactory factory;
+ private final SolrClient solrClient;
private final MetricsHandler metricsHandler;
private final SolrMetricManager metricManager;
+ private final SolrCloudManager cloudManager;
private final ScheduledThreadPoolExecutor collectService;
private final TimeSource timeSource;
private final int collectPeriod;
private final Map<String, Set<String>> counters = new HashMap<>();
private final Map<String, Set<String>> gauges = new HashMap<>();
- public MetricsHistoryHandler(CoreContainer coreContainer) {
- factory = new SolrRrdBackendFactory(new CloudSolrClient.Builder(
- Collections.singletonList(coreContainer.getZkController().getZkServerAddress()),
- Optional.empty())
- .withHttpClient(coreContainer.getUpdateShardHandler().getHttpClient())
- .build(), CollectionAdminParams.SYSTEM_COLL);
+ private boolean logMissingCollection = true;
+
+ public MetricsHistoryHandler(SolrMetricManager metricManager, MetricsHandler metricsHandler,
+ SolrClient solrClient, SolrCloudManager cloudManager) {
+ factory = new SolrRrdBackendFactory(solrClient, CollectionAdminParams.SYSTEM_COLL,
+ SolrRrdBackendFactory.DEFAULT_SYNC_PERIOD, cloudManager.getTimeSource());
RrdBackendFactory.registerAndSetAsDefaultFactory(factory);
- metricsHandler = coreContainer.getMetricsHandler();
- metricManager = coreContainer.getMetricManager();
+ this.solrClient = solrClient;
+ this.metricsHandler = metricsHandler;
+ this.metricManager = metricManager;
+ this.cloudManager = cloudManager;
collectPeriod = DEFAULT_COLLECT_PERIOD;
- timeSource = coreContainer.getZkController().getSolrCloudManager().getTimeSource();
+ this.timeSource = cloudManager.getTimeSource();
counters.put(Group.core.toString(), DEFAULT_CORE_COUNTERS);
counters.put(Group.node.toString(), Collections.emptySet());
@@ -116,7 +128,26 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
collectService.scheduleWithFixedDelay(() -> collectMetrics(), collectPeriod, collectPeriod, TimeUnit.SECONDS);
}
+ public SolrClient getSolrClient() {
+ return solrClient;
+ }
+
private void collectMetrics() {
+ // check that .system exists
+ try {
+ ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState();
+ if (clusterState.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL) == null) {
+ if (logMissingCollection) {
+ log.warn("Missing " + CollectionAdminParams.SYSTEM_COLL + ", skipping metrics collection");
+ logMissingCollection = false;
+ }
+ return;
+ }
+ } catch (Exception e) {
+ log.warn("Error getting cluster state, skipping metrics collection", e);
+ return;
+ }
+ logMissingCollection = true;
// get metrics
for (Group group : Arrays.asList(Group.core, Group.node, Group.jvm)) {
ModifiableSolrParams params = new ModifiableSolrParams();
@@ -148,6 +179,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
if (db == null) {
continue;
}
+ // set the timestamp
Sample s = db.createSample(TimeUnit.MILLISECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS));
NamedList<Object> values = (NamedList<Object>)entry.getValue();
AtomicBoolean dirty = new AtomicBoolean(false);
@@ -179,16 +211,23 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
RrdDef createDef(String registry, Group group) {
registry = SolrMetricManager.overridableRegistryName(registry);
+ // base sampling period is collectPeriod - samples more frequent than
+ // that will be dropped, samples less frequent will be interpolated
RrdDef def = new RrdDef(URI_PREFIX + registry, collectPeriod);
def.setStartTime(TimeUnit.MILLISECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS));
// add datasources
+
+ // use NaN when more than 1 sample is missing
counters.get(group.toString()).forEach(c ->
def.addDatasource(c, DsType.COUNTER, collectPeriod * 2, Double.NaN, Double.NaN));
gauges.get(group.toString()).forEach(g ->
def.addDatasource(g, DsType.GAUGE, collectPeriod * 2, Double.NaN, Double.NaN));
// add archives
+
+ // use AVERAGE consolidation,
+ // use NaN when >50% samples are missing
def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 120); // 2 hours
def.addArchive(ConsolFun.AVERAGE, 0.5, 10, 288); // 48 hours
def.addArchive(ConsolFun.AVERAGE, 0.5, 60, 336); // 2 weeks
@@ -197,6 +236,16 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
}
@Override
+ public void close() {
+ if (collectService != null) {
+ collectService.shutdown();
+ }
+ if (factory != null) {
+ factory.close();
+ }
+ }
+
+ @Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
index c395a53..956aabb 100644
--- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackend.java
@@ -101,6 +101,10 @@ public class SolrRrdBackend extends RrdByteArrayBackend implements Closeable {
}
}
+ public void markClean() {
+ dirty = false;
+ }
+
@Override
public void close() throws IOException {
super.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
index 1401ee3..ca673e4 100644
--- a/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
+++ b/solr/core/src/java/org/apache/solr/metrics/rrd/SolrRrdBackendFactory.java
@@ -18,10 +18,17 @@ package org.apache.solr.metrics.rrd;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.net.URI;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -42,6 +49,7 @@ import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.rrd4j.core.RrdBackend;
@@ -64,21 +72,25 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
public static final String DATA_FIELD = "data_bin";
private final SolrClient solrClient;
+ private final TimeSource timeSource;
private final String collection;
private ScheduledThreadPoolExecutor syncService;
- private int syncPeriod = DEFAULT_SYNC_PERIOD;
private volatile boolean closed = false;
private final Map<String, SolrRrdBackend> backends = new ConcurrentHashMap<>();
- public SolrRrdBackendFactory(SolrClient solrClient, String collection) {
+ public SolrRrdBackendFactory(SolrClient solrClient, String collection, int syncPeriod, TimeSource timeSource) {
this.solrClient = solrClient;
+ this.timeSource = timeSource;
this.collection = collection;
syncService = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(2,
new DefaultSolrThreadFactory("SolrRrdBackendFactory"));
syncService.setRemoveOnCancelPolicy(true);
syncService.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
- syncService.scheduleWithFixedDelay(() -> maybeSyncBackends(), syncPeriod, syncPeriod, TimeUnit.SECONDS);
+ syncService.scheduleWithFixedDelay(() -> maybeSyncBackends(),
+ timeSource.convertDelay(TimeUnit.SECONDS, syncPeriod, TimeUnit.MILLISECONDS),
+ timeSource.convertDelay(TimeUnit.SECONDS, syncPeriod, TimeUnit.MILLISECONDS),
+ TimeUnit.MILLISECONDS);
}
private void ensureOpen() throws IOException {
@@ -88,6 +100,23 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
}
@Override
+ public boolean canStore(URI uri) {
+ if (uri == null) {
+ return false;
+ }
+ if (uri.getScheme().toUpperCase(Locale.ROOT).equals(getName())) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public String getPath(URI uri) {
+ return uri.getSchemeSpecificPart();
+ }
+
+ @Override
protected synchronized RrdBackend open(String path, boolean readOnly) throws IOException {
ensureOpen();
SolrRrdBackend backend = backends.computeIfAbsent(path, p -> new SolrRrdBackend(p, readOnly, this));
@@ -143,7 +172,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
}
public List<String> list() throws IOException {
- ArrayList<String> names = new ArrayList<>();
+ Set<String> names = new HashSet<>();
try {
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CommonParams.Q, "*:*");
@@ -159,7 +188,11 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
} catch (SolrServerException e) {
log.warn("Error retrieving RRD list", e);
}
- return names;
+ // add in-memory backends not yet stored
+ names.addAll(backends.keySet());
+ ArrayList<String> list = new ArrayList<>(names);
+ Collections.sort(list);
+ return list;
}
public void remove(String path) throws IOException {
@@ -179,6 +212,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
if (closed) {
return;
}
+ log.info("-- maybe sync backends: " + backends.keySet());
Map<String, byte[]> syncData = new HashMap<>();
backends.forEach((path, backend) -> {
byte[] data = backend.getSyncData();
@@ -189,6 +223,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
if (syncData.isEmpty()) {
return;
}
+ log.info("-- syncing " + syncData.keySet());
// write updates
try {
syncData.forEach((path, data) -> {
@@ -196,6 +231,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
doc.setField("id", ID_PREFIX + path);
doc.addField(CommonParams.TYPE, DOC_TYPE);
doc.addField(DATA_FIELD, data);
+ doc.setField("timestamp", new Date(TimeUnit.MILLISECONDS.convert(timeSource.getEpochTimeNs(), TimeUnit.NANOSECONDS)));
try {
solrClient.add(collection, doc);
} catch (SolrServerException | IOException e) {
@@ -207,6 +243,12 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
} catch (SolrServerException e) {
log.warn("Error committing RRD data updates", e);
}
+ syncData.forEach((path, data) -> {
+ SolrRrdBackend backend = backends.get(path);
+ if (backend != null) {
+ backend.markClean();
+ }
+ });
} catch (IOException e) {
log.warn("Error sending RRD data updates", e);
}
@@ -214,6 +256,10 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
@Override
protected boolean exists(String path) throws IOException {
+ // check in-memory backends first
+ if (backends.containsKey(path)) {
+ return true;
+ }
try {
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CommonParams.Q, "{!term f=id}" + ID_PREFIX + path);
@@ -249,7 +295,7 @@ public class SolrRrdBackendFactory extends RrdBackendFactory implements SolrClos
}
@Override
- public void close() throws IOException {
+ public void close() {
closed = true;
backends.forEach((p, b) -> IOUtils.closeQuietly(b));
backends.clear();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index 0fe5ad7..392bdfc 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -48,7 +48,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
@Test
public void test() throws Exception {
- MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer().getMetricManager());
SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", MetricsHandler.COMPACT_PARAM, "false", CommonParams.WT, "json"), resp);
@@ -179,7 +179,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
@Test
public void testCompact() throws Exception {
- MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer().getMetricManager());
SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json", MetricsHandler.COMPACT_PARAM, "true"), resp);
@@ -197,7 +197,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
public void testPropertyFilter() throws Exception {
assertQ(req("*:*"), "//result[@numFound='0']");
- MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer().getMetricManager());
SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
@@ -234,7 +234,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
@Test
public void testKeyMetrics() throws Exception {
- MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer().getMetricManager());
String key1 = "solr.core.collection1:CACHE.core.fieldCache";
SolrQueryResponse resp = new SolrQueryResponse();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
new file mode 100644
index 0000000..58010fe
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.admin;
+
+import org.apache.solr.SolrTestCaseJ4;
+
+/**
+ *
+ */
+public class MetricsHistoryHandlerTest extends SolrTestCaseJ4 {
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
new file mode 100644
index 0000000..9c74a47
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.metrics.rrd;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.TimeSource;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.rrd4j.ConsolFun;
+import org.rrd4j.DsType;
+import org.rrd4j.core.FetchData;
+import org.rrd4j.core.FetchRequest;
+import org.rrd4j.core.RrdBackendFactory;
+import org.rrd4j.core.RrdDb;
+import org.rrd4j.core.RrdDef;
+import org.rrd4j.core.Sample;
+
+/**
+ *
+ */
+public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {
+
+ private SolrRrdBackendFactory factory;
+ private MockSolrClient solrClient;
+ private TimeSource timeSource;
+
+ @Before
+ public void setup() throws Exception {
+ solrClient = new MockSolrClient();
+ if (random().nextBoolean()) {
+ timeSource = TimeSource.NANO_TIME;
+ } else {
+ timeSource = TimeSource.get("simTime:50");
+ }
+ factory = new SolrRrdBackendFactory(solrClient, CollectionAdminParams.SYSTEM_COLL, 1, timeSource);
+ RrdBackendFactory.registerAndSetAsDefaultFactory(factory);
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (factory != null) {
+ factory.close();
+ }
+ }
+
+ private RrdDef createDef() {
+ RrdDef def = new RrdDef("solr:foo", 60);
+ def.addDatasource("one", DsType.COUNTER, 120, Double.NaN, Double.NaN);
+ def.addDatasource("two", DsType.GAUGE, 120, Double.NaN, Double.NaN);
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 1, 120); // 2 hours
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 10, 288); // 48 hours
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 60, 336); // 2 weeks
+ def.addArchive(ConsolFun.AVERAGE, 0.5, 240, 180); // 2 months
+ return def;
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ RrdDb db = new RrdDb(createDef());
+ List<String> list = factory.list();
+ assertEquals(list.toString(), 1, list.size());
+ assertEquals(list.toString(), "foo", list.get(0));
+ timeSource.sleep(2000);
+ // there should be one sync data
+ assertEquals(solrClient.docs.toString(), 1, solrClient.docs.size());
+ SolrInputDocument doc = solrClient.docs.get(SolrRrdBackendFactory.ID_PREFIX + "foo");
+ long timestamp = ((Date)doc.getFieldValue("timestamp")).getTime();
+ timeSource.sleep(2000);
+ SolrInputDocument newDoc = solrClient.docs.get(SolrRrdBackendFactory.ID_PREFIX + "foo");
+ assertEquals(newDoc.toString(), newDoc, doc);
+ long firstTimestamp = TimeUnit.SECONDS.convert(timestamp, TimeUnit.MILLISECONDS);
+ long lastTimestamp = firstTimestamp + 60;
+ // update the db
+ Sample s = db.createSample();
+ for (int i = 0; i < 100; i++) {
+ s.setTime(lastTimestamp);
+ s.setValue("one", 1000 + i * 60);
+ s.setValue("two", 100);
+ s.update();
+ lastTimestamp = lastTimestamp + 60;
+ }
+ timeSource.sleep(3000);
+ newDoc = solrClient.docs.get(SolrRrdBackendFactory.ID_PREFIX + "foo");
+ assertFalse(newDoc.toString(), newDoc.equals(doc));
+ long newTimestamp = ((Date)newDoc.getFieldValue("timestamp")).getTime();
+ assertNotSame(newTimestamp, timestamp);
+ FetchRequest fr = db.createFetchRequest(ConsolFun.AVERAGE, firstTimestamp + 60, lastTimestamp - 60, 60);
+ FetchData fd = fr.fetchData();
+ int rowCount = fd.getRowCount();
+ double[] one = fd.getValues("one");
+ assertEquals("one", 101, one.length);
+ assertEquals(Double.NaN, one[0], 0.00001);
+ assertEquals(Double.NaN, one[100], 0.00001);
+ for (int i = 1; i < 100; i++) {
+ assertEquals(1.0, one[i], 0.00001);
+ }
+ double[] two = fd.getValues("two");
+ assertEquals(Double.NaN, two[100], 0.00001);
+ for (int i = 0; i < 100; i++) {
+ assertEquals(100.0, two[i], 0.00001);
+ }
+ db.close();
+
+ // should still be listed
+ list = factory.list();
+ assertEquals(list.toString(), 1, list.size());
+ assertEquals(list.toString(), "foo", list.get(0));
+
+ // re-open read-write
+ db = new RrdDb("solr:foo");
+ s = db.createSample();
+ s.setTime(lastTimestamp);
+ s.setValue("one", 7000);
+ s.setValue("two", 100);
+ s.update();
+ timeSource.sleep(3000);
+ // should update
+ timestamp = newTimestamp;
+ doc = newDoc;
+ newDoc = solrClient.docs.get(SolrRrdBackendFactory.ID_PREFIX + "foo");
+ assertFalse(newDoc.toString(), newDoc.equals(doc));
+ newTimestamp = ((Date)newDoc.getFieldValue("timestamp")).getTime();
+ assertNotSame(newTimestamp, timestamp);
+ fr = db.createFetchRequest(ConsolFun.AVERAGE, firstTimestamp + 60, lastTimestamp, 60);
+ fd = fr.fetchData();
+ rowCount = fd.getRowCount();
+ one = fd.getValues("one");
+ assertEquals("one", 102, one.length);
+ assertEquals(Double.NaN, one[0], 0.00001);
+ assertEquals(Double.NaN, one[101], 0.00001);
+ for (int i = 1; i < 101; i++) {
+ assertEquals(1.0, one[i], 0.00001);
+ }
+ two = fd.getValues("two");
+ assertEquals(Double.NaN, two[101], 0.00001);
+ for (int i = 0; i < 101; i++) {
+ assertEquals(100.0, two[i], 0.00001);
+ }
+
+ // open a read-only version of the db
+ RrdDb readOnly = new RrdDb("solr:foo", true);
+ s = readOnly.createSample();
+ s.setTime(lastTimestamp + 120);
+ s.setValue("one", 10000001);
+ s.setValue("two", 100);
+ s.update();
+ // these updates should not be synced
+ timeSource.sleep(3000);
+ doc = newDoc;
+ timestamp = newTimestamp;
+ newDoc = solrClient.docs.get(SolrRrdBackendFactory.ID_PREFIX + "foo");
+ assertTrue(newDoc.toString(), newDoc.equals(doc));
+ newTimestamp = ((Date)newDoc.getFieldValue("timestamp")).getTime();
+ assertEquals(newTimestamp, timestamp);
+ }
+
+ static class MockSolrClient extends SolrClient {
+ LinkedHashMap<String, SolrInputDocument> docs = new LinkedHashMap<>();
+
+ @Override
+ public NamedList<Object> request(SolrRequest request, String collection) throws SolrServerException, IOException {
+ NamedList<Object> res = new NamedList<>();
+ if (request instanceof UpdateRequest) {
+ List<SolrInputDocument> docList = ((UpdateRequest)request).getDocuments();
+ if (docList != null) {
+ assertEquals(docList.toString(), 1, docList.size());
+ SolrInputDocument doc = docList.get(0);
+ String id = (String)doc.getFieldValue("id");
+ assertNotNull(doc.toString(), id);
+ docs.put(id, doc);
+ }
+ } else if (request instanceof QueryRequest) {
+ SolrParams params = request.getParams();
+ String query = params.get("q");
+ final SolrDocumentList lst = new SolrDocumentList();
+ if (query != null) {
+ if (query.startsWith("{!term f=id}")) {
+ String id = query.substring(12);
+ SolrInputDocument doc = docs.get(id);
+ if (doc != null) {
+ SolrDocument d = new SolrDocument();
+ doc.forEach((k, f) -> {
+ f.forEach(v -> d.addField(k, v));
+ });
+ lst.add(d);
+ lst.setNumFound(1);
+ }
+ } else if (query.equals("*:*")) {
+ if (!docs.isEmpty()) {
+ lst.setNumFound(docs.size());
+ docs.values().forEach(doc -> {
+ SolrDocument d = new SolrDocument();
+ doc.forEach((k, f) -> {
+ f.forEach(v -> d.addField(k, v));
+ });
+ lst.add(d);
+ });
+ }
+ }
+ }
+ res.add("response", lst);
+ }
+ return res;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b5fa6653/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
index a67f433..899214d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
@@ -183,6 +183,7 @@ public interface CommonParams {
String AUTHC_PATH = "/admin/authentication";
String ZK_PATH = "/admin/zookeeper";
String METRICS_PATH = "/admin/metrics";
+ String METRICS_HISTORY_PATH = "/admin/metrics/history";
String AUTOSCALING_PATH = "/admin/autoscaling";
String AUTOSCALING_HISTORY_PATH = "/admin/autoscaling/history";
String AUTOSCALING_DIAGNOSTICS_PATH = "/admin/autoscaling/diagnostics";
[43/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12290: We must
close ContentStreams because we don't know the source of the inputstream -
use a CloseShield to prevent tripping our close assert in SolrDispatchFilter.
Posted by ab...@apache.org.
SOLR-12290: We must close ContentStreams because we don't know the source of the inputstream - use a CloseShield to prevent tripping our close assert in SolrDispatchFilter.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5fc72515
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5fc72515
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5fc72515
Branch: refs/heads/jira/solr-11779
Commit: 5fc725154001c6283315802e1a2193d51d00f9aa
Parents: 0922e58
Author: markrmiller <ma...@apache.org>
Authored: Sun May 6 14:25:59 2018 -0500
Committer: markrmiller <ma...@apache.org>
Committed: Sun May 6 14:25:59 2018 -0500
----------------------------------------------------------------------
.../org/apache/solr/handler/BlobHandler.java | 6 +-
.../solr/handler/loader/CSVLoaderBase.java | 75 +++++++++++---------
.../solr/handler/loader/JavabinLoader.java | 16 ++++-
.../apache/solr/servlet/SolrRequestParsers.java | 6 +-
4 files changed, 64 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fc72515/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
index a998657..30301c0 100644
--- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
@@ -17,6 +17,7 @@
package org.apache.solr.handler;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.math.BigInteger;
@@ -108,8 +109,9 @@ public class BlobHandler extends RequestHandlerBase implements PluginInfoInitial
for (ContentStream stream : req.getContentStreams()) {
ByteBuffer payload;
- payload = SimplePostTool.inputStreamToByteArray(stream.getStream(), maxSize);
-
+ try (InputStream is = stream.getStream()) {
+ payload = SimplePostTool.inputStreamToByteArray(is, maxSize);
+ }
MessageDigest m = MessageDigest.getInstance("MD5");
m.update(payload.array(), payload.position(), payload.limit());
String md5 = new BigInteger(1, m.digest()).toString(16);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fc72515/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
index fd8935d..b503fa3 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
@@ -28,6 +28,7 @@ import org.apache.solr.update.*;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.internal.csv.CSVStrategy;
import org.apache.solr.internal.csv.CSVParser;
+import org.apache.commons.io.IOUtils;
import java.util.regex.Pattern;
import java.util.List;
@@ -314,48 +315,54 @@ abstract class CSVLoaderBase extends ContentStreamLoader {
/** load the CSV input */
@Override
- public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor processor)
- throws IOException {
+ public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor processor) throws IOException {
errHeader = "CSVLoader: input=" + stream.getSourceInfo();
- Reader reader = stream.getReader();
- if (skipLines > 0) {
- if (!(reader instanceof BufferedReader)) {
- reader = new BufferedReader(reader);
- }
- BufferedReader r = (BufferedReader) reader;
- for (int i = 0; i < skipLines; i++) {
- r.readLine();
+ Reader reader = null;
+ try {
+ reader = stream.getReader();
+ if (skipLines>0) {
+ if (!(reader instanceof BufferedReader)) {
+ reader = new BufferedReader(reader);
+ }
+ BufferedReader r = (BufferedReader)reader;
+ for (int i=0; i<skipLines; i++) {
+ r.readLine();
+ }
}
- }
- CSVParser parser = new CSVParser(reader, strategy);
+ CSVParser parser = new CSVParser(reader, strategy);
- // parse the fieldnames from the header of the file
- if (fieldnames == null) {
- fieldnames = parser.getLine();
- if (fieldnames == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Expected fieldnames in CSV input");
+ // parse the fieldnames from the header of the file
+ if (fieldnames==null) {
+ fieldnames = parser.getLine();
+ if (fieldnames==null) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Expected fieldnames in CSV input");
+ }
+ prepareFields();
}
- prepareFields();
- }
- // read the rest of the CSV file
- for (;;) {
- int line = parser.getLineNumber(); // for error reporting in MT mode
- String[] vals = null;
- try {
- vals = parser.getLine();
- } catch (IOException e) {
- // Catch the exception and rethrow it with more line information
- input_err("can't read line: " + line, null, line, e);
- }
- if (vals == null) break;
+ // read the rest of the CSV file
+ for(;;) {
+ int line = parser.getLineNumber(); // for error reporting in MT mode
+ String[] vals = null;
+ try {
+ vals = parser.getLine();
+ } catch (IOException e) {
+ //Catch the exception and rethrow it with more line information
+ input_err("can't read line: " + line, null, line, e);
+ }
+ if (vals==null) break;
- if (vals.length != fieldnames.length) {
- input_err("expected " + fieldnames.length + " values but got " + vals.length, vals, line);
- }
+ if (vals.length != fieldnames.length) {
+ input_err("expected "+fieldnames.length+" values but got "+vals.length, vals, line);
+ }
- addDoc(line, vals);
+ addDoc(line,vals);
+ }
+ } finally{
+ if (reader != null) {
+ IOUtils.closeQuietly(reader);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fc72515/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
index aca3df4..f502a8e 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
@@ -48,8 +48,20 @@ import org.apache.solr.update.processor.UpdateRequestProcessor;
public class JavabinLoader extends ContentStreamLoader {
@Override
- public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream cs, UpdateRequestProcessor processor) throws Exception {
- InputStream stream = cs.getStream();
+ public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor processor) throws Exception {
+ InputStream is = null;
+ try {
+ is = stream.getStream();
+ parseAndLoadDocs(req, rsp, is, processor);
+ } finally {
+ if(is != null) {
+ is.close();
+ }
+ }
+ }
+
+ private void parseAndLoadDocs(final SolrQueryRequest req, SolrQueryResponse rsp, InputStream stream,
+ final UpdateRequestProcessor processor) throws IOException {
UpdateRequest update = null;
JavaBinUpdateRequestCodec.StreamingUpdateHandler handler = new JavaBinUpdateRequestCodec.StreamingUpdateHandler() {
private AddUpdateCommand addCmd = null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fc72515/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
index 4bcb8d8..8a664c4 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
@@ -519,7 +519,11 @@ public class SolrRequestParsers
@Override
public InputStream getStream() throws IOException {
- return req.getInputStream();
+ // we explicitly protect this servlet stream from being closed
+ // so that it does not trip our test assert in our close shield
+ // in SolrDispatchFilter - we must allow closes from getStream
+ // due to the other impls of ContentStream
+ return new CloseShieldInputStream(req.getInputStream());
}
}
[20/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8142: Make
postings APIs expose raw impacts rather than scores.
Posted by ab...@apache.org.
LUCENE-8142: Make postings APIs expose raw impacts rather than scores.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/af680af7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/af680af7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/af680af7
Branch: refs/heads/jira/solr-11779
Commit: af680af77f3f80c779e038a0ad8a136c9dcb9f5d
Parents: 555b7ef
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed May 2 11:52:36 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed May 2 14:49:32 2018 +0200
----------------------------------------------------------------------
.../codecs/blockterms/BlockTermsReader.java | 7 +-
.../blocktreeords/OrdsIntersectTermsEnum.java | 5 +-
.../blocktreeords/OrdsSegmentTermsEnum.java | 5 +-
.../bloom/BloomFilteringPostingsFormat.java | 7 +-
.../codecs/memory/DirectPostingsFormat.java | 9 +-
.../lucene/codecs/memory/FSTOrdTermsReader.java | 7 +-
.../lucene/codecs/memory/FSTTermsReader.java | 5 +-
.../codecs/memory/MemoryDocValuesProducer.java | 7 +-
.../codecs/memory/MemoryPostingsFormat.java | 7 +-
.../simpletext/SimpleTextFieldsReader.java | 5 +-
.../simpletext/SimpleTextTermVectorsReader.java | 7 +-
.../codecs/CompetitiveFreqNormAccumulator.java | 163 -------------
.../codecs/CompetitiveImpactAccumulator.java | 127 ++++++++++
.../lucene/codecs/PostingsReaderBase.java | 3 +-
.../codecs/blocktree/IntersectTermsEnum.java | 5 +-
.../codecs/blocktree/SegmentTermsEnum.java | 5 +-
.../CompressingTermVectorsReader.java | 6 +-
.../codecs/lucene50/Lucene50PostingsReader.java | 39 ++-
.../codecs/lucene50/Lucene50PostingsWriter.java | 4 +-
.../lucene50/Lucene50ScoreSkipReader.java | 141 +++++++----
.../codecs/lucene50/Lucene50SkipReader.java | 3 +-
.../codecs/lucene50/Lucene50SkipWriter.java | 36 +--
.../lucene70/Lucene70DocValuesProducer.java | 3 +-
.../apache/lucene/document/FeatureQuery.java | 8 +-
.../org/apache/lucene/index/CheckIndex.java | 152 ++++++++----
.../apache/lucene/index/FilterLeafReader.java | 5 +-
.../apache/lucene/index/FilteredTermsEnum.java | 5 +-
.../org/apache/lucene/index/FreqProxFields.java | 3 +-
.../java/org/apache/lucene/index/Impact.java | 61 +++++
.../java/org/apache/lucene/index/Impacts.java | 51 ++++
.../org/apache/lucene/index/ImpactsEnum.java | 39 +--
.../org/apache/lucene/index/MultiTermsEnum.java | 5 +-
.../apache/lucene/index/SlowImpactsEnum.java | 39 ++-
.../lucene/index/SortedDocValuesTermsEnum.java | 3 +-
.../index/SortedSetDocValuesTermsEnum.java | 3 +-
.../java/org/apache/lucene/index/TermsEnum.java | 7 +-
.../search/BlockMaxConjunctionScorer.java | 3 +
.../apache/lucene/search/FuzzyTermsEnum.java | 5 +-
.../org/apache/lucene/search/MaxScoreCache.java | 138 +++++++++++
.../java/org/apache/lucene/search/Scorer.java | 4 +-
.../org/apache/lucene/search/TermScorer.java | 40 ++-
.../org/apache/lucene/search/WANDScorer.java | 1 +
.../TestCompetitiveFreqNormAccumulator.java | 44 ++--
.../lucene50/TestBlockPostingsFormat.java | 57 +++--
.../org/apache/lucene/index/TestCodecs.java | 3 +-
.../apache/lucene/index/memory/MemoryIndex.java | 5 +-
.../idversion/IDVersionPostingsReader.java | 5 +-
.../idversion/IDVersionSegmentTermsEnum.java | 5 +-
.../codecs/ramonly/RAMOnlyPostingsFormat.java | 5 +-
.../lucene/index/AssertingLeafReader.java | 55 ++++-
.../lucene/index/RandomPostingsTester.java | 241 ++++++++++---------
.../apache/lucene/search/AssertingScorer.java | 1 +
.../org/apache/lucene/search/CheckHits.java | 2 +-
.../org/apache/solr/query/SolrRangeQuery.java | 5 +-
.../apache/solr/uninverting/DocTermOrds.java | 5 +-
55 files changed, 1004 insertions(+), 607 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
index a405ccb..4260dc3 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
@@ -30,16 +30,15 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
@@ -661,9 +660,9 @@ public class BlockTermsReader extends FieldsProducer {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
decodeMetaData();
- return postingsReader.impacts(fieldInfo, state, scorer, flags);
+ return postingsReader.impacts(fieldInfo, state, flags);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java
index fdb54df..a892549 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java
@@ -24,7 +24,6 @@ import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -208,9 +207,9 @@ final class OrdsIntersectTermsEnum extends TermsEnum {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
currentFrame.decodeMetaData();
- return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.termState, scorer, flags);
+ return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.termState, flags);
}
private int getState() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java
index 8d55a1d..bd67adc 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java
@@ -29,7 +29,6 @@ import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
@@ -936,7 +935,7 @@ public final class OrdsSegmentTermsEnum extends TermsEnum {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
assert !eof;
//if (DEBUG) {
//System.out.println("BTTR.docs seg=" + segment);
@@ -945,7 +944,7 @@ public final class OrdsSegmentTermsEnum extends TermsEnum {
//if (DEBUG) {
//System.out.println(" state=" + currentFrame.state);
//}
- return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.state, scorer, flags);
+ return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.state, flags);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
index b826102..28febf3 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
@@ -32,16 +32,15 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.bloom.FuzzySet.ContainsResult;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
@@ -375,8 +374,8 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return delegate().impacts(scorer, flags);
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return delegate().impacts(flags);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
index 901e1ae..6c8853d 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
@@ -39,7 +39,6 @@ import org.apache.lucene.index.SlowImpactsEnum;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.Accountable;
@@ -948,8 +947,8 @@ public final class DirectPostingsFormat extends PostingsFormat {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return new SlowImpactsEnum(postings(null, flags), scorer.score(Float.MAX_VALUE, 1));
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return new SlowImpactsEnum(postings(null, flags));
}
}
@@ -1503,8 +1502,8 @@ public final class DirectPostingsFormat extends PostingsFormat {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return new SlowImpactsEnum(postings(null, flags), scorer.score(Float.MAX_VALUE, 1));
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return new SlowImpactsEnum(postings(null, flags));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
index 4ecf4d6..2b948ff 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
@@ -31,18 +31,17 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
@@ -435,9 +434,9 @@ public class FSTOrdTermsReader extends FieldsProducer {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
decodeMetaData();
- return postingsReader.impacts(fieldInfo, state, scorer, flags);
+ return postingsReader.impacts(fieldInfo, state, flags);
}
// TODO: this can be achieved by making use of Util.getByOutput()
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
index b1b61e1..7afbc7c 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
@@ -42,7 +42,6 @@ import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
@@ -301,9 +300,9 @@ public class FSTTermsReader extends FieldsProducer {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
decodeMetaData();
- return postingsReader.impacts(fieldInfo, state, scorer, flags);
+ return postingsReader.impacts(fieldInfo, state, flags);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
index 855002c..ac81360 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
@@ -29,7 +29,6 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.*;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
@@ -44,11 +43,11 @@ import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
import org.apache.lucene.util.fst.BytesRefFSTEnum;
+import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
+import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FST.Arc;
import org.apache.lucene.util.fst.FST.BytesReader;
-import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.Util;
import org.apache.lucene.util.packed.BlockPackedReader;
@@ -871,7 +870,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
index 0df7d92..2433de7 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
@@ -31,19 +31,18 @@ import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SlowImpactsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IOContext;
@@ -819,8 +818,8 @@ public final class MemoryPostingsFormat extends PostingsFormat {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return new SlowImpactsEnum(postings(null, flags), scorer.score(Float.MAX_VALUE, 1));
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return new SlowImpactsEnum(postings(null, flags));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
index cbd79de..743dc4f 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
@@ -36,7 +36,6 @@ import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SlowImpactsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.BufferedChecksumIndexInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
@@ -234,8 +233,8 @@ class SimpleTextFieldsReader extends FieldsProducer {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return new SlowImpactsEnum(postings(null, flags), scorer.score(Float.MAX_VALUE, 1));
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return new SlowImpactsEnum(postings(null, flags));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
index ee0757d..a306e8c 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
@@ -25,15 +25,14 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.lucene.codecs.TermVectorsReader;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SlowImpactsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.BufferedChecksumIndexInput;
import org.apache.lucene.store.ChecksumIndexInput;
@@ -414,8 +413,8 @@ public class SimpleTextTermVectorsReader extends TermVectorsReader {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return new SlowImpactsEnum(postings(null, PostingsEnum.FREQS), scorer.score(Float.MAX_VALUE, 1));
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return new SlowImpactsEnum(postings(null, PostingsEnum.FREQS));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveFreqNormAccumulator.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveFreqNormAccumulator.java b/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveFreqNormAccumulator.java
deleted file mode 100644
index 3dd9d35..0000000
--- a/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveFreqNormAccumulator.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.codecs;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-/**
- * This class accumulates the (freq, norm) pairs that may produce competitive scores.
- */
-public final class CompetitiveFreqNormAccumulator {
-
- // We speed up accumulation for common norm values by first computing
- // the max freq for all norms in -128..127
- private final int[] maxFreqs;
- private boolean dirty;
- private final TreeSet<FreqAndNorm> freqNormPairs;
-
- /** Sole constructor. */
- public CompetitiveFreqNormAccumulator() {
- maxFreqs = new int[256];
- Comparator<FreqAndNorm> comparator = new Comparator<CompetitiveFreqNormAccumulator.FreqAndNorm>() {
- @Override
- public int compare(FreqAndNorm o1, FreqAndNorm o2) {
- // greater freqs compare greater
- int cmp = Integer.compare(o1.freq, o2.freq);
- if (cmp == 0) {
- // greater norms compare lower
- cmp = Long.compareUnsigned(o2.norm, o1.norm);
- }
- return cmp;
- }
- };
- freqNormPairs = new TreeSet<>(comparator);
- }
-
- /** Reset to the same state it was in after creation. */
- public void clear() {
- Arrays.fill(maxFreqs, 0);
- dirty = false;
- freqNormPairs.clear();
- }
-
- /**
- * A (freq, norm) pair.
- */
- public static class FreqAndNorm {
- /** Doc-term frequency. */
- public final int freq;
- /** Normalization factor. */
- public final long norm;
-
- /** Sole constructor. */
- public FreqAndNorm(int freq, long norm) {
- this.freq = freq;
- this.norm = norm;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null || obj instanceof FreqAndNorm == false) {
- return false;
- }
- FreqAndNorm that = (FreqAndNorm) obj;
- return freq == that.freq && norm == that.norm;
- }
-
- @Override
- public int hashCode() {
- int h = getClass().hashCode();
- h = 31 * h + freq;
- h = 31 * h + Long.hashCode(norm);
- return h;
- }
-
- @Override
- public String toString() {
- return "{" + freq + "," + norm + "}";
- }
- }
-
- /** Accumulate a (freq,norm) pair, updating this structure if there is no
- * equivalent or more competitive entry already. */
- public void add(int freq, long norm) {
- if (norm >= Byte.MIN_VALUE && norm <= Byte.MAX_VALUE) {
- int index = Byte.toUnsignedInt((byte) norm);
- maxFreqs[index] = Math.max(maxFreqs[index], freq);
- dirty = true;
- } else {
- add(new FreqAndNorm(freq, norm));
- }
- }
-
- /** Merge {@code acc} into this. */
- public void addAll(CompetitiveFreqNormAccumulator acc) {
- for (FreqAndNorm entry : acc.getCompetitiveFreqNormPairs()) {
- add(entry);
- }
- }
-
- /** Get the set of competitive freq and norm pairs, orderer by increasing freq and norm. */
- public SortedSet<FreqAndNorm> getCompetitiveFreqNormPairs() {
- if (dirty) {
- for (int i = 0; i < maxFreqs.length; ++i) {
- if (maxFreqs[i] > 0) {
- add(new FreqAndNorm(maxFreqs[i], (byte) i));
- maxFreqs[i] = 0;
- }
- }
- dirty = false;
- }
- return Collections.unmodifiableSortedSet(freqNormPairs);
- }
-
- private void add(FreqAndNorm newEntry) {
- FreqAndNorm next = freqNormPairs.ceiling(newEntry);
- if (next == null) {
- // nothing is more competitive
- freqNormPairs.add(newEntry);
- } else if (Long.compareUnsigned(next.norm, newEntry.norm) <= 0) {
- // we already have this entry or more competitive entries in the tree
- return;
- } else {
- // some entries have a greater freq but a less competitive norm, so we
- // don't know which one will trigger greater scores, still add to the tree
- freqNormPairs.add(newEntry);
- }
-
- for (Iterator<FreqAndNorm> it = freqNormPairs.headSet(newEntry, false).descendingIterator(); it.hasNext(); ) {
- FreqAndNorm entry = it.next();
- if (Long.compareUnsigned(entry.norm, newEntry.norm) >= 0) {
- // less competitive
- it.remove();
- } else {
- // lesser freq but better norm, further entries are not comparable
- break;
- }
- }
- }
-
- @Override
- public String toString() {
- return getCompetitiveFreqNormPairs().toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveImpactAccumulator.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveImpactAccumulator.java b/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveImpactAccumulator.java
new file mode 100644
index 0000000..34f7d79
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveImpactAccumulator.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.lucene.index.Impact;
+
+/**
+ * This class accumulates the (freq, norm) pairs that may produce competitive scores.
+ */
+public final class CompetitiveImpactAccumulator {
+
+ // We speed up accumulation for common norm values by first computing
+ // the max freq for all norms in -128..127
+ private final int[] maxFreqs;
+ private boolean dirty;
+ private final TreeSet<Impact> freqNormPairs;
+
+ /** Sole constructor. */
+ public CompetitiveImpactAccumulator() {
+ maxFreqs = new int[256];
+ Comparator<Impact> comparator = new Comparator<Impact>() {
+ @Override
+ public int compare(Impact o1, Impact o2) {
+ // greater freqs compare greater
+ int cmp = Integer.compare(o1.freq, o2.freq);
+ if (cmp == 0) {
+ // greater norms compare lower
+ cmp = Long.compareUnsigned(o2.norm, o1.norm);
+ }
+ return cmp;
+ }
+ };
+ freqNormPairs = new TreeSet<>(comparator);
+ }
+
+ /** Reset to the same state it was in after creation. */
+ public void clear() {
+ Arrays.fill(maxFreqs, 0);
+ dirty = false;
+ freqNormPairs.clear();
+ }
+
+ /** Accumulate a (freq,norm) pair, updating this structure if there is no
+ * equivalent or more competitive entry already. */
+ public void add(int freq, long norm) {
+ if (norm >= Byte.MIN_VALUE && norm <= Byte.MAX_VALUE) {
+ int index = Byte.toUnsignedInt((byte) norm);
+ maxFreqs[index] = Math.max(maxFreqs[index], freq);
+ dirty = true;
+ } else {
+ add(new Impact(freq, norm));
+ }
+ }
+
+ /** Merge {@code acc} into this. */
+ public void addAll(CompetitiveImpactAccumulator acc) {
+ for (Impact entry : acc.getCompetitiveFreqNormPairs()) {
+ add(entry);
+ }
+ }
+
+ /** Get the set of competitive freq and norm pairs, orderer by increasing freq and norm. */
+ public SortedSet<Impact> getCompetitiveFreqNormPairs() {
+ if (dirty) {
+ for (int i = 0; i < maxFreqs.length; ++i) {
+ if (maxFreqs[i] > 0) {
+ add(new Impact(maxFreqs[i], (byte) i));
+ maxFreqs[i] = 0;
+ }
+ }
+ dirty = false;
+ }
+ return Collections.unmodifiableSortedSet(freqNormPairs);
+ }
+
+ private void add(Impact newEntry) {
+ Impact next = freqNormPairs.ceiling(newEntry);
+ if (next == null) {
+ // nothing is more competitive
+ freqNormPairs.add(newEntry);
+ } else if (Long.compareUnsigned(next.norm, newEntry.norm) <= 0) {
+ // we already have this entry or more competitive entries in the tree
+ return;
+ } else {
+ // some entries have a greater freq but a less competitive norm, so we
+ // don't know which one will trigger greater scores, still add to the tree
+ freqNormPairs.add(newEntry);
+ }
+
+ for (Iterator<Impact> it = freqNormPairs.headSet(newEntry, false).descendingIterator(); it.hasNext(); ) {
+ Impact entry = it.next();
+ if (Long.compareUnsigned(entry.norm, newEntry.norm) >= 0) {
+ // less competitive
+ it.remove();
+ } else {
+ // lesser freq but better norm, further entries are not comparable
+ break;
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getCompetitiveFreqNormPairs().toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java b/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java
index ca403fa..4fed1a0 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java
@@ -24,7 +24,6 @@ import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
@@ -72,7 +71,7 @@ public abstract class PostingsReaderBase implements Closeable, Accountable {
* Return a {@link ImpactsEnum} that computes impacts with {@code scorer}.
* @see #postings(FieldInfo, BlockTermState, PostingsEnum, int)
*/
- public abstract ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, Similarity.SimScorer scorer, int flags) throws IOException;
+ public abstract ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags) throws IOException;
/**
* Checks consistency of this reader.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
index 6bccddc..bbd7e7b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java
@@ -24,7 +24,6 @@ import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -235,9 +234,9 @@ final class IntersectTermsEnum extends TermsEnum {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
currentFrame.decodeMetaData();
- return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.termState, scorer, flags);
+ return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.termState, flags);
}
private int getState() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java
index ef83f49..327c181 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java
@@ -25,7 +25,6 @@ import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
@@ -1005,7 +1004,7 @@ final class SegmentTermsEnum extends TermsEnum {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
assert !eof;
//if (DEBUG) {
//System.out.println("BTTR.docs seg=" + segment);
@@ -1014,7 +1013,7 @@ final class SegmentTermsEnum extends TermsEnum {
//if (DEBUG) {
//System.out.println(" state=" + currentFrame.state);
//}
- return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.state, scorer, flags);
+ return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.state, flags);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
index a0f5292..cbe1050 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
@@ -37,7 +37,6 @@ import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
@@ -946,10 +945,9 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
final PostingsEnum delegate = postings(null, PostingsEnum.FREQS);
- final float maxScore = scorer.score(Float.MAX_VALUE, 1);
- return new SlowImpactsEnum(delegate, maxScore);
+ return new SlowImpactsEnum(delegate);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java
index 62c7f3f..ede8bdd 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java
@@ -19,20 +19,19 @@ package org.apache.lucene.codecs.lucene50;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Objects;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.Impacts;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SlowImpactsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
@@ -239,13 +238,12 @@ public final class Lucene50PostingsReader extends PostingsReaderBase {
}
@Override
- public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, SimScorer scorer, int flags) throws IOException {
- Objects.requireNonNull(scorer);
+ public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags) throws IOException {
if (state.docFreq <= BLOCK_SIZE || version < Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) {
// no skip data
- return new SlowImpactsEnum(postings(fieldInfo, state, null, flags), scorer.score(Float.MAX_VALUE, 1));
+ return new SlowImpactsEnum(postings(fieldInfo, state, null, flags));
}
- return new BlockImpactsEverythingEnum(fieldInfo, (IntBlockTermState) state, scorer, flags);
+ return new BlockImpactsEverythingEnum(fieldInfo, (IntBlockTermState) state, flags);
}
final class BlockDocsEnum extends PostingsEnum {
@@ -1367,7 +1365,7 @@ public final class Lucene50PostingsReader extends PostingsReaderBase {
private long seekTo = -1;
- public BlockImpactsEverythingEnum(FieldInfo fieldInfo, IntBlockTermState termState, SimScorer scorer, int flags) throws IOException {
+ public BlockImpactsEverythingEnum(FieldInfo fieldInfo, IntBlockTermState termState, int flags) throws IOException {
indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
indexHasPos = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
@@ -1440,8 +1438,7 @@ public final class Lucene50PostingsReader extends PostingsReaderBase {
MAX_SKIP_LEVELS,
indexHasPos,
indexHasOffsets,
- indexHasPayloads,
- scorer);
+ indexHasPayloads);
skipper.init(docTermStartFP+termState.skipOffset, docTermStartFP, posTermStartFP, payTermStartFP, docFreq);
if (indexHasFreq == false) {
@@ -1544,17 +1541,7 @@ public final class Lucene50PostingsReader extends PostingsReaderBase {
}
@Override
- public int nextDoc() throws IOException {
- return advance(doc + 1);
- }
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return skipper.getMaxScore(upTo);
- }
-
- @Override
- public int advanceShallow(int target) throws IOException {
+ public void advanceShallow(int target) throws IOException {
if (target > nextSkipDoc) {
// always plus one to fix the result, since skip position in Lucene50SkipReader
// is a little different from MultiLevelSkipListReader
@@ -1580,7 +1567,17 @@ public final class Lucene50PostingsReader extends PostingsReaderBase {
nextSkipDoc = skipper.getNextSkipDoc();
}
assert nextSkipDoc >= target;
- return nextSkipDoc;
+ }
+
+ @Override
+ public Impacts getImpacts() throws IOException {
+ advanceShallow(doc);
+ return skipper.getImpacts();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return advance(doc + 1);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java
index 06b9a0c..a600e61 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java
@@ -31,7 +31,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator;
+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.PushPostingsWriterBase;
import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.CorruptIndexException;
@@ -101,7 +101,7 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase {
private boolean fieldHasNorms;
private NumericDocValues norms;
- private final CompetitiveFreqNormAccumulator competitiveFreqNormAccumulator = new CompetitiveFreqNormAccumulator();
+ private final CompetitiveImpactAccumulator competitiveFreqNormAccumulator = new CompetitiveImpactAccumulator();
/** Creates a postings writer */
public Lucene50PostingsWriter(SegmentWriteState state) throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50ScoreSkipReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50ScoreSkipReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50ScoreSkipReader.java
index cb1f54a..ad69616 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50ScoreSkipReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50ScoreSkipReader.java
@@ -17,90 +17,143 @@
package org.apache.lucene.codecs.lucene50;
import java.io.IOException;
+import java.util.AbstractList;
import java.util.Arrays;
-import java.util.Objects;
+import java.util.List;
+import java.util.RandomAccess;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
+import org.apache.lucene.index.Impact;
+import org.apache.lucene.index.Impacts;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
final class Lucene50ScoreSkipReader extends Lucene50SkipReader {
- private final SimScorer scorer;
- private final float[] maxScore;
- private final byte[][] impacts;
- private final int[] impactsLength;
- private final float globalMaxScore;
+ private final byte[][] impactData;
+ private final int[] impactDataLength;
private final ByteArrayDataInput badi = new ByteArrayDataInput();
+ private final Impacts impacts;
+ private int numLevels = 1;
+ private final MutableImpactList[] perLevelImpacts;
public Lucene50ScoreSkipReader(int version, IndexInput skipStream, int maxSkipLevels,
- boolean hasPos, boolean hasOffsets, boolean hasPayloads, SimScorer scorer) {
+ boolean hasPos, boolean hasOffsets, boolean hasPayloads) {
super(version, skipStream, maxSkipLevels, hasPos, hasOffsets, hasPayloads);
if (version < Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) {
throw new IllegalStateException("Cannot skip based on scores if impacts are not indexed");
}
- this.scorer = Objects.requireNonNull(scorer);
- this.maxScore = new float[maxSkipLevels];
- this.impacts = new byte[maxSkipLevels][];
- Arrays.fill(impacts, new byte[0]);
- this.impactsLength = new int[maxSkipLevels];
- this.globalMaxScore = scorer.score(Float.MAX_VALUE, 1);
- }
+ this.impactData = new byte[maxSkipLevels][];
+ Arrays.fill(impactData, new byte[0]);
+ this.impactDataLength = new int[maxSkipLevels];
+ this.perLevelImpacts = new MutableImpactList[maxSkipLevels];
+ for (int i = 0; i < perLevelImpacts.length; ++i) {
+ perLevelImpacts[i] = new MutableImpactList();
+ }
+ impacts = new Impacts() {
- @Override
- public void init(long skipPointer, long docBasePointer, long posBasePointer, long payBasePointer, int df) throws IOException {
- super.init(skipPointer, docBasePointer, posBasePointer, payBasePointer, df);
- Arrays.fill(impactsLength, 0);
- Arrays.fill(maxScore, globalMaxScore);
- }
+ @Override
+ public int numLevels() {
+ return numLevels;
+ }
- /** Upper bound of scores up to {@code upTo} included. */
- public float getMaxScore(int upTo) throws IOException {
- for (int level = 0; level < numberOfSkipLevels; ++level) {
- if (upTo <= skipDoc[level]) {
- return maxScore(level);
+ @Override
+ public int getDocIdUpTo(int level) {
+ return skipDoc[level];
}
- }
- return globalMaxScore;
+
+ @Override
+ public List<Impact> getImpacts(int level) {
+ assert level < numLevels;
+ if (impactDataLength[level] > 0) {
+ badi.reset(impactData[level], 0, impactDataLength[level]);
+ perLevelImpacts[level] = readImpacts(badi, perLevelImpacts[level]);
+ impactDataLength[level] = 0;
+ }
+ return perLevelImpacts[level];
+ }
+ };
}
- private float maxScore(int level) throws IOException {
- assert level < numberOfSkipLevels;
- if (impactsLength[level] > 0) {
- badi.reset(impacts[level], 0, impactsLength[level]);
- maxScore[level] = readImpacts(badi, scorer);
- impactsLength[level] = 0;
+ @Override
+ public int skipTo(int target) throws IOException {
+ int result = super.skipTo(target);
+ if (numberOfSkipLevels > 0) {
+ numLevels = numberOfSkipLevels;
+ } else {
+ // End of postings don't have skip data anymore, so we fill with dummy data
+ // like SlowImpactsEnum.
+ numLevels = 1;
+ perLevelImpacts[0].length = 1;
+ perLevelImpacts[0].impacts[0].freq = Integer.MAX_VALUE;
+ perLevelImpacts[0].impacts[0].norm = 1L;
+ impactDataLength[0] = 0;
}
- return maxScore[level];
+ return result;
+ }
+
+ Impacts getImpacts() {
+ return impacts;
}
@Override
protected void readImpacts(int level, IndexInput skipStream) throws IOException {
int length = skipStream.readVInt();
- if (impacts[level].length < length) {
- impacts[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
+ if (impactData[level].length < length) {
+ impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
}
- skipStream.readBytes(impacts[level], 0, length);
- impactsLength[level] = length;
+ skipStream.readBytes(impactData[level], 0, length);
+ impactDataLength[level] = length;
}
- static float readImpacts(ByteArrayDataInput in, SimScorer scorer) throws IOException {
+ static MutableImpactList readImpacts(ByteArrayDataInput in, MutableImpactList reuse) {
+ int maxNumImpacts = in.length(); // at most one impact per byte
+ if (reuse.impacts.length < maxNumImpacts) {
+ int oldLength = reuse.impacts.length;
+ reuse.impacts = ArrayUtil.grow(reuse.impacts, maxNumImpacts);
+ for (int i = oldLength; i < reuse.impacts.length; ++i) {
+ reuse.impacts[i] = new Impact(Integer.MAX_VALUE, 1L);
+ }
+ }
+
int freq = 0;
long norm = 0;
- float maxScore = 0;
+ int length = 0;
while (in.getPosition() < in.length()) {
int freqDelta = in.readVInt();
if ((freqDelta & 0x01) != 0) {
freq += 1 + (freqDelta >>> 1);
- norm += 1 + in.readZLong();
+ try {
+ norm += 1 + in.readZLong();
+ } catch (IOException e) {
+ throw new RuntimeException(e); // cannot happen on a BADI
+ }
} else {
freq += 1 + (freqDelta >>> 1);
norm++;
}
- maxScore = Math.max(maxScore, scorer.score(freq, norm));
+ Impact impact = reuse.impacts[length];
+ impact.freq = freq;
+ impact.norm = norm;
+ length++;
+ }
+ reuse.length = length;
+ return reuse;
+ }
+
+ static class MutableImpactList extends AbstractList<Impact> implements RandomAccess {
+ int length = 1;
+ Impact[] impacts = new Impact[] { new Impact(Integer.MAX_VALUE, 1L) };
+
+ @Override
+ public Impact get(int index) {
+ return impacts[index];
+ }
+
+ @Override
+ public int size() {
+ return length;
}
- return maxScore;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java
index b92cd42..ebddde0 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java
@@ -200,8 +200,7 @@ class Lucene50SkipReader extends MultiLevelSkipListReader {
return delta;
}
- // The default impl skips impacts since they are only useful if we have a SimScorer
- // to compute the scores that impacts map to.
+ // The default impl skips impacts
protected void readImpacts(int level, IndexInput skipStream) throws IOException {
if (version >= Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) {
// The base implementation skips impacts, they are not used
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java
index cc94ed0..37044b5 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java
@@ -22,9 +22,9 @@ import java.util.Arrays;
import java.util.Set;
import java.util.SortedSet;
-import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator;
-import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator.FreqAndNorm;
+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.MultiLevelSkipListWriter;
+import org.apache.lucene.index.Impact;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMOutputStream;
@@ -65,7 +65,7 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter {
private long curPayPointer;
private int curPosBufferUpto;
private int curPayloadByteUpto;
- private CompetitiveFreqNormAccumulator[] curCompetitiveFreqNorms;
+ private CompetitiveImpactAccumulator[] curCompetitiveFreqNorms;
private boolean fieldHasPositions;
private boolean fieldHasOffsets;
private boolean fieldHasPayloads;
@@ -85,9 +85,9 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter {
}
lastPayloadByteUpto = new int[maxSkipLevels];
}
- curCompetitiveFreqNorms = new CompetitiveFreqNormAccumulator[maxSkipLevels];
+ curCompetitiveFreqNorms = new CompetitiveImpactAccumulator[maxSkipLevels];
for (int i = 0; i < maxSkipLevels; ++i) {
- curCompetitiveFreqNorms[i] = new CompetitiveFreqNormAccumulator();
+ curCompetitiveFreqNorms[i] = new CompetitiveImpactAccumulator();
}
}
@@ -116,7 +116,7 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter {
}
}
if (initialized) {
- for (CompetitiveFreqNormAccumulator acc : curCompetitiveFreqNorms) {
+ for (CompetitiveImpactAccumulator acc : curCompetitiveFreqNorms) {
acc.clear();
}
}
@@ -139,7 +139,7 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter {
}
// sets of competitive freq,norm pairs should be empty at this point
assert Arrays.stream(curCompetitiveFreqNorms)
- .map(CompetitiveFreqNormAccumulator::getCompetitiveFreqNormPairs)
+ .map(CompetitiveImpactAccumulator::getCompetitiveFreqNormPairs)
.mapToInt(Set::size)
.sum() == 0;
initialized = true;
@@ -149,7 +149,7 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter {
/**
* Sets the values for the current skip data.
*/
- public void bufferSkip(int doc, CompetitiveFreqNormAccumulator competitiveFreqNorms,
+ public void bufferSkip(int doc, CompetitiveImpactAccumulator competitiveFreqNorms,
int numDocs, long posFP, long payFP, int posBufferUpto, int payloadByteUpto) throws IOException {
initSkip();
this.curDoc = doc;
@@ -191,7 +191,7 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter {
}
}
- CompetitiveFreqNormAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level];
+ CompetitiveImpactAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level];
assert competitiveFreqNorms.getCompetitiveFreqNormPairs().size() > 0;
if (level + 1 < numberOfSkipLevels) {
curCompetitiveFreqNorms[level + 1].addAll(competitiveFreqNorms);
@@ -203,14 +203,14 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter {
competitiveFreqNorms.clear();
}
- static void writeImpacts(CompetitiveFreqNormAccumulator acc, IndexOutput out) throws IOException {
- SortedSet<FreqAndNorm> freqAndNorms = acc.getCompetitiveFreqNormPairs();
- FreqAndNorm previous = new FreqAndNorm(0, 0);
- for (FreqAndNorm freqAndNorm : freqAndNorms) {
- assert freqAndNorm.freq > previous.freq;
- assert Long.compareUnsigned(freqAndNorm.norm, previous.norm) > 0;
- int freqDelta = freqAndNorm.freq - previous.freq - 1;
- long normDelta = freqAndNorm.norm - previous.norm - 1;
+ static void writeImpacts(CompetitiveImpactAccumulator acc, IndexOutput out) throws IOException {
+ SortedSet<Impact> impacts = acc.getCompetitiveFreqNormPairs();
+ Impact previous = new Impact(0, 0);
+ for (Impact impact : impacts) {
+ assert impact.freq > previous.freq;
+ assert Long.compareUnsigned(impact.norm, previous.norm) > 0;
+ int freqDelta = impact.freq - previous.freq - 1;
+ long normDelta = impact.norm - previous.norm - 1;
if (normDelta == 0) {
// most of time, norm only increases by 1, so we can fold everything in a single byte
out.writeVInt(freqDelta << 1);
@@ -218,7 +218,7 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter {
out.writeVInt((freqDelta << 1) | 1);
out.writeZLong(normDelta);
}
- previous = freqAndNorm;
+ previous = impact;
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70DocValuesProducer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70DocValuesProducer.java
index 7bea274..b0f6e84 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70DocValuesProducer.java
@@ -38,7 +38,6 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
@@ -1160,7 +1159,7 @@ final class Lucene70DocValuesProducer extends DocValuesProducer implements Close
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java
index 841b2ad..2b38712 100644
--- a/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/document/FeatureQuery.java
@@ -30,6 +30,7 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MaxScoreCache;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
@@ -114,7 +115,8 @@ final class FeatureQuery extends Query {
}
SimScorer scorer = function.scorer(fieldName, boost);
- ImpactsEnum impacts = termsEnum.impacts(scorer, PostingsEnum.FREQS);
+ ImpactsEnum impacts = termsEnum.impacts(PostingsEnum.FREQS);
+ MaxScoreCache maxScoreCache = new MaxScoreCache(impacts, scorer);
return new Scorer(this) {
@@ -135,12 +137,12 @@ final class FeatureQuery extends Query {
@Override
public int advanceShallow(int target) throws IOException {
- return impacts.advanceShallow(target);
+ return maxScoreCache.advanceShallow(target);
}
@Override
public float getMaxScore(int upTo) throws IOException {
- return impacts.getMaxScore(upTo);
+ return maxScoreCache.getMaxScore(upTo);
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index 9126b1d..ca27066 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Deque;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
@@ -48,7 +49,6 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -1602,58 +1602,45 @@ public final class CheckIndex implements Closeable {
// Checking score blocks is heavy, we only do it on long postings lists, on every 1024th term
// or if slow checks are enabled.
if (doSlowChecks || docFreq > 1024 || (status.termCount + status.delTermCount) % 1024 == 0) {
- // Test score blocks
- // We only score on freq to keep things simple and not pull norms
- SimScorer scorer = new SimScorer(field) {
- @Override
- public float score(float freq, long norm) {
- return freq;
- }
- };
-
// First check max scores and block uptos
// But only if slok checks are enabled since we visit all docs
if (doSlowChecks) {
int max = -1;
- float maxScore = 0;
- ImpactsEnum impacts = termsEnum.impacts(scorer, PostingsEnum.FREQS);
+ int maxFreq = 0;
+ ImpactsEnum impactsEnum = termsEnum.impacts(PostingsEnum.FREQS);
postings = termsEnum.postings(postings, PostingsEnum.FREQS);
- for (int doc = impacts.nextDoc(); ; doc = impacts.nextDoc()) {
+ for (int doc = impactsEnum.nextDoc(); ; doc = impactsEnum.nextDoc()) {
if (postings.nextDoc() != doc) {
throw new RuntimeException("Wrong next doc: " + doc + ", expected " + postings.docID());
}
if (doc == DocIdSetIterator.NO_MORE_DOCS) {
break;
}
- if (postings.freq() != impacts.freq()) {
- throw new RuntimeException("Wrong freq, expected " + postings.freq() + ", but got " + impacts.freq());
+ if (postings.freq() != impactsEnum.freq()) {
+ throw new RuntimeException("Wrong freq, expected " + postings.freq() + ", but got " + impactsEnum.freq());
}
if (doc > max) {
- max = impacts.advanceShallow(doc);
- if (max < doc) {
- throw new RuntimeException("max block doc id " + max + " must be greater than the target: " + doc);
- }
- maxScore = impacts.getMaxScore(max);
- }
- int max2 = impacts.advanceShallow(doc);
- if (max != max2) {
- throw new RuntimeException("max is not stable, initially had " + max + " but now " + max2);
+ impactsEnum.advanceShallow(doc);
+ Impacts impacts = impactsEnum.getImpacts();
+ checkImpacts(impacts, doc);
+ max = impacts.getDocIdUpTo(0);
+ List<Impact> impacts0 = impacts.getImpacts(0);
+ maxFreq = impacts0.get(impacts0.size() - 1).freq;
}
- float score = scorer.score(impacts.freq(), 1);
- if (score > maxScore) {
- throw new RuntimeException("score " + score + " is greater than the max score " + maxScore);
+ if (impactsEnum.freq() > maxFreq) {
+ throw new RuntimeException("freq " + impactsEnum.freq() + " is greater than the max freq according to impacts " + maxFreq);
}
}
}
// Now check advancing
- ImpactsEnum impacts = termsEnum.impacts(scorer, PostingsEnum.FREQS);
+ ImpactsEnum impactsEnum = termsEnum.impacts(PostingsEnum.FREQS);
postings = termsEnum.postings(postings, PostingsEnum.FREQS);
int max = -1;
- float maxScore = 0;
+ int maxFreq = 0;
while (true) {
- int doc = impacts.docID();
+ int doc = impactsEnum.docID();
boolean advance;
int target;
if (((field.hashCode() + doc) & 1) == 1) {
@@ -1662,23 +1649,29 @@ public final class CheckIndex implements Closeable {
} else {
advance = true;
int delta = Math.min(1 + ((31 * field.hashCode() + doc) & 0x1ff), DocIdSetIterator.NO_MORE_DOCS - doc);
- target = impacts.docID() + delta;
+ target = impactsEnum.docID() + delta;
}
if (target > max && target % 2 == 1) {
int delta = Math.min((31 * field.hashCode() + target) & 0x1ff, DocIdSetIterator.NO_MORE_DOCS - target);
max = target + delta;
- int m = impacts.advanceShallow(target);
- if (m < target) {
- throw new RuntimeException("Block max doc: " + m + " is less than the target " + target);
+ impactsEnum.advanceShallow(target);
+ Impacts impacts = impactsEnum.getImpacts();
+ checkImpacts(impacts, doc);
+ maxFreq = Integer.MAX_VALUE;
+ for (int level = 0; level < impacts.numLevels(); ++level) {
+ if (impacts.getDocIdUpTo(level) >= max) {
+ List<Impact> perLevelImpacts = impacts.getImpacts(level);
+ maxFreq = perLevelImpacts.get(perLevelImpacts.size() - 1).freq;
+ break;
+ }
}
- maxScore = impacts.getMaxScore(max);
}
if (advance) {
- doc = impacts.advance(target);
+ doc = impactsEnum.advance(target);
} else {
- doc = impacts.nextDoc();
+ doc = impactsEnum.nextDoc();
}
if (postings.advance(target) != doc) {
@@ -1687,23 +1680,28 @@ public final class CheckIndex implements Closeable {
if (doc == DocIdSetIterator.NO_MORE_DOCS) {
break;
}
- if (postings.freq() != impacts.freq()) {
- throw new RuntimeException("Wrong freq, expected " + postings.freq() + ", but got " + impacts.freq());
+ if (postings.freq() != impactsEnum.freq()) {
+ throw new RuntimeException("Wrong freq, expected " + postings.freq() + ", but got " + impactsEnum.freq());
}
if (doc >= max) {
int delta = Math.min((31 * field.hashCode() + target & 0x1ff), DocIdSetIterator.NO_MORE_DOCS - doc);
max = doc + delta;
- int m = impacts.advanceShallow(doc);
- if (m < doc) {
- throw new RuntimeException("Block max doc: " + m + " is less than the target " + doc);
+ impactsEnum.advanceShallow(doc);
+ Impacts impacts = impactsEnum.getImpacts();
+ checkImpacts(impacts, doc);
+ maxFreq = Integer.MAX_VALUE;
+ for (int level = 0; level < impacts.numLevels(); ++level) {
+ if (impacts.getDocIdUpTo(level) >= max) {
+ List<Impact> perLevelImpacts = impacts.getImpacts(level);
+ maxFreq = perLevelImpacts.get(perLevelImpacts.size() - 1).freq;
+ break;
+ }
}
- maxScore = impacts.getMaxScore(max);
}
- float score = scorer.score(impacts.freq(), 1);
- if (score > maxScore) {
- throw new RuntimeException("score " + score + " is greater than the max score " + maxScore);
+ if (impactsEnum.freq() > maxFreq) {
+ throw new RuntimeException("Term frequency " + impactsEnum.freq() + " is greater than the max freq according to impacts " + maxFreq);
}
}
}
@@ -1850,6 +1848,68 @@ public final class CheckIndex implements Closeable {
return status;
}
+ static void checkImpacts(Impacts impacts, int lastTarget) {
+ final int numLevels = impacts.numLevels();
+ if (numLevels < 1) {
+ throw new RuntimeException("The number of levels must be >= 1, got " + numLevels);
+ }
+
+ int docIdUpTo0 = impacts.getDocIdUpTo(0);
+ if (docIdUpTo0 < lastTarget) {
+ throw new RuntimeException("getDocIdUpTo returned " + docIdUpTo0 + " on level 0, which is less than the target " + lastTarget);
+ }
+
+ for (int level = 1; level < numLevels; ++level) {
+ int docIdUpTo = impacts.getDocIdUpTo(level);
+ int previousDocIdUpTo = impacts.getDocIdUpTo(level - 1);
+ if (docIdUpTo < previousDocIdUpTo) {
+ throw new RuntimeException("Decreasing return for getDocIdUpTo: level " + (level-1) + " returned " + previousDocIdUpTo
+ + " but level " + level + " returned " + docIdUpTo + " for target " + lastTarget);
+ }
+ }
+
+ for (int level = 0; level < numLevels; ++level) {
+ List<Impact> perLevelImpacts = impacts.getImpacts(level);
+ if (perLevelImpacts.isEmpty()) {
+ throw new RuntimeException("Got empty list of impacts on level " + level);
+ }
+ Impact first = perLevelImpacts.get(0);
+ if (first.freq < 1) {
+ throw new RuntimeException("First impact had a freq <= 0: " + first);
+ }
+ if (first.norm == 0) {
+ throw new RuntimeException("First impact had a norm == 0: " + first);
+ }
+ // Impacts must be in increasing order of norm AND freq
+ Impact previous = first;
+ for (int i = 1; i < perLevelImpacts.size(); ++i) {
+ Impact impact = perLevelImpacts.get(i);
+ if (impact.freq <= previous.freq || Long.compareUnsigned(impact.norm, previous.norm) <= 0) {
+ throw new RuntimeException("Impacts are not ordered or contain dups, got " + previous + " then " + impact);
+ }
+ }
+ if (level > 0) {
+ // Make sure that impacts at level N trigger better scores than an level N-1
+ Iterator<Impact> previousIt = impacts.getImpacts(level-1).iterator();
+ previous = previousIt.next();
+ Iterator<Impact> it = perLevelImpacts.iterator();
+ Impact impact = it.next();
+ while (previousIt.hasNext()) {
+ previous = previousIt.next();
+ if (previous.freq <= impact.freq && Long.compareUnsigned(previous.norm, impact.norm) >= 0) {
+ // previous triggers a lower score than the current impact, all good
+ continue;
+ }
+ if (it.hasNext() == false) {
+ throw new RuntimeException("Found impact " + previous + " on level " + (level-1) + " but no impact on level "
+ + level + " triggers a better score: " + perLevelImpacts);
+ }
+ impact = it.next();
+ }
+ }
+ }
+ }
+
/**
* Test the term index.
* @lucene.experimental
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
index 4a9b660..3d86bee 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Iterator;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -216,8 +215,8 @@ public abstract class FilterLeafReader extends LeafReader {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return in.impacts(scorer, flags);
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return in.impacts(flags);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java
index 61392c3a..72ca421 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.AttributeSource;
/**
@@ -184,8 +183,8 @@ public abstract class FilteredTermsEnum extends TermsEnum {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return tenum.impacts(scorer, flags);
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return tenum.impacts(flags);
}
/** This enum does not support seeking!
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java b/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
index c3e7d71..f902022 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
@@ -24,7 +24,6 @@ import java.util.List;
import java.util.Map;
import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -275,7 +274,7 @@ class FreqProxFields extends Fields {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/Impact.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/Impact.java b/lucene/core/src/java/org/apache/lucene/index/Impact.java
new file mode 100644
index 0000000..b8ee20e
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/Impact.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+/**
+ * Per-document scoring factors.
+ */
+public final class Impact {
+
+ /**
+ * Term frequency of the term in the document.
+ */
+ public int freq;
+
+ /**
+ * Norm factor of the document.
+ */
+ public long norm;
+
+ /**
+ * Constructor.
+ */
+ public Impact(int freq, long norm) {
+ this.freq = freq;
+ this.norm = norm;
+ }
+
+ @Override
+ public String toString() {
+ return "{freq=" + freq + ",norm=" + norm + "}";
+ }
+
+ @Override
+ public int hashCode() {
+ int h = freq;
+ h = 31 * h + Long.hashCode(norm);
+ return h;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || getClass() != obj.getClass()) return false;
+ Impact other = (Impact) obj;
+ return freq == other.freq && norm == other.norm;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/Impacts.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/Impacts.java b/lucene/core/src/java/org/apache/lucene/index/Impacts.java
new file mode 100644
index 0000000..ce3ece4
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/Impacts.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.util.List;
+
+/**
+ * Information about upcoming impacts, ie. (freq, norm) pairs.
+ */
+public abstract class Impacts {
+
+ /**
+ * Return the number of levels on which we have impacts.
+ * The returned value is always greater than 0 and may not always be the
+ * same, even on a single postings list, depending on the current doc ID.
+ */
+ public abstract int numLevels();
+
+ /**
+ * Return the maximum inclusive doc ID until which the list of impacts
+ * returned by {@link #getImpacts(int)} is valid. This is a non-decreasing
+ * function of {@code level}.
+ */
+ public abstract int getDocIdUpTo(int level);
+
+ /**
+ * Return impacts on the given level. These impacts are sorted by increasing
+ * frequency and increasing unsigned norm, and only valid until the doc ID
+ * returned by {@link #getDocIdUpTo(int)} for the same level, included.
+ * The returned list is never empty.
+ * NOTE: There is no guarantee that these impacts actually appear in postings,
+ * only that they trigger scores that are greater than or equal to the impacts
+ * that actually appear in postings.
+ */
+ public abstract List<Impact> getImpacts(int level);
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/ImpactsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ImpactsEnum.java b/lucene/core/src/java/org/apache/lucene/index/ImpactsEnum.java
index 8deccff..5a35477 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ImpactsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ImpactsEnum.java
@@ -18,11 +18,9 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.search.DocIdSetIterator;
-
/**
- * Extension of {@link PostingsEnum} which also provides information about the
- * produced scores.
+ * Extension of {@link PostingsEnum} which also provides information about
+ * upcoming impacts.
* @lucene.experimental
*/
public abstract class ImpactsEnum extends PostingsEnum {
@@ -31,23 +29,28 @@ public abstract class ImpactsEnum extends PostingsEnum {
protected ImpactsEnum() {}
/**
- * Advance to the block of documents that contains {@code target} in order to
- * get scoring information about this block. This method is implicitly called
- * by {@link DocIdSetIterator#advance(int)} and
- * {@link DocIdSetIterator#nextDoc()}. Calling this method doesn't modify the
- * current {@link DocIdSetIterator#docID()}.
- * It returns a number that is greater than or equal to all documents
- * contained in the current block, but less than any doc IDS of the next block.
- * {@code target} must be >= {@link #docID()} as well as all targets that
- * have been passed to {@link #advanceShallow(int)} so far.
+ * Shallow-advance to {@code target}. This is cheaper than calling
+ * {@link #advance(int)} and allows further calls to {@link #getImpacts()}
+ * to ignore doc IDs that are less than {@code target} in order to get more
+ * precise information about impacts.
+ * This method may not be called on targets that are less than the current
+ * {@link #docID()}.
+ * After this method has been called, {@link #nextDoc()} may not be called
+ * if the current doc ID is less than {@code target - 1} and
+ * {@link #advance(int)} may not be called on targets that are less than
+ * {@code target}.
*/
- public abstract int advanceShallow(int target) throws IOException;
+ public abstract void advanceShallow(int target) throws IOException;
/**
- * Return the maximum score that documents between the last {@code target}
- * that this iterator was {@link #advanceShallow(int) shallow-advanced} to
- * included and {@code upTo} included.
+ * Get information about upcoming impacts for doc ids that are greater than
+ * or equal to the maximum of {@link #docID()} and the last target that was
+ * passed to {@link #advanceShallow(int)}.
+ * This method may not be called on an unpositioned iterator on which
+ * {@link #advanceShallow(int)} has never been called.
+ * NOTE: advancing this iterator may invalidate the returned impacts, so they
+ * should not be used after the iterator has been advanced.
*/
- public abstract float getMaxScore(int upTo) throws IOException;
+ public abstract Impacts getImpacts() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java
index 4aad5c8..d20c6c1 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -369,9 +368,9 @@ public final class MultiTermsEnum extends TermsEnum {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
// implemented to not fail CheckIndex, but you shouldn't be using impacts on a slow reader
- return new SlowImpactsEnum(postings(null, flags), scorer.score(Float.MAX_VALUE, 1));
+ return new SlowImpactsEnum(postings(null, flags));
}
final static class TermsEnumWithSlice {
[29/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12288: Add more MDC
logging information to core admin requests
Posted by ab...@apache.org.
SOLR-12288: Add more MDC logging information to core admin requests
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8b9c2a31
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8b9c2a31
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8b9c2a31
Branch: refs/heads/jira/solr-11779
Commit: 8b9c2a3185d824a9aaae5c993b872205358729dd
Parents: ed948ef
Author: Varun Thacker <va...@apache.org>
Authored: Wed May 2 19:43:14 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed May 2 23:36:50 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../java/org/apache/solr/handler/admin/CoreAdminHandler.java | 6 ++++++
.../src/java/org/apache/solr/logging/MDCLoggingContext.java | 2 +-
3 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8b9c2a31/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b2235fd..b925fe1 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -276,6 +276,8 @@ Other Changes
* SOLR-12289: Add more MDC logging information to collection admin requests (Varun Thacker)
+* SOLR-12288: Add more MDC logging information to core admin requests (Varun Thacker)
+
================== 7.3.1 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8b9c2a31/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
index 8932c21..66dc39e 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
@@ -44,6 +44,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -170,6 +171,11 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
}
final CallInfo callInfo = new CallInfo(this, req, rsp, op);
+ String coreName = req.getParams().get(CoreAdminParams.CORE);
+ if (coreName == null) {
+ coreName = req.getParams().get(CoreAdminParams.NAME);
+ }
+ MDCLoggingContext.setCoreName(coreName);
if (taskId == null) {
callInfo.call();
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8b9c2a31/solr/core/src/java/org/apache/solr/logging/MDCLoggingContext.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/logging/MDCLoggingContext.java b/solr/core/src/java/org/apache/solr/logging/MDCLoggingContext.java
index 1f5324b..2276a9e6 100644
--- a/solr/core/src/java/org/apache/solr/logging/MDCLoggingContext.java
+++ b/solr/core/src/java/org/apache/solr/logging/MDCLoggingContext.java
@@ -69,7 +69,7 @@ public class MDCLoggingContext {
}
}
- private static void setCoreName(String core) {
+ public static void setCoreName(String core) {
if (core != null) {
MDC.put(CORE_NAME_PROP, "x:" + core);
} else {
[38/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12028: BadApple and
AwaitsFix annotations usage
Posted by ab...@apache.org.
SOLR-12028: BadApple and AwaitsFix annotations usage
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/89fc02a3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/89fc02a3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/89fc02a3
Branch: refs/heads/jira/solr-11779
Commit: 89fc02a3b04aedf2803a46782b3a8b8e9b5c9a7b
Parents: 3a2572d
Author: Erick Erickson <er...@apache.org>
Authored: Fri May 4 22:30:18 2018 -0700
Committer: Erick Erickson <er...@apache.org>
Committed: Fri May 4 22:30:18 2018 -0700
----------------------------------------------------------------------
.../src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java | 1 +
.../src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java | 3 +++
.../test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java | 1 +
solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java | 1 +
.../core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java | 1 +
.../solr/cloud/api/collections/TestHdfsCloudBackupRestore.java | 2 ++
.../src/test/org/apache/solr/handler/TestReplicationHandler.java | 1 +
.../core/src/test/org/apache/solr/schema/TestCloudSchemaless.java | 1 +
solr/core/src/test/org/apache/solr/search/TestStressRecovery.java | 1 +
.../apache/solr/client/solrj/io/stream/MathExpressionTest.java | 1 +
10 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89fc02a3/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java b/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
index 13ecea2..bda5bce 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
@@ -312,6 +312,7 @@ public class DocValuesNotIndexedTest extends SolrCloudTestCase {
// Verify that we actually form groups that are "expected". Most of the processing takes some care to
// make sure all the values for each field are unique. We need to have docs that have values that are _not_
// unique.
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void testGroupingDVOnly() throws IOException, SolrServerException {
List<SolrInputDocument> docs = new ArrayList<>(50);
for (int idx = 0; idx < 49; ++idx) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89fc02a3/solr/core/src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java b/solr/core/src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java
index 713a6ba..0d0d0f0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java
@@ -162,6 +162,7 @@ public class LIRRollingUpdatesTest extends SolrCloudTestCase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void testNewLeaderOldReplica() throws Exception {
// in case of new leader & old replica, new leader can still put old replica into LIR
@@ -330,11 +331,13 @@ public class LIRRollingUpdatesTest extends SolrCloudTestCase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void testNewLeaderAndMixedReplicas() throws Exception {
testLeaderAndMixedReplicas(false);
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void testOldLeaderAndMixedReplicas() throws Exception {
testLeaderAndMixedReplicas(true);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89fc02a3/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
index d15ecea..aeab472 100644
--- a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
@@ -60,6 +60,7 @@ public class LeaderElectionIntegrationTest extends SolrCloudTestCase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void testSimpleSliceLeaderElection() throws Exception {
String collection = "collection1";
createCollection(collection);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89fc02a3/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
index edc6695..6b5884c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
@@ -85,6 +85,7 @@ public class OverseerRolesTest extends SolrCloudTestCase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void testOverseerRole() throws Exception {
logOverseerState();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89fc02a3/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java
index f8b11d1..9a7ca69 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java
@@ -68,6 +68,7 @@ public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void testLoadDocsIntoGettingStartedCollection() throws Exception {
waitForThingsToLevelOut(30000);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89fc02a3/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
index 58ac17d..85ed8a8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
@@ -60,6 +61,7 @@ import static org.apache.solr.core.backup.BackupManager.ZK_STATE_DIR;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
+@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCase {
public static final String SOLR_XML = "<solr>\n" +
"\n" +
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89fc02a3/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
index e22f4f7..809d513 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
@@ -1305,6 +1305,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void doTestIndexAndConfigAliasReplication() throws Exception {
clearIndexWithReplication();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89fc02a3/solr/core/src/test/org/apache/solr/schema/TestCloudSchemaless.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/TestCloudSchemaless.java b/solr/core/src/test/org/apache/solr/schema/TestCloudSchemaless.java
index 4524495..06d1270 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestCloudSchemaless.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestCloudSchemaless.java
@@ -88,6 +88,7 @@ public class TestCloudSchemaless extends AbstractFullDistribZkTestBase {
@Test
@ShardsFixed(num = 8)
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void test() throws Exception {
setupRestTestHarnesses();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89fc02a3/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java b/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java
index aaac791..79648ae 100644
--- a/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java
+++ b/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java
@@ -59,6 +59,7 @@ public class TestStressRecovery extends TestRTGBase {
// This version simulates updates coming from the leader and sometimes being reordered
// and tests the ability to buffer updates and apply them later
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void testStressRecovery() throws Exception {
assumeFalse("FIXME: This test is horribly slow sometimes on Windows!", Constants.WINDOWS);
clearIndex();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89fc02a3/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 94d5334..f936fee 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -3553,6 +3553,7 @@ public class MathExpressionTest extends SolrCloudTestCase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
public void testGammaDistribution() throws Exception {
String cexpr = "#comment\nlet(echo=true, " +
"a=describe(sample(gammaDistribution(1, 10),10000)), " +
[11/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8282: Reduce
boxing and unnecessary object creation in DV updates
Posted by ab...@apache.org.
LUCENE-8282: Reduce boxing and unnecessary object creation in DV updates
DV updates used the boxed type Long to keep API generic. Yet, the missing
type caused a lot of code duplication, boxing and unnecessary object creation.
This change cuts over to type safe APIs using BytesRef and long (the primitive)
In this change most of the code that is almost identical between binary and numeric
is not shared reducing the maintenance overhead and likelihood of introducing bugs.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b43b0919
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b43b0919
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b43b0919
Branch: refs/heads/jira/solr-11779
Commit: b43b09190d52a959b8d3b10fcadfabfa58691955
Parents: 570fff8
Author: Simon Willnauer <si...@apache.org>
Authored: Mon Apr 30 11:41:56 2018 +0200
Committer: GitHub <no...@github.com>
Committed: Mon Apr 30 11:41:56 2018 +0200
----------------------------------------------------------------------
.../index/BinaryDocValuesFieldUpdates.java | 32 ++--
.../apache/lucene/index/BufferedUpdates.java | 62 ++++----
.../lucene/index/DocValuesFieldUpdates.java | 65 ++++----
.../apache/lucene/index/DocValuesUpdate.java | 92 ++++++++++--
.../index/DocumentsWriterDeleteQueue.java | 7 +-
.../lucene/index/FrozenBufferedUpdates.java | 147 +++++++------------
.../org/apache/lucene/index/IndexWriter.java | 2 +-
.../index/NumericDocValuesFieldUpdates.java | 44 ++++--
.../apache/lucene/index/ReadersAndUpdates.java | 7 +-
.../lucene/index/TestDocValuesFieldUpdates.java | 72 +++++++++
.../lucene/index/TestPendingSoftDeletes.java | 27 +++-
11 files changed, 342 insertions(+), 215 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
index 3faad49..db9aa09 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
@@ -33,7 +33,7 @@ import org.apache.lucene.util.packed.PagedMutable;
*
* @lucene.experimental
*/
-class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
+final class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
final static class Iterator extends DocValuesFieldUpdates.Iterator {
private final int size;
@@ -55,14 +55,14 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
value = values.clone();
this.delGen = delGen;
}
-
+
@Override
- BytesRef value() {
+ BytesRef binaryValue() {
value.offset = offset;
value.length = length;
return value;
}
-
+
@Override
public int nextDoc() {
if (idx >= size) {
@@ -94,6 +94,11 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
long delGen() {
return delGen;
}
+
+ @Override
+ long longValue() {
+ throw new UnsupportedOperationException();
+ }
}
private PagedMutable docs;
@@ -116,9 +121,18 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
return size;
}
- // NOTE: we fully consume the incoming BytesRef so caller is free to reuse it after we return:
@Override
- synchronized public void add(int doc, Object value) {
+ public void add(int doc, long value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void add(int docId, DocValuesFieldUpdates.Iterator iterator) {
+ add(docId, iterator.binaryValue());
+ }
+
+ @Override
+ synchronized public void add(int doc, BytesRef value) {
if (finished) {
throw new IllegalStateException("already finished");
}
@@ -130,8 +144,6 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
throw new IllegalStateException("cannot support more than Integer.MAX_VALUE doc/value entries");
}
- BytesRef val = (BytesRef) value;
-
// grow the structures to have room for more elements
if (docs.size() == size) {
docs = docs.grow(size + 1);
@@ -141,8 +153,8 @@ class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
docs.set(size, doc);
offsets.set(size, values.length());
- lengths.set(size, val.length);
- values.append(val);
+ lengths.set(size, value.length);
+ values.append(value);
++size;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java b/lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java
index a5a86e6..ae37f14 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.IntFunction;
import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
@@ -233,62 +234,49 @@ class BufferedUpdates {
}
}
- public void addNumericUpdate(NumericDocValuesUpdate update, int docIDUpto) {
- LinkedHashMap<Term,NumericDocValuesUpdate> fieldUpdates = numericUpdates.get(update.field);
- if (fieldUpdates == null) {
- fieldUpdates = new LinkedHashMap<>();
- numericUpdates.put(update.field, fieldUpdates);
- bytesUsed.addAndGet(BYTES_PER_NUMERIC_FIELD_ENTRY);
- }
- final NumericDocValuesUpdate current = fieldUpdates.get(update.term);
- if (current != null && docIDUpto < current.docIDUpto) {
- // Only record the new number if it's greater than or equal to the current
- // one. This is important because if multiple threads are replacing the
- // same doc at nearly the same time, it's possible that one thread that
- // got a higher docID is scheduled before the other threads.
- return;
- }
-
- update.docIDUpto = docIDUpto;
- // since it's a LinkedHashMap, we must first remove the Term entry so that
- // it's added last (we're interested in insertion-order).
- if (current != null) {
- fieldUpdates.remove(update.term);
- }
- fieldUpdates.put(update.term, update);
- numNumericUpdates.incrementAndGet();
- if (current == null) {
- bytesUsed.addAndGet(BYTES_PER_NUMERIC_UPDATE_ENTRY + update.sizeInBytes());
+ void addNumericUpdate(NumericDocValuesUpdate update, int docIDUpto) {
+ if (addDocValuesUpdate(numericUpdates, update, docIDUpto, update::prepareForApply, BYTES_PER_NUMERIC_UPDATE_ENTRY,
+ BYTES_PER_NUMERIC_FIELD_ENTRY)) {
+ numNumericUpdates.incrementAndGet();
}
}
- public void addBinaryUpdate(BinaryDocValuesUpdate update, int docIDUpto) {
- LinkedHashMap<Term,BinaryDocValuesUpdate> fieldUpdates = binaryUpdates.get(update.field);
+ void addBinaryUpdate(BinaryDocValuesUpdate update, int docIDUpto) {
+ if (addDocValuesUpdate(binaryUpdates, update, docIDUpto, update::prepareForApply, BYTES_PER_BINARY_UPDATE_ENTRY,
+ BYTES_PER_BINARY_FIELD_ENTRY)) {
+ numBinaryUpdates.incrementAndGet();
+ }
+ }
+
+ private <T extends DocValuesUpdate> boolean addDocValuesUpdate(Map<String,LinkedHashMap<Term,T>> updates, T update,
+ int docIDUpto, IntFunction<T> prepareForApply,
+ long bytesPerUpdateEntry, long bytesPerFieldEntry) {
+ LinkedHashMap<Term,T> fieldUpdates = updates.get(update.field);
if (fieldUpdates == null) {
fieldUpdates = new LinkedHashMap<>();
- binaryUpdates.put(update.field, fieldUpdates);
- bytesUsed.addAndGet(BYTES_PER_BINARY_FIELD_ENTRY);
+ updates.put(update.field, fieldUpdates);
+ bytesUsed.addAndGet(bytesPerFieldEntry);
}
- final BinaryDocValuesUpdate current = fieldUpdates.get(update.term);
+ final T current = fieldUpdates.get(update.term);
if (current != null && docIDUpto < current.docIDUpto) {
// Only record the new number if it's greater than or equal to the current
// one. This is important because if multiple threads are replacing the
// same doc at nearly the same time, it's possible that one thread that
// got a higher docID is scheduled before the other threads.
- return;
+ return false;
}
-
- update.docIDUpto = docIDUpto;
+
// since it's a LinkedHashMap, we must first remove the Term entry so that
// it's added last (we're interested in insertion-order).
if (current != null) {
fieldUpdates.remove(update.term);
}
- fieldUpdates.put(update.term, update);
- numBinaryUpdates.incrementAndGet();
+
+ fieldUpdates.put(update.term, prepareForApply.apply(docIDUpto)); // only make a copy if necessary
if (current == null) {
- bytesUsed.addAndGet(BYTES_PER_BINARY_UPDATE_ENTRY + update.sizeInBytes());
+ bytesUsed.addAndGet(bytesPerUpdateEntry + update.sizeInBytes());
}
+ return true;
}
void clearDeleteTerms() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
index a711f79..4a7f5b8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.index;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PriorityQueue;
@@ -26,7 +27,7 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
*
* @lucene.experimental
*/
-abstract class DocValuesFieldUpdates {
+abstract class DocValuesFieldUpdates implements Accountable {
protected static final int PAGE_SIZE = 1024;
@@ -51,13 +52,18 @@ abstract class DocValuesFieldUpdates {
throw new UnsupportedOperationException();
}
+ @Override
public abstract int nextDoc(); // no IOException
/**
- * Returns the value of the document returned from {@link #nextDoc()}. A
- * {@code null} value means that it was unset for this document.
+ * Returns a long value for the current document if this iterator is a long iterator.
+ */
+ abstract long longValue();
+
+ /**
+ * Returns a binary value for the current document if this iterator is a binary value iterator.
*/
- abstract Object value();
+ abstract BytesRef binaryValue();
/** Returns delGen for this packet. */
abstract long delGen();
@@ -73,7 +79,7 @@ abstract class DocValuesFieldUpdates {
}
@Override
public BytesRef binaryValue() {
- return (BytesRef) iterator.value();
+ return iterator.binaryValue();
}
@Override
public boolean advanceExact(int target) {
@@ -100,7 +106,7 @@ abstract class DocValuesFieldUpdates {
return new NumericDocValues() {
@Override
public long longValue() {
- return ((Long)iterator.value()).longValue();
+ return iterator.longValue();
}
@Override
public boolean advanceExact(int target) {
@@ -163,14 +169,9 @@ abstract class DocValuesFieldUpdates {
}
return new Iterator() {
- private int doc;
-
- private boolean first = true;
-
+ private int doc = -1;
@Override
public int nextDoc() {
- // TODO: can we do away with this first boolean?
- if (first == false) {
// Advance all sub iterators past current doc
while (true) {
if (queue.size() == 0) {
@@ -189,21 +190,22 @@ abstract class DocValuesFieldUpdates {
queue.updateTop();
}
}
- } else {
- doc = queue.top().docID();
- first = false;
- }
return doc;
}
-
+
@Override
public int docID() {
return doc;
}
@Override
- public Object value() {
- return queue.top().value();
+ long longValue() {
+ return queue.top().longValue();
+ }
+
+ @Override
+ BytesRef binaryValue() {
+ return queue.top().binaryValue();
}
@Override
@@ -229,31 +231,34 @@ abstract class DocValuesFieldUpdates {
this.type = type;
}
- public boolean getFinished() {
+ boolean getFinished() {
return finished;
}
+ abstract void add(int doc, long value);
+
+ abstract void add(int doc, BytesRef value);
+
/**
- * Add an update to a document. For unsetting a value you should pass
- * {@code null}.
+ * Adds the value for the given docID.
+ * This method prevents conditional calls to {@link Iterator#longValue()} or {@link Iterator#binaryValue()}
+ * since the implementation knows if it's a long value iterator or binary value
*/
- public abstract void add(int doc, Object value);
-
+ abstract void add(int docId, Iterator iterator);
+
/**
* Returns an {@link Iterator} over the updated documents and their
* values.
*/
// TODO: also use this for merging, instead of having to write through to disk first
- public abstract Iterator iterator();
+ abstract Iterator iterator();
/** Freezes internal data structures and sorts updates by docID for efficient iteration. */
- public abstract void finish();
+ abstract void finish();
/** Returns true if this instance contains any updates. */
- public abstract boolean any();
+ abstract boolean any();
- /** Returns approximate RAM bytes used. */
- public abstract long ramBytesUsed();
+ abstract int size();
- public abstract int size();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
index a66f930..8229b60 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
@@ -16,11 +16,16 @@
*/
package org.apache.lucene.index;
+import java.io.IOException;
+
import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_HEADER;
import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
/** An in-place update to a DocValues field. */
@@ -38,21 +43,22 @@ abstract class DocValuesUpdate {
final DocValuesType type;
final Term term;
final String field;
- final Object value;
- int docIDUpto = -1; // unassigned until applied, and confusing that it's here, when it's just used in BufferedDeletes...
+ // used in BufferedDeletes to apply this update only to a slice of docs. It's initialized to BufferedUpdates.MAX_INT
+ // since it's safe and most often used this way we safe object creations.
+ final int docIDUpto;
/**
* Constructor.
*
* @param term the {@link Term} which determines the documents that will be updated
* @param field the {@link NumericDocValuesField} to update
- * @param value the updated value
*/
- protected DocValuesUpdate(DocValuesType type, Term term, String field, Object value) {
+ protected DocValuesUpdate(DocValuesType type, Term term, String field, int docIDUpto) {
+ assert docIDUpto >= 0 : docIDUpto + "must be >= 0";
this.type = type;
this.term = term;
this.field = field;
- this.value = value;
+ this.docIDUpto = docIDUpto;
}
abstract long valueSizeInBytes();
@@ -65,38 +71,102 @@ abstract class DocValuesUpdate {
sizeInBytes += valueSizeInBytes();
return sizeInBytes;
}
+
+ protected abstract String valueToString();
+
+ abstract void writeTo(DataOutput output) throws IOException;
@Override
public String toString() {
- return "term=" + term + ",field=" + field + ",value=" + value + ",docIDUpto=" + docIDUpto;
+ return "term=" + term + ",field=" + field + ",value=" + valueToString() + ",docIDUpto=" + docIDUpto;
}
/** An in-place update to a binary DocValues field */
static final class BinaryDocValuesUpdate extends DocValuesUpdate {
+ private final BytesRef value;
/* Size of BytesRef: 2*INT + ARRAY_HEADER + PTR */
private static final long RAW_VALUE_SIZE_IN_BYTES = NUM_BYTES_ARRAY_HEADER + 2*Integer.BYTES + NUM_BYTES_OBJECT_REF;
-
+
BinaryDocValuesUpdate(Term term, String field, BytesRef value) {
- super(DocValuesType.BINARY, term, field, value);
+ this(term, field, value, BufferedUpdates.MAX_INT);
+ }
+
+ private BinaryDocValuesUpdate(Term term, String field, BytesRef value, int docIDUpTo) {
+ super(DocValuesType.BINARY, term, field, docIDUpTo);
+ this.value = value;
+ }
+
+ BinaryDocValuesUpdate prepareForApply(int docIDUpto) {
+ if (docIDUpto == this.docIDUpto) {
+ return this; // it's a final value so we can safely reuse this instance
+ }
+ return new BinaryDocValuesUpdate(term, field, value, docIDUpto);
}
@Override
long valueSizeInBytes() {
- return RAW_VALUE_SIZE_IN_BYTES + ((BytesRef) value).bytes.length;
+ return RAW_VALUE_SIZE_IN_BYTES + value.bytes.length;
+ }
+
+ @Override
+ protected String valueToString() {
+ return value.toString();
+ }
+
+ @Override
+ void writeTo(DataOutput out) throws IOException {
+ out.writeVInt(value.length);
+ out.writeBytes(value.bytes, value.offset, value.length);
+ }
+
+ static BytesRef readFrom(DataInput in, BytesRef scratch) throws IOException {
+ scratch.length = in.readVInt();
+ if (scratch.bytes.length < scratch.length) {
+ scratch.bytes = ArrayUtil.grow(scratch.bytes, scratch.length);
+ }
+ in.readBytes(scratch.bytes, 0, scratch.length);
+ return scratch;
}
}
/** An in-place update to a numeric DocValues field */
static final class NumericDocValuesUpdate extends DocValuesUpdate {
+ private final long value;
+
+ NumericDocValuesUpdate(Term term, String field, long value) {
+ this(term, field, value, BufferedUpdates.MAX_INT);
+ }
- NumericDocValuesUpdate(Term term, String field, Long value) {
- super(DocValuesType.NUMERIC, term, field, value);
+ private NumericDocValuesUpdate(Term term, String field, long value, int docIDUpTo) {
+ super(DocValuesType.NUMERIC, term, field, docIDUpTo);
+ this.value = value;
+ }
+
+ NumericDocValuesUpdate prepareForApply(int docIDUpto) {
+ if (docIDUpto == this.docIDUpto) {
+ return this;
+ }
+ return new NumericDocValuesUpdate(term, field, value, docIDUpto);
}
@Override
long valueSizeInBytes() {
return Long.BYTES;
}
+
+ @Override
+ protected String valueToString() {
+ return Long.toString(value);
+ }
+
+ @Override
+ void writeTo(DataOutput out) throws IOException {
+ out.writeZLong(value);
+ }
+
+ static long readFrom(DataInput in) throws IOException {
+ return in.readZLong();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
index ad9c0d1..0db043a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
@@ -25,7 +25,6 @@ import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.InfoStream;
/**
@@ -412,10 +411,10 @@ final class DocumentsWriterDeleteQueue implements Accountable {
for (DocValuesUpdate update : item) {
switch (update.type) {
case NUMERIC:
- bufferedUpdates.addNumericUpdate(new NumericDocValuesUpdate(update.term, update.field, (Long) update.value), docIDUpto);
+ bufferedUpdates.addNumericUpdate((NumericDocValuesUpdate) update, docIDUpto);
break;
case BINARY:
- bufferedUpdates.addBinaryUpdate(new BinaryDocValuesUpdate(update.term, update.field, (BytesRef) update.value), docIDUpto);
+ bufferedUpdates.addBinaryUpdate((BinaryDocValuesUpdate) update, docIDUpto);
break;
default:
throw new IllegalArgumentException(update.type + " DocValues updates not supported yet!");
@@ -436,7 +435,7 @@ final class DocumentsWriterDeleteQueue implements Accountable {
if (item.length > 0) {
sb.append("term=").append(item[0].term).append("; updates: [");
for (DocValuesUpdate update : item) {
- sb.append(update.field).append(':').append(update.value).append(',');
+ sb.append(update.field).append(':').append(update.valueToString()).append(',');
}
sb.setCharAt(sb.length()-1, ']');
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
index bebc059..6e4b6bd 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.IntConsumer;
import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
@@ -44,6 +45,7 @@ import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Counter;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.RamUsageEstimator;
@@ -76,8 +78,8 @@ final class FrozenBufferedUpdates {
// binary DV update term and their updates
final byte[] binaryDVUpdates;
- private int numericDVUpdateCount;
- private int binaryDVUpdateCount;
+ private final int numericDVUpdateCount;
+ private final int binaryDVUpdateCount;
/** Counts down once all deletes/updates have been applied */
public final CountDownLatch applied = new CountDownLatch(1);
@@ -116,19 +118,22 @@ final class FrozenBufferedUpdates {
deleteQueryLimits[upto] = ent.getValue();
upto++;
}
-
+ Counter counter = Counter.newCounter();
// TODO if a Term affects multiple fields, we could keep the updates key'd by Term
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
- numericDVUpdates = freezeNumericDVUpdates(updates.numericUpdates);
-
+ numericDVUpdates = freezeDVUpdates(updates.numericUpdates, counter::addAndGet);
+ numericDVUpdateCount = (int)counter.get();
+ counter.addAndGet(-counter.get());
+ assert counter.get() == 0;
// TODO if a Term affects multiple fields, we could keep the updates key'd by Term
// so that it maps to all fields it affects, sorted by their docUpto, and traverse
// that Term only once, applying the update to all fields that still need to be
// updated.
- binaryDVUpdates = freezeBinaryDVUpdates(updates.binaryUpdates);
-
+ binaryDVUpdates = freezeDVUpdates(updates.binaryUpdates, counter::addAndGet);
+ binaryDVUpdateCount = (int)counter.get();
+
bytesUsed = (int) (deleteTerms.ramBytesUsed() + deleteQueries.length * BYTES_PER_DEL_QUERY
+ numericDVUpdates.length + binaryDVUpdates.length);
@@ -141,60 +146,17 @@ final class FrozenBufferedUpdates {
}
}
- private byte[] freezeNumericDVUpdates(Map<String,LinkedHashMap<Term,NumericDocValuesUpdate>> numericDVUpdates)
- throws IOException {
- // TODO: we could do better here, e.g. collate the updates by field
- // so if you are updating 2 fields interleaved we don't keep writing the field strings
- try (RAMOutputStream out = new RAMOutputStream()) {
- String lastTermField = null;
- String lastUpdateField = null;
- for (LinkedHashMap<Term, NumericDocValuesUpdate> numericUpdates : numericDVUpdates.values()) {
- numericDVUpdateCount += numericUpdates.size();
- for (NumericDocValuesUpdate update : numericUpdates.values()) {
-
- int code = update.term.bytes().length << 2;
-
- String termField = update.term.field();
- if (termField.equals(lastTermField) == false) {
- code |= 1;
- }
- String updateField = update.field;
- if (updateField.equals(lastUpdateField) == false) {
- code |= 2;
- }
- out.writeVInt(code);
- out.writeVInt(update.docIDUpto);
- if ((code & 1) != 0) {
- out.writeString(termField);
- lastTermField = termField;
- }
- if ((code & 2) != 0) {
- out.writeString(updateField);
- lastUpdateField = updateField;
- }
-
- out.writeBytes(update.term.bytes().bytes, update.term.bytes().offset, update.term.bytes().length);
- out.writeZLong(((Long) update.value).longValue());
- }
- }
- byte[] bytes = new byte[(int) out.getFilePointer()];
- out.writeTo(bytes, 0);
- return bytes;
- }
- }
-
- private byte[] freezeBinaryDVUpdates(Map<String,LinkedHashMap<Term,BinaryDocValuesUpdate>> binaryDVUpdates)
+ private static <T extends DocValuesUpdate> byte[] freezeDVUpdates(Map<String,LinkedHashMap<Term, T>> dvUpdates,
+ IntConsumer updateSizeConsumer)
throws IOException {
// TODO: we could do better here, e.g. collate the updates by field
// so if you are updating 2 fields interleaved we don't keep writing the field strings
-
try (RAMOutputStream out = new RAMOutputStream()) {
String lastTermField = null;
String lastUpdateField = null;
- for (LinkedHashMap<Term, BinaryDocValuesUpdate> binaryUpdates : binaryDVUpdates.values()) {
- binaryDVUpdateCount += binaryUpdates.size();
- for (BinaryDocValuesUpdate update : binaryUpdates.values()) {
-
+ for (LinkedHashMap<Term, T> updates : dvUpdates.values()) {
+ updateSizeConsumer.accept(updates.size());
+ for (T update : updates.values()) {
int code = update.term.bytes().length << 2;
String termField = update.term.field();
@@ -216,10 +178,7 @@ final class FrozenBufferedUpdates {
lastUpdateField = updateField;
}
out.writeBytes(update.term.bytes().bytes, update.term.bytes().offset, update.term.bytes().length);
-
- BytesRef value = (BytesRef) update.value;
- out.writeVInt(value.length);
- out.writeBytes(value.bytes, value.offset, value.length);
+ update.writeTo(out);
}
}
byte[] bytes = new byte[(int) out.getFilePointer()];
@@ -521,13 +480,13 @@ final class FrozenBufferedUpdates {
// because we will run on the newly merged segment next:
continue;
}
-
+ final boolean isSegmentPrivateDeletes = privateSegment != null;
if (numericDVUpdates.length > 0) {
- updateCount += applyDocValuesUpdates(segState, numericDVUpdates, true);
+ updateCount += applyDocValuesUpdates(segState, numericDVUpdates, true, delGen, isSegmentPrivateDeletes);
}
if (binaryDVUpdates.length > 0) {
- updateCount += applyDocValuesUpdates(segState, binaryDVUpdates, false);
+ updateCount += applyDocValuesUpdates(segState, binaryDVUpdates, false, delGen, isSegmentPrivateDeletes);
}
}
@@ -544,8 +503,9 @@ final class FrozenBufferedUpdates {
return updateCount;
}
- private long applyDocValuesUpdates(BufferedUpdatesStream.SegmentState segState,
- byte[] updates, boolean isNumeric) throws IOException {
+ private static long applyDocValuesUpdates(BufferedUpdatesStream.SegmentState segState, byte[] updates,
+ boolean isNumeric, long delGen,
+ boolean segmentPrivateDeletes) throws IOException {
TermsEnum termsEnum = null;
PostingsEnum postingsEnum = null;
@@ -592,9 +552,9 @@ final class FrozenBufferedUpdates {
}
in.readBytes(term.bytes, 0, term.length);
- int limit;
+ final int limit;
if (delGen == segState.delGen) {
- assert privateSegment != null;
+ assert segmentPrivateDeletes;
limit = docIDUpto;
} else {
limit = Integer.MAX_VALUE;
@@ -622,17 +582,14 @@ final class FrozenBufferedUpdates {
}
}
- // TODO: can we avoid boxing here w/o fully forking this method?
- Object value;
+ final BytesRef binaryValue;
+ final long longValue;
if (isNumeric) {
- value = Long.valueOf(in.readZLong());
+ longValue = NumericDocValuesUpdate.readFrom(in);
+ binaryValue = null;
} else {
- value = scratch;
- scratch.length = in.readVInt();
- if (scratch.bytes.length < scratch.length) {
- scratch.bytes = ArrayUtil.grow(scratch.bytes, scratch.length);
- }
- in.readBytes(scratch.bytes, 0, scratch.length);
+ longValue = -1;
+ binaryValue = BinaryDocValuesUpdate.readFrom(in, scratch);
}
if (termsEnum == null) {
@@ -641,10 +598,8 @@ final class FrozenBufferedUpdates {
}
if (termsEnum.seekExact(term)) {
-
// we don't need term frequencies for this
postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
-
DocValuesFieldUpdates dvUpdates = holder.get(updateField);
if (dvUpdates == null) {
if (isNumeric) {
@@ -652,38 +607,38 @@ final class FrozenBufferedUpdates {
} else {
dvUpdates = new BinaryDocValuesFieldUpdates(delGen, updateField, segState.reader.maxDoc());
}
-
holder.put(updateField, dvUpdates);
}
-
- if (segState.rld.sortMap != null && privateSegment != null) {
+ final IntConsumer docIdConsumer;
+ final DocValuesFieldUpdates update = dvUpdates;
+ if (isNumeric) {
+ docIdConsumer = doc -> update.add(doc, longValue);
+ } else {
+ docIdConsumer = doc -> update.add(doc, binaryValue);
+ }
+ final Bits acceptDocs = segState.rld.getLiveDocs();
+ if (segState.rld.sortMap != null && segmentPrivateDeletes) {
// This segment was sorted on flush; we must apply seg-private deletes carefully in this case:
int doc;
- final Bits acceptDocs = segState.rld.getLiveDocs();
while ((doc = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-
- if (acceptDocs != null && acceptDocs.get(doc) == false) {
- continue;
- }
-
- // The limit is in the pre-sorted doc space:
- if (segState.rld.sortMap.newToOld(doc) < limit) {
- dvUpdates.add(doc, value);
- updateCount++;
+ if (acceptDocs == null || acceptDocs.get(doc)) {
+ // The limit is in the pre-sorted doc space:
+ if (segState.rld.sortMap.newToOld(doc) < limit) {
+ docIdConsumer.accept(doc);
+ updateCount++;
+ }
}
}
} else {
int doc;
- final Bits acceptDocs = segState.rld.getLiveDocs();
while ((doc = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
if (doc >= limit) {
break; // no more docs that can be updated for this term
}
- if (acceptDocs != null && acceptDocs.get(doc) == false) {
- continue;
+ if (acceptDocs == null || acceptDocs.get(doc)) {
+ docIdConsumer.accept(doc);
+ updateCount++;
}
- dvUpdates.add(doc, value);
- updateCount++;
}
}
}
@@ -888,7 +843,7 @@ final class FrozenBufferedUpdates {
}
}
if (deleteQueries.length != 0) {
- s += " numDeleteQuerys=" + deleteQueries.length;
+ s += " numDeleteQueries=" + deleteQueries.length;
}
if (numericDVUpdates.length > 0) {
s += " numNumericDVUpdates=" + numericDVUpdateCount;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index ccbfe5c..b209146 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -3699,7 +3699,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
int mappedDoc = segDocMap.get(segLeafDocMap.get(doc));
if (mappedDoc != -1) {
// not deleted
- mappedUpdates.add(mappedDoc, it.value());
+ mappedUpdates.add(mappedDoc, it);
anyDVUpdates = true;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
index 94a9643..724c58c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PackedInts;
@@ -31,7 +32,7 @@ import org.apache.lucene.util.packed.PagedMutable;
*
* @lucene.experimental
*/
-class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
+final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
// TODO: can't this just be NumericDocValues now? avoid boxing the long value...
final static class Iterator extends DocValuesFieldUpdates.Iterator {
@@ -40,7 +41,7 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
private final PagedMutable docs;
private long idx = 0; // long so we don't overflow if size == Integer.MAX_VALUE
private int doc = -1;
- private Long value = null;
+ private long value;
private final long delGen;
Iterator(int size, PagedGrowableWriter values, PagedMutable docs, long delGen) {
@@ -50,15 +51,20 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
this.delGen = delGen;
}
+
@Override
- Long value() {
+ long longValue() {
return value;
}
-
+
+ @Override
+ BytesRef binaryValue() {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public int nextDoc() {
if (idx >= size) {
- value = null;
return doc = DocIdSetIterator.NO_MORE_DOCS;
}
doc = (int) docs.get(idx);
@@ -68,7 +74,7 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
++idx;
}
// idx points to the "next" element
- value = Long.valueOf(values.get(idx - 1));
+ value = values.get(idx - 1);
return doc;
}
@@ -101,28 +107,36 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
@Override
- public synchronized void add(int doc, Object value) {
+ void add(int doc, BytesRef value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ void add(int docId, DocValuesFieldUpdates.Iterator iterator) {
+ add(docId, iterator.longValue());
+ }
+
+ synchronized void add(int doc, long value) {
if (finished) {
throw new IllegalStateException("already finished");
}
assert doc < maxDoc;
-
+
// TODO: if the Sorter interface changes to take long indexes, we can remove that limitation
if (size == Integer.MAX_VALUE) {
throw new IllegalStateException("cannot support more than Integer.MAX_VALUE doc/value entries");
}
- Long val = (Long) value;
-
+
// grow the structures to have room for more elements
if (docs.size() == size) {
docs = docs.grow(size + 1);
values = values.grow(size + 1);
}
-
+
docs.set(size, doc);
- values.set(size, val.longValue());
+ values.set(size, value);
++size;
}
@@ -156,13 +170,13 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
// increasing docID order:
// NOTE: we can have ties here, when the same docID was updated in the same segment, in which case we rely on sort being
// stable and preserving original order so the last update to that docID wins
- return Integer.compare((int) docs.get(i), (int) docs.get(j));
+ return Long.compare(docs.get(i), docs.get(j));
}
}.sort(0, size);
}
@Override
- public Iterator iterator() {
+ Iterator iterator() {
if (finished == false) {
throw new IllegalStateException("call finish first");
}
@@ -170,7 +184,7 @@ class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
@Override
- public boolean any() {
+ boolean any() {
return size > 0;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index d61e8ed..b31bc49 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -146,12 +146,7 @@ final class ReadersAndUpdates {
if (update.getFinished() == false) {
throw new IllegalArgumentException("call finish first");
}
- List<DocValuesFieldUpdates> fieldUpdates = pendingDVUpdates.get(update.field);
- if (fieldUpdates == null) {
- fieldUpdates = new ArrayList<>();
- pendingDVUpdates.put(update.field, fieldUpdates);
- }
-
+ List<DocValuesFieldUpdates> fieldUpdates = pendingDVUpdates.computeIfAbsent(update.field, key -> new ArrayList<>());
assert assertNoDupGen(fieldUpdates, update);
ramBytesUsed.addAndGet(update.ramBytesUsed());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
new file mode 100644
index 0000000..2066dcd
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestDocValuesFieldUpdates extends LuceneTestCase {
+
+ public void testMergeIterator() {
+ NumericDocValuesFieldUpdates updates1 = new NumericDocValuesFieldUpdates(0, "test", 6);
+ NumericDocValuesFieldUpdates updates2 = new NumericDocValuesFieldUpdates(1, "test", 6);
+ NumericDocValuesFieldUpdates updates3 = new NumericDocValuesFieldUpdates(2, "test", 6);
+ NumericDocValuesFieldUpdates updates4 = new NumericDocValuesFieldUpdates(2, "test", 6);
+
+ updates1.add(0, 1);
+ updates1.add(4, 0);
+ updates1.add(1, 4);
+ updates1.add(2, 5);
+ updates1.add(4, 9);
+ assertTrue(updates1.any());
+
+ updates2.add(0, 18);
+ updates2.add(1, 7);
+ updates2.add(2, 19);
+ updates2.add(5, 24);
+ assertTrue(updates2.any());
+
+ updates3.add(2, 42);
+ assertTrue(updates3.any());
+ assertFalse(updates4.any());
+ updates1.finish();
+ updates2.finish();
+ updates3.finish();
+ updates4.finish();
+ List<DocValuesFieldUpdates.Iterator> iterators = Arrays.asList(updates1.iterator(), updates2.iterator(),
+ updates3.iterator(), updates4.iterator());
+ Collections.shuffle(iterators, random());
+ DocValuesFieldUpdates.Iterator iterator = DocValuesFieldUpdates
+ .mergedIterator(iterators.toArray(new DocValuesFieldUpdates.Iterator[0]));
+ assertEquals(0, iterator.nextDoc());
+ assertEquals(18, iterator.longValue());
+ assertEquals(1, iterator.nextDoc());
+ assertEquals(7, iterator.longValue());
+ assertEquals(2, iterator.nextDoc());
+ assertEquals(42, iterator.longValue());
+ assertEquals(4, iterator.nextDoc());
+ assertEquals(9, iterator.longValue());
+ assertEquals(5, iterator.nextDoc());
+ assertEquals(24, iterator.longValue());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b43b0919/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
index 4df6d16..9119993 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
@@ -32,6 +32,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.Version;
@@ -201,7 +202,18 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
private DocValuesFieldUpdates singleUpdate(List<Integer> docsDeleted, int maxDoc) {
return new DocValuesFieldUpdates(maxDoc, 0, "_soft_deletes", DocValuesType.NUMERIC) {
@Override
- public void add(int doc, Object value) {
+ public void add(int doc, long value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void add(int doc, BytesRef value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void add(int docId, Iterator iterator) {
+ throw new UnsupportedOperationException();
}
@Override
@@ -216,13 +228,18 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
}
@Override
- public int docID() {
- return doc;
+ long longValue() {
+ return 1;
}
@Override
- Object value() {
- return 1;
+ BytesRef binaryValue() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int docID() {
+ return doc;
}
@Override
[40/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-8998: documentation
fix.
Posted by ab...@apache.org.
SOLR-8998: documentation fix.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/beaf3a47
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/beaf3a47
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/beaf3a47
Branch: refs/heads/jira/solr-11779
Commit: beaf3a47ebe6ad79572bccaeafc2551dc86f19c6
Parents: 1b76011
Author: Mikhail Khludnev <mk...@apache.org>
Authored: Sun May 6 16:46:56 2018 +0300
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Sun May 6 16:46:56 2018 +0300
----------------------------------------------------------------------
solr/solr-ref-guide/src/blockjoin-faceting.adoc | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/beaf3a47/solr/solr-ref-guide/src/blockjoin-faceting.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/blockjoin-faceting.adoc b/solr/solr-ref-guide/src/blockjoin-faceting.adoc
index 7e2408b..7299d1e 100644
--- a/solr/solr-ref-guide/src/blockjoin-faceting.adoc
+++ b/solr/solr-ref-guide/src/blockjoin-faceting.adoc
@@ -21,8 +21,7 @@ BlockJoin facets allow you to aggregate children facet counts by their parents.
It is a common requirement that if a parent document has several children documents, all of them need to increment facet value count only once. This functionality is provided by `BlockJoinDocSetFacetComponent`, and `BlockJoinFacetComponent` just an alias for compatibility.
CAUTION: This functionality is considered deprecated. Users are encouraged to use `uniqueBlock(\_root_)` aggregation under terms facet in <<json-facet-api.adoc#Blockjoinfacetexample,JSON Facet API>>.
-
-CAUTION: This component is considered experimental, and must be explicitly enabled for a request handler in `solrconfig.xml`, in the same way as any other <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#requesthandlers-and-searchcomponents-in-solrconfig,search component>>.
+If this component is used, it must be explicitly enabled for a request handler in `solrconfig.xml`, in the same way as any other <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#requesthandlers-and-searchcomponents-in-solrconfig,search component>>.
This example shows how you could add this search components to `solrconfig.xml` and define it in request handler:
[08/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE_8265: add CHANGES
entry
Posted by ab...@apache.org.
LUCENE_8265: add CHANGES entry
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/70abbe74
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/70abbe74
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/70abbe74
Branch: refs/heads/jira/solr-11779
Commit: 70abbe7433fc205e4abd05ebfc0fcf9399bf0f46
Parents: fc0878c
Author: Mike McCandless <mi...@apache.org>
Authored: Sat Apr 28 09:57:58 2018 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Sat Apr 28 09:57:58 2018 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/70abbe74/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4d902fe..a5d0816 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -150,6 +150,9 @@ New Features
but to handle Korean using mecab-ko-dic and morphological analysis.
(Robert Muir, Jim Ferenczi)
+* LUCENE-8265: WordDelimter/GraphFilter now have an option to skip tokens
+ marked with KeywordAttribute (Mike Sokolov via Mike McCandless)
+
Bug Fixes
* LUCENE-8266: Detect bogus tiles when creating a standard polygon and
[23/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8289: Share logic
between Numeric and Binary DocValuesFieldUpdates
Posted by ab...@apache.org.
LUCENE-8289: Share logic between Numeric and Binary DocValuesFieldUpdates
NumericDocValuesFieldUpdates and BinaryDocValuesFieldUpdates duplicate
a significant amount of logic that can all be pushed into the base class.
This change moves all the logic that is independent of the type to the base
class.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/82e7cb23
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/82e7cb23
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/82e7cb23
Branch: refs/heads/jira/solr-11779
Commit: 82e7cb2322a1978c6e6d03710b4483f447f36f61
Parents: df713fc
Author: Simon Willnauer <si...@apache.org>
Authored: Wed May 2 11:55:32 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Wed May 2 17:22:45 2018 +0200
----------------------------------------------------------------------
.../index/BinaryDocValuesFieldUpdates.java | 156 +++++--------------
.../lucene/index/DocValuesFieldUpdates.java | 145 ++++++++++++++++-
.../index/NumericDocValuesFieldUpdates.java | 141 ++++-------------
.../lucene/index/TestPendingSoftDeletes.java | 19 ---
4 files changed, 202 insertions(+), 259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/82e7cb23/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
index db9aa09..fa60929 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
@@ -18,10 +18,8 @@ package org.apache.lucene.index;
import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PagedGrowableWriter;
@@ -35,25 +33,18 @@ import org.apache.lucene.util.packed.PagedMutable;
*/
final class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
- final static class Iterator extends DocValuesFieldUpdates.Iterator {
- private final int size;
+ final static class Iterator extends DocValuesFieldUpdates.AbstractIterator {
private final PagedGrowableWriter offsets;
private final PagedGrowableWriter lengths;
- private final PagedMutable docs;
- private long idx = 0; // long so we don't overflow if size == Integer.MAX_VALUE
- private int doc = -1;
private final BytesRef value;
private int offset, length;
- private final long delGen;
-
+
Iterator(int size, PagedGrowableWriter offsets, PagedGrowableWriter lengths,
PagedMutable docs, BytesRef values, long delGen) {
+ super(size, docs, delGen);
this.offsets = offsets;
- this.size = size;
this.lengths = lengths;
- this.docs = docs;
value = values.clone();
- this.delGen = delGen;
}
@Override
@@ -64,35 +55,9 @@ final class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
@Override
- public int nextDoc() {
- if (idx >= size) {
- offset = -1;
- return doc = DocIdSetIterator.NO_MORE_DOCS;
- }
- doc = (int) docs.get(idx);
- ++idx;
- while (idx < size && docs.get(idx) == doc) {
- // scan forward to last update to this doc
- ++idx;
- }
- // idx points to the "next" element
- long prevIdx = idx - 1;
- // cannot change 'value' here because nextDoc is called before the
- // value is used, and it's a waste to clone the BytesRef when we
- // obtain the value
- offset = (int) offsets.get(prevIdx);
- length = (int) lengths.get(prevIdx);
- return doc;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- long delGen() {
- return delGen;
+ protected void set(long idx) {
+ offset = (int) offsets.get(idx);
+ length = (int) lengths.get(idx);
}
@Override
@@ -101,27 +66,17 @@ final class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
}
- private PagedMutable docs;
private PagedGrowableWriter offsets, lengths;
private BytesRefBuilder values;
- private int size;
- private final int bitsPerValue;
-
+
public BinaryDocValuesFieldUpdates(long delGen, String field, int maxDoc) {
super(maxDoc, delGen, field, DocValuesType.BINARY);
- bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
- docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
offsets = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);
lengths = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);
values = new BytesRefBuilder();
}
@Override
- public int size() {
- return size;
- }
-
- @Override
public void add(int doc, long value) {
throw new UnsupportedOperationException();
}
@@ -133,92 +88,53 @@ final class BinaryDocValuesFieldUpdates extends DocValuesFieldUpdates {
@Override
synchronized public void add(int doc, BytesRef value) {
- if (finished) {
- throw new IllegalStateException("already finished");
- }
-
- assert doc < maxDoc: "doc=" + doc + " maxDoc=" + maxDoc;
-
- // TODO: if the Sorter interface changes to take long indexes, we can remove that limitation
- if (size == Integer.MAX_VALUE) {
- throw new IllegalStateException("cannot support more than Integer.MAX_VALUE doc/value entries");
- }
-
- // grow the structures to have room for more elements
- if (docs.size() == size) {
- docs = docs.grow(size + 1);
- offsets = offsets.grow(size + 1);
- lengths = lengths.grow(size + 1);
- }
-
- docs.set(size, doc);
- offsets.set(size, values.length());
- lengths.set(size, value.length);
+ int index = add(doc);
+ offsets.set(index, values.length());
+ lengths.set(index, value.length);
values.append(value);
- ++size;
}
@Override
- public void finish() {
- if (finished) {
- throw new IllegalStateException("already finished");
- }
- finished = true;
+ protected void swap(int i, int j) {
+ super.swap(i, j);
- // shrink wrap
- if (size < docs.size()) {
- docs = docs.resize(size);
- offsets = offsets.resize(size);
- lengths = lengths.resize(size);
- }
+ long tmpOffset = offsets.get(j);
+ offsets.set(j, offsets.get(i));
+ offsets.set(i, tmpOffset);
- new InPlaceMergeSorter() {
- @Override
- protected void swap(int i, int j) {
- long tmpDoc = docs.get(j);
- docs.set(j, docs.get(i));
- docs.set(i, tmpDoc);
-
- long tmpOffset = offsets.get(j);
- offsets.set(j, offsets.get(i));
- offsets.set(i, tmpOffset);
-
- long tmpLength = lengths.get(j);
- lengths.set(j, lengths.get(i));
- lengths.set(i, tmpLength);
- }
-
- @Override
- protected int compare(int i, int j) {
- // increasing docID order:
- // NOTE: we can have ties here, when the same docID was updated in the same segment, in which case we rely on sort being
- // stable and preserving original order so the last update to that docID wins
- return Integer.compare((int) docs.get(i), (int) docs.get(j));
- }
- }.sort(0, size);
+ long tmpLength = lengths.get(j);
+ lengths.set(j, lengths.get(i));
+ lengths.set(i, tmpLength);
}
@Override
- public Iterator iterator() {
- if (finished == false) {
- throw new IllegalStateException("call finish first");
- }
- return new Iterator(size, offsets, lengths, docs, values.get(), delGen);
+ protected void grow(int size) {
+ super.grow(size);
+ offsets = offsets.grow(size);
+ lengths = lengths.grow(size);
+ }
+
+ @Override
+ protected void resize(int size) {
+ super.resize(size);
+ offsets = offsets.resize(size);
+ lengths = lengths.resize(size);
}
@Override
- public boolean any() {
- return size > 0;
+ public Iterator iterator() {
+ ensureFinished();
+ return new Iterator(size, offsets, lengths, docs, values.get(), delGen);
}
@Override
public long ramBytesUsed() {
- return offsets.ramBytesUsed()
+ return super.ramBytesUsed()
+ + offsets.ramBytesUsed()
+ lengths.ramBytesUsed()
- + docs.ramBytesUsed()
+ RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
- + 4 * Integer.BYTES
- + 5 * RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ + 2 * Integer.BYTES
+ + 3 * RamUsageEstimator.NUM_BYTES_OBJECT_REF
+ values.bytes().length;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/82e7cb23/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
index 4a7f5b8..f4e15f0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
@@ -16,9 +16,14 @@
*/
package org.apache.lucene.index;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PagedMutable;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
@@ -218,9 +223,12 @@ abstract class DocValuesFieldUpdates implements Accountable {
final String field;
final DocValuesType type;
final long delGen;
- protected boolean finished;
+ private final int bitsPerValue;
+ private boolean finished;
protected final int maxDoc;
-
+ protected PagedMutable docs;
+ protected int size;
+
protected DocValuesFieldUpdates(int maxDoc, long delGen, String field, DocValuesType type) {
this.maxDoc = maxDoc;
this.delGen = delGen;
@@ -229,9 +237,11 @@ abstract class DocValuesFieldUpdates implements Accountable {
throw new NullPointerException("DocValuesType must not be null");
}
this.type = type;
+ bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
+ docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
}
- boolean getFinished() {
+ final boolean getFinished() {
return finished;
}
@@ -254,11 +264,132 @@ abstract class DocValuesFieldUpdates implements Accountable {
abstract Iterator iterator();
/** Freezes internal data structures and sorts updates by docID for efficient iteration. */
- abstract void finish();
+ final synchronized void finish() {
+ if (finished) {
+ throw new IllegalStateException("already finished");
+ }
+ finished = true;
+
+ // shrink wrap
+ if (size < docs.size()) {
+ resize(size);
+ }
+ new InPlaceMergeSorter() {
+ @Override
+ protected void swap(int i, int j) {
+ DocValuesFieldUpdates.this.swap(i, j);
+ }
+
+ @Override
+ protected int compare(int i, int j) {
+ // increasing docID order:
+ // NOTE: we can have ties here, when the same docID was updated in the same segment, in which case we rely on sort being
+ // stable and preserving original order so the last update to that docID wins
+ return Long.compare(docs.get(i), docs.get(j));
+ }
+ }.sort(0, size);
+ }
/** Returns true if this instance contains any updates. */
- abstract boolean any();
-
- abstract int size();
+ synchronized final boolean any() {
+ return size > 0;
+ }
+ synchronized final int size() {
+ return size;
+ }
+
+ final synchronized int add(int doc) {
+ if (finished) {
+ throw new IllegalStateException("already finished");
+ }
+ assert doc < maxDoc;
+
+ // TODO: if the Sorter interface changes to take long indexes, we can remove that limitation
+ if (size == Integer.MAX_VALUE) {
+ throw new IllegalStateException("cannot support more than Integer.MAX_VALUE doc/value entries");
+ }
+ // grow the structures to have room for more elements
+ if (docs.size() == size) {
+ grow(size+1);
+ }
+
+ docs.set(size, doc);
+ ++size;
+ return size-1;
+ }
+
+ protected void swap(int i, int j) {
+ long tmpDoc = docs.get(j);
+ docs.set(j, docs.get(i));
+ docs.set(i, tmpDoc);
+ }
+
+ protected void grow(int size) {
+ docs = docs.grow(size);
+ }
+
+ protected void resize(int size) {
+ docs = docs.resize(size);
+ }
+
+ protected final void ensureFinished() {
+ if (finished == false) {
+ throw new IllegalStateException("call finish first");
+ }
+ }
+ @Override
+ public long ramBytesUsed() {
+ return docs.ramBytesUsed()
+ + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
+ + 2 * Integer.BYTES
+ + 2 + Long.BYTES
+ + RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ }
+
+ // TODO: can't this just be NumericDocValues now? avoid boxing the long value...
+ protected abstract static class AbstractIterator extends DocValuesFieldUpdates.Iterator {
+ private final int size;
+ private final PagedMutable docs;
+ private long idx = 0; // long so we don't overflow if size == Integer.MAX_VALUE
+ private int doc = -1;
+ private final long delGen;
+
+ AbstractIterator(int size, PagedMutable docs, long delGen) {
+ this.size = size;
+ this.docs = docs;
+ this.delGen = delGen;
+ }
+
+ @Override
+ public final int nextDoc() {
+ if (idx >= size) {
+ return doc = DocIdSetIterator.NO_MORE_DOCS;
+ }
+ doc = (int) docs.get(idx);
+ ++idx;
+ while (idx < size && docs.get(idx) == doc) {
+ // scan forward to last update to this doc
+ ++idx;
+ }
+ set(idx-1);
+ return doc;
+ }
+
+ /**
+ * Called when the iterator moved to the next document
+ * @param idx the internal index to set the value to
+ */
+ protected abstract void set(long idx);
+
+ @Override
+ public final int docID() {
+ return doc;
+ }
+
+ @Override
+ final long delGen() {
+ return delGen;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/82e7cb23/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
index 724c58c..626a61e 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
@@ -17,9 +17,7 @@
package org.apache.lucene.index;
import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PagedGrowableWriter;
@@ -35,23 +33,14 @@ import org.apache.lucene.util.packed.PagedMutable;
final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
// TODO: can't this just be NumericDocValues now? avoid boxing the long value...
- final static class Iterator extends DocValuesFieldUpdates.Iterator {
- private final int size;
+ final static class Iterator extends DocValuesFieldUpdates.AbstractIterator {
private final PagedGrowableWriter values;
- private final PagedMutable docs;
- private long idx = 0; // long so we don't overflow if size == Integer.MAX_VALUE
- private int doc = -1;
private long value;
- private final long delGen;
-
+
Iterator(int size, PagedGrowableWriter values, PagedMutable docs, long delGen) {
- this.size = size;
+ super(size, docs, delGen);
this.values = values;
- this.docs = docs;
- this.delGen = delGen;
}
-
-
@Override
long longValue() {
return value;
@@ -63,49 +52,16 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
}
@Override
- public int nextDoc() {
- if (idx >= size) {
- return doc = DocIdSetIterator.NO_MORE_DOCS;
- }
- doc = (int) docs.get(idx);
- ++idx;
- while (idx < size && docs.get(idx) == doc) {
- // scan forward to last update to this doc
- ++idx;
- }
- // idx points to the "next" element
- value = values.get(idx - 1);
- return doc;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- long delGen() {
- return delGen;
+ protected void set(long idx) {
+ value = values.get(idx);
}
}
-
- private final int bitsPerValue;
- private PagedMutable docs;
private PagedGrowableWriter values;
- private int size;
-
+
public NumericDocValuesFieldUpdates(long delGen, String field, int maxDoc) {
super(maxDoc, delGen, field, DocValuesType.NUMERIC);
- bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
- docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
values = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);
}
-
- @Override
- public int size() {
- return size;
- }
-
@Override
void add(int doc, BytesRef value) {
throw new UnsupportedOperationException();
@@ -116,84 +72,43 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
add(docId, iterator.longValue());
}
+ @Override
synchronized void add(int doc, long value) {
- if (finished) {
- throw new IllegalStateException("already finished");
- }
-
- assert doc < maxDoc;
-
- // TODO: if the Sorter interface changes to take long indexes, we can remove that limitation
- if (size == Integer.MAX_VALUE) {
- throw new IllegalStateException("cannot support more than Integer.MAX_VALUE doc/value entries");
- }
-
-
- // grow the structures to have room for more elements
- if (docs.size() == size) {
- docs = docs.grow(size + 1);
- values = values.grow(size + 1);
- }
-
- docs.set(size, doc);
- values.set(size, value);
- ++size;
+ int add = add(doc);
+ values.set(add, value);
}
@Override
- public void finish() {
- if (finished) {
- throw new IllegalStateException("already finished");
- }
- finished = true;
-
- // shrink wrap
- if (size < docs.size()) {
- docs = docs.resize(size);
- values = values.resize(size);
- }
+ protected void swap(int i, int j) {
+ super.swap(i, j);
+ long tmpVal = values.get(j);
+ values.set(j, values.get(i));
+ values.set(i, tmpVal);
+ }
- new InPlaceMergeSorter() {
- @Override
- protected void swap(int i, int j) {
- long tmpDoc = docs.get(j);
- docs.set(j, docs.get(i));
- docs.set(i, tmpDoc);
-
- long tmpVal = values.get(j);
- values.set(j, values.get(i));
- values.set(i, tmpVal);
- }
+ @Override
+ protected void grow(int size) {
+ super.grow(size);
+ values = values.grow(size);
+ }
- @Override
- protected int compare(int i, int j) {
- // increasing docID order:
- // NOTE: we can have ties here, when the same docID was updated in the same segment, in which case we rely on sort being
- // stable and preserving original order so the last update to that docID wins
- return Long.compare(docs.get(i), docs.get(j));
- }
- }.sort(0, size);
+ @Override
+ protected void resize(int size) {
+ super.resize(size);
+ values = values.resize(size);
}
@Override
Iterator iterator() {
- if (finished == false) {
- throw new IllegalStateException("call finish first");
- }
+ ensureFinished();
return new Iterator(size, values, docs, delGen);
}
@Override
- boolean any() {
- return size > 0;
- }
-
- @Override
public long ramBytesUsed() {
return values.ramBytesUsed()
- + docs.ramBytesUsed()
- + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
- + 2 * Integer.BYTES
- + 2 * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ + super.ramBytesUsed()
+ + Long.BYTES
+ + RamUsageEstimator.NUM_BYTES_OBJECT_REF;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/82e7cb23/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
index 9119993..8bd7ca6 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
@@ -248,25 +248,6 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
}
};
}
-
- @Override
- public void finish() {
- }
-
- @Override
- public boolean any() {
- return true;
- }
-
- @Override
- public long ramBytesUsed() {
- return 0;
- }
-
- @Override
- public int size() {
- return 1;
- }
};
}
}
[31/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8231: Add missing
part of speech filter in the SPI META-INF file
Posted by ab...@apache.org.
LUCENE-8231: Add missing part of speech filter in the SPI META-INF file
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9b261087
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9b261087
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9b261087
Branch: refs/heads/jira/solr-11779
Commit: 9b261087abcd7ef350e49d4fa4e72e075a135799
Parents: 591fc66
Author: Jim Ferenczi <ji...@elastic.co>
Authored: Thu May 3 16:11:52 2018 +0200
Committer: Jim Ferenczi <ji...@elastic.co>
Committed: Thu May 3 16:11:52 2018 +0200
----------------------------------------------------------------------
.../services/org.apache.lucene.analysis.util.TokenFilterFactory | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b261087/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory b/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
index 07a41c7..4fff753 100644
--- a/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
+++ b/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
@@ -13,4 +13,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+org.apache.lucene.analysis.ko.KoreanPartOfSpeechStopFilterFactory
org.apache.lucene.analysis.ko.KoreanReadingFormFilterFactory
\ No newline at end of file
[33/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-11277: Add auto
hard commit setting based on tlog size (this closes #358)
Posted by ab...@apache.org.
SOLR-11277: Add auto hard commit setting based on tlog size (this closes #358)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b6174896
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b6174896
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b6174896
Branch: refs/heads/jira/solr-11779
Commit: b617489638db4ddca63e5fbc45a58c5695a021d3
Parents: ab11867
Author: Anshum Gupta <an...@apache.org>
Authored: Thu May 3 15:00:47 2018 -0700
Committer: Anshum Gupta <an...@apache.org>
Committed: Thu May 3 15:00:47 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../java/org/apache/solr/core/SolrConfig.java | 45 ++-
.../org/apache/solr/update/CommitTracker.java | 66 ++-
.../solr/update/DirectUpdateHandler2.java | 45 ++-
.../org/apache/solr/update/TransactionLog.java | 7 +
.../java/org/apache/solr/update/UpdateLog.java | 7 +
.../conf/bad-solrconfig-no-autocommit-tag.xml | 52 +++
.../solr/collection1/conf/solrconfig-tlog.xml | 4 +
.../test/org/apache/solr/core/TestConfig.java | 36 ++
.../solr/update/MaxSizeAutoCommitTest.java | 398 +++++++++++++++++++
10 files changed, 642 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b925fe1..d4c2097 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -110,6 +110,8 @@ New Features
* SOLR-11278: Add IgnoreLargeDocumentProcessFactory (Cao Manh Dat, David Smiley)
+* SOLR-11277: Add auto hard-commit settings based on tlog size (Rupa Shankar, Anshum Gupta)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/java/org/apache/solr/core/SolrConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
index 6c67645..6bfa08d 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
@@ -54,6 +54,7 @@ import java.util.regex.Pattern;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathConstants;
+import org.apache.commons.io.FileUtils;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.util.Version;
@@ -448,6 +449,7 @@ public class SolrConfig extends Config implements MapSerializable {
return new UpdateHandlerInfo(get("updateHandler/@class", null),
getInt("updateHandler/autoCommit/maxDocs", -1),
getInt("updateHandler/autoCommit/maxTime", -1),
+ convertHeapOptionStyleConfigStringToBytes(get("updateHandler/autoCommit/maxSize", "")),
getBool("updateHandler/indexWriter/closeWaitsForMerges", true),
getBool("updateHandler/autoCommit/openSearcher", true),
getInt("updateHandler/autoSoftCommit/maxDocs", -1),
@@ -455,6 +457,44 @@ public class SolrConfig extends Config implements MapSerializable {
getBool("updateHandler/commitWithin/softCommit", true));
}
+ /**
+ * Converts a Java heap option-like config string to bytes. Valid suffixes are: 'k', 'm', 'g'
+ * (case insensitive). If there is no suffix, the default unit is bytes.
+ * For example, 50k = 50KB, 20m = 20MB, 4g = 4GB, 300 = 300 bytes
+ * @param configStr the config setting to parse
+ * @return the size, in bytes. -1 if the given config string is empty
+ */
+ protected static long convertHeapOptionStyleConfigStringToBytes(String configStr) {
+ if (configStr.isEmpty()) {
+ return -1;
+ }
+ long multiplier = 1;
+ String numericValueStr = configStr;
+ char suffix = Character.toLowerCase(configStr.charAt(configStr.length() - 1));
+ if (Character.isLetter(suffix)) {
+ if (suffix == 'k') {
+ multiplier = FileUtils.ONE_KB;
+ }
+ else if (suffix == 'm') {
+ multiplier = FileUtils.ONE_MB;
+ }
+ else if (suffix == 'g') {
+ multiplier = FileUtils.ONE_GB;
+ } else {
+ throw new RuntimeException("Invalid suffix. Valid suffixes are 'k' (KB), 'm' (MB), 'g' (G). "
+ + "No suffix means the amount is in bytes. ");
+ }
+ numericValueStr = configStr.substring(0, configStr.length() - 1);
+ }
+ try {
+ return Long.parseLong(numericValueStr) * multiplier;
+ } catch (NumberFormatException e) {
+ throw new RuntimeException("Invalid format. The config setting should be a long with an "
+ + "optional letter suffix. Valid suffixes are 'k' (KB), 'm' (MB), 'g' (G). "
+ + "No suffix means the amount is in bytes.");
+ }
+ }
+
private void loadPluginInfo(SolrPluginInfo pluginInfo) {
boolean requireName = pluginInfo.options.contains(REQUIRE_NAME);
boolean requireClass = pluginInfo.options.contains(REQUIRE_CLASS);
@@ -631,6 +671,7 @@ public class SolrConfig extends Config implements MapSerializable {
public final String className;
public final int autoCommmitMaxDocs, autoCommmitMaxTime,
autoSoftCommmitMaxDocs, autoSoftCommmitMaxTime;
+ public final long autoCommitMaxSizeBytes;
public final boolean indexWriterCloseWaitsForMerges;
public final boolean openSearcher; // is opening a new searcher part of hard autocommit?
public final boolean commitWithinSoftCommit;
@@ -638,12 +679,14 @@ public class SolrConfig extends Config implements MapSerializable {
/**
* @param autoCommmitMaxDocs set -1 as default
* @param autoCommmitMaxTime set -1 as default
+ * @param autoCommitMaxSize set -1 as default
*/
- public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, boolean indexWriterCloseWaitsForMerges, boolean openSearcher,
+ public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, long autoCommitMaxSize, boolean indexWriterCloseWaitsForMerges, boolean openSearcher,
int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean commitWithinSoftCommit) {
this.className = className;
this.autoCommmitMaxDocs = autoCommmitMaxDocs;
this.autoCommmitMaxTime = autoCommmitMaxTime;
+ this.autoCommitMaxSizeBytes = autoCommitMaxSize;
this.indexWriterCloseWaitsForMerges = indexWriterCloseWaitsForMerges;
this.openSearcher = openSearcher;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/java/org/apache/solr/update/CommitTracker.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/CommitTracker.java b/solr/core/src/java/org/apache/solr/update/CommitTracker.java
index 6cf7504..8f06d11 100644
--- a/solr/core/src/java/org/apache/solr/update/CommitTracker.java
+++ b/solr/core/src/java/org/apache/solr/update/CommitTracker.java
@@ -49,10 +49,13 @@ public final class CommitTracker implements Runnable {
// scheduler delay for maxDoc-triggered autocommits
public static final int DOC_COMMIT_DELAY_MS = 1;
+ // scheduler delay for maxSize-triggered autocommits
+ public static final int SIZE_COMMIT_DELAY_MS = 1;
// settings, not final so we can change them in testing
private int docsUpperBound;
private long timeUpperBound;
+ private long tLogFileSizeUpperBound;
private final ScheduledExecutorService scheduler =
Executors.newScheduledThreadPool(1, new DefaultSolrThreadFactory("commitScheduler"));
@@ -70,13 +73,15 @@ public final class CommitTracker implements Runnable {
private String name;
- public CommitTracker(String name, SolrCore core, int docsUpperBound, int timeUpperBound, boolean openSearcher, boolean softCommit) {
+ public CommitTracker(String name, SolrCore core, int docsUpperBound, int timeUpperBound, long tLogFileSizeUpperBound,
+ boolean openSearcher, boolean softCommit) {
this.core = core;
this.name = name;
pending = null;
this.docsUpperBound = docsUpperBound;
this.timeUpperBound = timeUpperBound;
+ this.tLogFileSizeUpperBound = tLogFileSizeUpperBound;
this.softCommit = softCommit;
this.openSearcher = openSearcher;
@@ -154,9 +159,34 @@ public final class CommitTracker implements Runnable {
/**
* Indicate that documents have been added
+ * @param commitWithin amount of time (in ms) within which a commit should be scheduled
*/
public void addedDocument(int commitWithin) {
- // maxDocs-triggered autoCommit. Use == instead of > so we only trigger once on the way up
+ addedDocument(commitWithin, -1);
+ }
+
+ /**
+ * Indicate that documents have been added
+ * @param commitWithin amount of time (in ms) within which a commit should be scheduled
+ * @param currentTlogSize current tlog size (in bytes). Use -1 if we don't want to check for a max size triggered commit
+ */
+ public void addedDocument(int commitWithin, long currentTlogSize) {
+ // maxDocs-triggered autoCommit
+ _scheduleMaxDocsTriggeredCommitIfNeeded();
+
+ // maxTime-triggered autoCommit
+ _scheduleCommitWithinIfNeeded(commitWithin);
+
+ // maxSize-triggered autoCommit
+ _scheduleMaxSizeTriggeredCommitIfNeeded(currentTlogSize);
+ }
+
+ /**
+ * If a doc size upper bound is set, and the current number of documents has exceeded it, then
+ * schedule a commit and reset the counter
+ */
+ private void _scheduleMaxDocsTriggeredCommitIfNeeded() {
+ // Use == instead of > so we only trigger once on the way up
if (docsUpperBound > 0) {
long docs = docsSinceCommit.incrementAndGet();
if (docs == docsUpperBound + 1) {
@@ -165,9 +195,6 @@ public final class CommitTracker implements Runnable {
_scheduleCommitWithin(DOC_COMMIT_DELAY_MS);
}
}
-
- // maxTime-triggered autoCommit
- _scheduleCommitWithinIfNeeded(commitWithin);
}
/**
@@ -176,6 +203,26 @@ public final class CommitTracker implements Runnable {
public void deletedDocument( int commitWithin ) {
_scheduleCommitWithinIfNeeded(commitWithin);
}
+
+ /**
+ * If the given current tlog size is greater than the file size upper bound, then schedule a commit
+ * @param currentTlogSize current tlog size (in bytes)
+ */
+ public void scheduleMaxSizeTriggeredCommitIfNeeded(long currentTlogSize) {
+ _scheduleMaxSizeTriggeredCommitIfNeeded(currentTlogSize);
+ }
+
+ /**
+ * If the given current tlog size is greater than the file size upper bound, then schedule a commit
+ * @param currentTlogSize current tlog size (in bytes)
+ */
+ private void _scheduleMaxSizeTriggeredCommitIfNeeded(long currentTlogSize) {
+ if (tLogFileSizeUpperBound > 0 && currentTlogSize > tLogFileSizeUpperBound) {
+ docsSinceCommit.set(0);
+ _scheduleCommitWithin(SIZE_COMMIT_DELAY_MS);
+ }
+ }
+
/** Inform tracker that a commit has occurred */
public void didCommit() {
@@ -254,6 +301,10 @@ public final class CommitTracker implements Runnable {
return docsUpperBound;
}
+ long getTLogFileSizeUpperBound() {
+ return tLogFileSizeUpperBound;
+ }
+
void setDocsUpperBound(int docsUpperBound) {
this.docsUpperBound = docsUpperBound;
}
@@ -262,6 +313,11 @@ public final class CommitTracker implements Runnable {
public void setTimeUpperBound(long timeUpperBound) {
this.timeUpperBound = timeUpperBound;
}
+
+ // only for testing - not thread safe
+ public void setTLogFileSizeUpperBound(int sizeUpperBound) {
+ this.tLogFileSizeUpperBound = sizeUpperBound;
+ }
// only for testing - not thread safe
public void setOpenSearcher(boolean openSearcher) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
index 922419c..db12dc3 100644
--- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
+++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
@@ -73,6 +73,9 @@ import org.slf4j.LoggerFactory;
* directly to the main Lucene index as opposed to adding to a separate smaller index.
*/
public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState.IndexWriterCloser, SolrMetricProducer {
+
+ private static final int NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER = -1;
+
protected final SolrCoreState solrCoreState;
// stats
@@ -118,13 +121,14 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
UpdateHandlerInfo updateHandlerInfo = core.getSolrConfig()
.getUpdateHandlerInfo();
- int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs; // getInt("updateHandler/autoCommit/maxDocs", -1);
- int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime; // getInt("updateHandler/autoCommit/maxTime", -1);
- commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, updateHandlerInfo.openSearcher, false);
+ int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs;
+ int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime;
+ long fileSizeUpperBound = updateHandlerInfo.autoCommitMaxSizeBytes;
+ commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, fileSizeUpperBound, updateHandlerInfo.openSearcher, false);
- int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1);
- int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1);
- softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true);
+ int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs;
+ int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime;
+ softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER, true, true);
commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
indexWriterCloseWaitsForMerges = updateHandlerInfo.indexWriterCloseWaitsForMerges;
@@ -143,13 +147,14 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
UpdateHandlerInfo updateHandlerInfo = core.getSolrConfig()
.getUpdateHandlerInfo();
- int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs; // getInt("updateHandler/autoCommit/maxDocs", -1);
- int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime; // getInt("updateHandler/autoCommit/maxTime", -1);
- commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, updateHandlerInfo.openSearcher, false);
+ int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs;
+ int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime;
+ long fileSizeUpperBound = updateHandlerInfo.autoCommitMaxSizeBytes;
+ commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, fileSizeUpperBound, updateHandlerInfo.openSearcher, false);
- int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1);
- int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1);
- softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, updateHandlerInfo.openSearcher, true);
+ int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs;
+ int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime;
+ softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER, updateHandlerInfo.openSearcher, true);
commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
indexWriterCloseWaitsForMerges = updateHandlerInfo.indexWriterCloseWaitsForMerges;
@@ -178,6 +183,10 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
manager.registerGauge(this, registryName, () -> "" + commitTracker.getTimeUpperBound() + "ms", tag, true, "autoCommitMaxTime",
getCategory().toString(), scope);
}
+ if (commitTracker.getTLogFileSizeUpperBound() > 0) {
+ manager.registerGauge(this, registryName, () -> commitTracker.getTLogFileSizeUpperBound(), tag, true, "autoCommitMaxSize",
+ getCategory().toString(), scope);
+ }
if (softCommitTracker.getDocsUpperBound() > 0) {
manager.registerGauge(this, registryName, () -> softCommitTracker.getDocsUpperBound(), tag, true, "softAutoCommitMaxDocs",
getCategory().toString(), scope);
@@ -279,12 +288,13 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
}
if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
+ long currentTlogSize = getCurrentTLogSize();
if (commitWithinSoftCommit) {
- commitTracker.addedDocument(-1);
+ commitTracker.addedDocument(-1, currentTlogSize);
softCommitTracker.addedDocument(cmd.commitWithin);
} else {
softCommitTracker.addedDocument(-1);
- commitTracker.addedDocument(cmd.commitWithin);
+ commitTracker.addedDocument(cmd.commitWithin, currentTlogSize);
}
}
@@ -418,6 +428,9 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
if (commitTracker.getTimeUpperBound() > 0) {
commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());
}
+
+ long currentTlogSize = getCurrentTLogSize();
+ commitTracker.scheduleMaxSizeTriggeredCommitIfNeeded(currentTlogSize);
if (softCommitTracker.getTimeUpperBound() > 0) {
softCommitTracker.scheduleCommitWithin(softCommitTracker
@@ -990,6 +1003,10 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
return solrCoreState;
}
+ private long getCurrentTLogSize() {
+ return ulog != null && ulog.hasUncommittedChanges() ? ulog.getCurrentLogSizeFromStream() : -1;
+ }
+
// allow access for tests
public CommitTracker getCommitTracker() {
return commitTracker;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/java/org/apache/solr/update/TransactionLog.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/TransactionLog.java b/solr/core/src/java/org/apache/solr/update/TransactionLog.java
index be4dabc..96a928c 100644
--- a/solr/core/src/java/org/apache/solr/update/TransactionLog.java
+++ b/solr/core/src/java/org/apache/solr/update/TransactionLog.java
@@ -630,6 +630,13 @@ public class TransactionLog implements Closeable {
return 0;
}
+ /**
+ * @return the FastOutputStream size
+ */
+ public synchronized long getLogSizeFromStream() {
+ return fos.size();
+ }
+
/** Returns a reader that can be used while a log is still in use.
* Currently only *one* LogReader may be outstanding, and that log may only
* be used from a single thread. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/java/org/apache/solr/update/UpdateLog.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
index fbdf616..09ff146 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@ -292,6 +292,13 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
return size;
}
+ /**
+ * @return the current transaction log's size (based on its output stream)
+ */
+ public long getCurrentLogSizeFromStream() {
+ return tlog.getLogSizeFromStream();
+ }
+
public long getTotalLogsNumber() {
synchronized (this) {
return logs.size();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/test-files/solr/collection1/conf/bad-solrconfig-no-autocommit-tag.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-solrconfig-no-autocommit-tag.xml b/solr/core/src/test-files/solr/collection1/conf/bad-solrconfig-no-autocommit-tag.xml
new file mode 100644
index 0000000..1040db6
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/bad-solrconfig-no-autocommit-tag.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with basic autoCommit settings, but without a valid "autoCommit" tag, to test
+ autoCommit-related defaults -->
+
+<config>
+
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <!-- autocommit pending docs if certain criteria are met -->
+ <!-- <autoCommit> -->
+ <maxSize>5k</maxSize>
+ <!-- </autoCommit> -->
+
+ <updateLog class="${solr.ulog:solr.UpdateLog}"></updateLog>
+ </updateHandler>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ <str name="df">text</str>
+ </lst>
+
+ </requestHandler>
+</config>
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
index c8884c6..989023d 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
@@ -54,6 +54,10 @@
</peerSync>
<updateHandler class="solr.DirectUpdateHandler2">
+ <!-- autocommit pending docs if certain criteria are met -->
+ <autoCommit>
+ <maxSize>${solr.autoCommit.maxSize:}</maxSize>
+ </autoCommit>
<updateLog class="${solr.tests.ulog:solr.UpdateLog}">
<str name="dir">${solr.ulog.dir:}</str>
<str name="maxNumLogsToKeep">${solr.ulog.maxNumLogsToKeep:10}</str>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/test/org/apache/solr/core/TestConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestConfig.java b/solr/core/src/test/org/apache/solr/core/TestConfig.java
index 5a7b706..aa43c25 100644
--- a/solr/core/src/test/org/apache/solr/core/TestConfig.java
+++ b/solr/core/src/test/org/apache/solr/core/TestConfig.java
@@ -30,6 +30,7 @@ import org.apache.solr.handler.admin.ShowFileRequestHandler;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
import org.apache.solr.update.SolrIndexConfig;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.w3c.dom.Node;
@@ -181,6 +182,41 @@ public class TestConfig extends SolrTestCaseJ4 {
assertEquals("numDefaultsTested vs. numDefaultsMapped+numNullDefaults ="+sic.toMap(new LinkedHashMap<>()).keySet(), numDefaultsTested, numDefaultsMapped+numNullDefaults);
}
+ @Test
+ public void testConvertAutoCommitMaxSizeStringToBytes() {
+
+ // Valid values
+ Assert.assertEquals(300, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300"));
+ Assert.assertEquals(307200, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300k"));
+ Assert.assertEquals(307200, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300K"));
+ Assert.assertEquals(314572800, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300m"));
+ Assert.assertEquals(314572800, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300M"));
+ Assert.assertEquals(322122547200L, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300g"));
+ Assert.assertEquals(322122547200L, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300G"));
+ Assert.assertEquals(-1, SolrConfig.convertHeapOptionStyleConfigStringToBytes(""));
+
+ // Invalid values
+ try {
+ SolrConfig.convertHeapOptionStyleConfigStringToBytes("3jbk32k"); // valid suffix but non-numeric prefix
+ Assert.fail();
+ } catch (RuntimeException e) {
+ Assert.assertTrue(e.getMessage().contains("Invalid"));
+ }
+ try {
+ SolrConfig.convertHeapOptionStyleConfigStringToBytes("300x"); // valid prefix but invalid suffix
+ Assert.fail();
+ } catch (RuntimeException e) {
+ Assert.assertTrue(e.getMessage().contains("Invalid"));
+ }
+ }
+
+ @Test
+ public void testMaxSizeSettingWithoutAutoCommit() throws Exception {
+ SolrConfig solrConfig = new SolrConfig(new SolrResourceLoader(TEST_PATH().resolve("collection1")), "bad-solrconfig-no-autocommit-tag.xml", null);
+ Assert.assertEquals(-1, solrConfig.getUpdateHandlerInfo().autoCommitMaxSizeBytes);
+ Assert.assertEquals(-1, solrConfig.getUpdateHandlerInfo().autoCommmitMaxDocs);
+ Assert.assertEquals(-1, solrConfig.getUpdateHandlerInfo().autoCommmitMaxTime);
+ }
// sanity check that sys propertis are working as expected
public void testSanityCheckTestSysPropsAreUsed() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java b/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java
new file mode 100644
index 0000000..0db3616
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.UpdateRequestHandler;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.response.SolrQueryResponse;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MaxSizeAutoCommitTest extends SolrTestCaseJ4 {
+
+ // Given an ID, returns an XML string for an "add document" request
+ private static final Function<Integer, String> ADD_DOC_FN = (id) -> adoc("id", Integer.toString(id));
+ // Given an ID, returns an XML string for a "delete document" request
+ private static final Function<Integer, String> DELETE_DOC_FN = (id) -> delI(Integer.toString(id));
+
+ private ObjectMapper objectMapper; // for JSON parsing
+ private SolrCore core;
+ private DirectUpdateHandler2 updateHandler;
+ private CommitTracker hardCommitTracker;
+ private UpdateRequestHandler updateRequestHandler;
+ private String tlogDirPath;
+
+ @Before
+ public void setup() throws Exception {
+ objectMapper = new ObjectMapper();
+ System.setProperty("solr.autoCommit.maxSize", "5k");
+ System.setProperty("solr.ulog", "solr.UpdateLog");
+ initCore("solrconfig-tlog.xml", "schema.xml");
+ core = h.getCore();
+ updateHandler = (DirectUpdateHandler2) core.getUpdateHandler();
+ hardCommitTracker = updateHandler.commitTracker;
+ // Only testing file-size based auto hard commits - disable other checks
+ hardCommitTracker.setTimeUpperBound(-1);
+ hardCommitTracker.setDocsUpperBound(-1);
+ updateRequestHandler = new UpdateRequestHandler();
+ updateRequestHandler.init( null );
+ tlogDirPath = core.getDataDir() + "/tlog";
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ super.tearDown();
+ System.clearProperty("solr.autoCommit.maxSize");
+ System.clearProperty("solr.ulog");
+ deleteCore();
+ }
+
+ @Test
+ public void simpleTest() throws Exception {
+ int maxFileSizeBound = 1000;
+ int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.25);
+ // Set max size bound
+ hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound);
+
+ // Adding these docs will place the tlog size just under the threshold
+ int numDocs = 27;
+ int batchSize = 3;
+ int numBatches = numDocs / batchSize;
+ SolrQueryResponse updateResp = new SolrQueryResponse();
+ int numTlogs = -1;
+ TreeMap<String, Long> tlogsInfo = null;
+
+ for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) {
+ int docStartId = batchSize * batchCounter;
+
+ // Send batch update request
+ updateRequestHandler.handleRequest(constructBatchAddDocRequest(docStartId, batchSize), updateResp);
+
+ // The sleep is to allow existing commits to finish (or at least mostly finish) before querying/submitting more documents
+ waitForCommit(200);
+
+ // There should just be 1 tlog and its size should be within the (buffered) file size bound
+ tlogsInfo = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer);
+ numTlogs = parseTotalNumTlogs(tlogsInfo);
+ Assert.assertEquals(1, numTlogs);
+ }
+
+ // Now that the core's tlog size is just under the threshold, one more update should induce a commit
+ int docStartId = batchSize * numBatches;
+ updateRequestHandler.handleRequest(constructBatchAddDocRequest(docStartId, batchSize), updateResp);
+ waitForCommit(200);
+
+ // Verify that a commit happened. There should now be 2 tlogs, both of which are < maxFileSizeBound.
+ TreeMap<String, Long> tlogsInfoPostCommit = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer);
+ Assert.assertEquals(2, parseTotalNumTlogs(tlogsInfoPostCommit));
+
+ // And the current tlog's size should be less than the previous tlog's size
+ Assert.assertTrue(tlogsInfoPostCommit.lastEntry().getValue() < tlogsInfo.lastEntry().getValue());
+ }
+
+ @Test
+ public void testRedundantDeletes() throws Exception {
+ int maxFileSizeBound = 1000;
+ int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.25);
+
+ // Set max size bound
+ hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound);
+
+ // Add docs
+ int numDocsToAdd = 150;
+ SolrQueryResponse updateResp = new SolrQueryResponse();
+ updateRequestHandler.handleRequest(constructBatchAddDocRequest(0, numDocsToAdd), updateResp);
+ waitForCommit(200);
+
+ // Get the tlog file info
+ TreeMap<String, Long> tlogsInfoPreDeletes = getTlogFileSizes(tlogDirPath);
+
+ // Send a bunch of redundant deletes
+ int numDeletesToSend = 5000;
+ int docIdToDelete = 100;
+
+ SolrQueryRequestBase requestWithOneDelete = new SolrQueryRequestBase(core, new MapSolrParams(new HashMap<String, String>())) {};
+ List<String> docs = new ArrayList<>();
+ docs.add(delI(Integer.toString(docIdToDelete)));
+
+ requestWithOneDelete.setContentStreams(toContentStreams(docs));
+
+ for (int i = 0; i < numDeletesToSend; i++) {
+ if (i % 50 == 0) {
+ // Wait periodically to allow existing commits to finish before
+ // sending more delete requests
+ waitForCommit(200);
+ }
+ updateRequestHandler.handleRequest(requestWithOneDelete, updateResp);
+ }
+
+ // Verify that new tlogs have been created, and that their sizes are as expected
+ TreeMap<String, Long> tlogsInfoPostDeletes = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer);
+ Assert.assertTrue(parseTotalNumTlogs(tlogsInfoPreDeletes) < parseTotalNumTlogs(tlogsInfoPostDeletes));
+ }
+
+ @Test
+ public void deleteTest() throws Exception {
+ int maxFileSizeBound = 1000;
+ int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.25);
+
+ // Set max size bound
+ hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound);
+
+ // Add docs
+ int numDocsToAdd = 150;
+ SolrQueryResponse updateResp = new SolrQueryResponse();
+ updateRequestHandler.handleRequest(constructBatchAddDocRequest(0, numDocsToAdd), updateResp);
+ waitForCommit(200);
+
+ // Get the tlog file info
+ TreeMap<String, Long> tlogsInfoPreDeletes = getTlogFileSizes(tlogDirPath);
+
+ // Delete documents (in batches, so we can allow commits to finish and new tlog files to be created)
+ int batchSize = 15;
+ int numBatches = numDocsToAdd / batchSize;
+ for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) {
+ int docStartId = batchSize * batchCounter;
+
+ // Send batch delete doc request
+ updateRequestHandler.handleRequest(constructBatchDeleteDocRequest(docStartId, batchSize), updateResp);
+
+ // The sleep is to allow existing commits to finish before deleting more documents
+ waitForCommit(200);
+ }
+
+ // Verify that the commit happened by seeing if a new tlog file was opened
+ TreeMap<String, Long> tlogsInfoPostDeletes = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer);
+ Assert.assertTrue(parseTotalNumTlogs(tlogsInfoPreDeletes) < parseTotalNumTlogs(tlogsInfoPostDeletes));
+ }
+
+ @Test
+ @Repeat(iterations = 5)
+ public void endToEndTest() throws Exception {
+ int maxFileSizeBound = 5000;
+ // Set max size bound
+ hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound);
+
+ // Giving a 10% buffer for the max size bound
+ int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.1);
+
+ SolrQueryRequest selectQuery = req("*:*");
+ List<Integer> docCounts = new ArrayList<>();
+
+ int numDocs = 1000;
+ int batchSize = 20;
+ int numBatches = numDocs / batchSize;
+ for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) {
+ SolrQueryResponse updateResp = new SolrQueryResponse();
+ int docStartId = batchSize * batchCounter;
+
+ // Send batch add doc request
+ updateRequestHandler.handleRequest(constructBatchAddDocRequest(docStartId, batchSize), updateResp);
+
+ // The sleep is to allow existing commits to finish before querying/submitting more documents
+ waitForCommit(200);
+
+ // Check tlog file sizes
+ getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer);
+
+ // See how many documents are currently visible. This should increase as more commits occur.
+ docCounts.add(queryCore(selectQuery));
+ }
+
+ // One final commit, after which all documents should be visible
+ CommitUpdateCommand commitUpdateCommand = new CommitUpdateCommand(req(), false);
+ updateHandler.commit(commitUpdateCommand);
+ waitForCommit(200);
+ docCounts.add(queryCore(selectQuery));
+
+ // Evaluate the document counts
+ checkNumFoundDocuments(docCounts, numDocs);
+ }
+
+ /**
+ * Sleeps in increments of 50 ms while checking to see if a commit completed. If it did, then return. If not, continue
+ * this cycle for at most the amount of time specified
+ * @param maxTotalWaitTimeMillis the max amount of time (in ms) to wait/check for a commit
+ */
+ private void waitForCommit(long maxTotalWaitTimeMillis) throws Exception {
+ long startTimeNanos = System.nanoTime();
+ long maxTotalWaitTimeNanos = TimeUnit.MILLISECONDS.toNanos(maxTotalWaitTimeMillis);
+ while (System.nanoTime() - startTimeNanos < maxTotalWaitTimeNanos) {
+ Thread.sleep(50);
+ if (!updateHandler.getUpdateLog().hasUncommittedChanges()) {
+ return;
+ }
+ }
+ }
+
+ /**
+ * Returns the total number of tlogs that have been created for the core.
+ *
+ * The tlogs in a core's tlog directory are named: tlog.0000000000000000000, tlog.0000000000000000001, tlog.0000000000000000002, etc.
+ * Because old tlogs are periodically deleted, we can't just count the number of existing files. Instead, we take the
+ * highest ordering tlog file name (which would be the newest) and parse the extension.
+ *
+ * e.g if the most recently created tlog file is tlog.0000000000000000003, we know that this core has had 4 tlogs.
+ *
+ * @param tlogsInfo TreeMap of (tlog file name, tlog file size (in bytes)) pairs
+ * @return total number of tlogs created for this core
+ */
+ private int parseTotalNumTlogs(TreeMap<String, Long> tlogsInfo) {
+ String mostRecentFileName = tlogsInfo.lastKey();
+ int extensionDelimiterIndex = mostRecentFileName.lastIndexOf(".");
+ if (extensionDelimiterIndex == -1) {
+ throw new RuntimeException("Invalid tlog filename: " + mostRecentFileName);
+ }
+ String extension = mostRecentFileName.substring(extensionDelimiterIndex + 1);
+ try {
+ return Integer.parseInt(extension) + 1;
+ } catch (NumberFormatException e) {
+ throw new RuntimeException("Could not parse tlog filename: " + mostRecentFileName, e);
+ }
+ }
+
+ /**
+ * Construct a batch add document request with a series of very simple Solr docs with increasing IDs.
+ * @param startId the document ID to begin with
+ * @param batchSize the number of documents to include in the batch
+ * @return a SolrQueryRequestBase
+ */
+ private SolrQueryRequestBase constructBatchAddDocRequest(int startId, int batchSize) {
+ return constructBatchRequestHelper(startId, batchSize, ADD_DOC_FN);
+ }
+
+ /**
+ * Construct a batch delete document request, with IDs incrementing from startId
+ * @param startId the document ID to begin with
+ * @param batchSize the number of documents to include in the batch
+ * @return a SolrQueryRequestBase
+ */
+ private SolrQueryRequestBase constructBatchDeleteDocRequest(int startId, int batchSize) {
+ return constructBatchRequestHelper(startId, batchSize, DELETE_DOC_FN);
+ }
+
+ /**
+ * Helper for constructing a batch update request
+ * @param startId the document ID to begin with
+ * @param batchSize the number of documents to include in the batch
+ * @param requestFn a function that takes an (int) ID and returns an XML string of the request to add to the batch request
+ * @return a SolrQueryRequestBase
+ */
+ private SolrQueryRequestBase constructBatchRequestHelper(int startId, int batchSize, Function<Integer, String> requestFn) {
+ SolrQueryRequestBase updateReq = new SolrQueryRequestBase(core, new MapSolrParams(new HashMap<>())) {};
+ List<String> docs = new ArrayList<>();
+ for (int i = startId; i < startId + batchSize; i++) {
+ docs.add(requestFn.apply(i));
+ }
+ updateReq.setContentStreams(toContentStreams(docs));
+ return updateReq;
+ }
+
+ /**
+ * Executes the given query
+ * @param query the query to execute
+ * @return the number of documents found
+ */
+ public int queryCore(SolrQueryRequest query) throws Exception {
+ String responseStr = h.query(query);
+ try {
+ Map<String, Object> root = (Map<String, Object>) objectMapper.readValue(responseStr, Object.class);
+ Map<String, Object> rootResponse = (Map<String, Object>) root.get("response");
+ return (int) rootResponse.get("numFound");
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to parse Solr query response", e);
+ }
+ }
+
+ /**
+ * Checks the given list of document counts to make sure that they are increasing (as commits occur).
+ * @param numDocList list of the number of documents found in a given core. Ascending from oldest to newest
+ */
+ private void checkNumFoundDocuments(List<Integer> numDocList, int finalValue) {
+ long currentTotal = 0;
+ for (Integer count : numDocList) {
+ Assert.assertTrue(count >= currentTotal);
+ currentTotal = count;
+ }
+ Assert.assertEquals(finalValue, numDocList.get(numDocList.size() - 1).intValue());
+ }
+
+
+ /**
+ * Goes through the given tlog directory and inspects each tlog.
+ * @param tlogDirPath tlog directory path
+ * @return a TreeMap of (tlog file name, tlog file size (in bytes)) pairs
+ */
+ private TreeMap<String, Long> getTlogFileSizes(String tlogDirPath) {
+ return getTlogFileSizes(tlogDirPath, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Goes through the given tlog directory and inspects each tlog. Asserts that each tlog's size is <= the given max size bound.
+ * @param tlogDirPath tlog directory path
+ * @param maxSizeBound the max tlog size
+ * @return a TreeMap of (tlog file name, tlog file size (in bytes)) pairs
+ */
+ private TreeMap<String, Long> getTlogFileSizes(String tlogDirPath, int maxSizeBound) {
+ File tlogDir = new File(tlogDirPath);
+ File[] tlogs = tlogDir.listFiles();
+ TreeMap<String, Long> tlogInfo = new TreeMap<>();
+ if (tlogs != null) {
+ for (File tlog : tlogs) {
+ String message = String.format(Locale.getDefault(), "Tlog size exceeds the max size bound. Tlog path: %s, tlog size: %d",
+ tlog.getPath(), tlog.length());
+ Assert.assertTrue(message, tlog.length() <= maxSizeBound);
+ tlogInfo.put(tlog.getName(), tlog.length());
+ }
+ }
+ return tlogInfo;
+ }
+
+ /**
+ * Convert the given list of strings into a list of streams, for Solr update requests
+ * @param strs strings to convert into streams
+ * @return list of streams
+ */
+ private List<ContentStream> toContentStreams(List<String> strs) {
+ ArrayList<ContentStream> streams = new ArrayList<>();
+ for (String str : strs) {
+ streams.addAll(ClientUtils.toContentStreams(str, "text/xml"));
+ }
+ return streams;
+ }
+}
[19/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8142: Make
postings APIs expose raw impacts rather than scores.
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/SlowImpactsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowImpactsEnum.java b/lucene/core/src/java/org/apache/lucene/index/SlowImpactsEnum.java
index 9ba27e2..6177ea4 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SlowImpactsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SlowImpactsEnum.java
@@ -17,23 +17,45 @@
package org.apache.lucene.index;
import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
/**
* {@link ImpactsEnum} that doesn't index impacts but implements the API in a
- * legal way. This should typically be used for short postings that do not need
+ * legal way. This is typically used for short postings that do not need
* skipping.
*/
public final class SlowImpactsEnum extends ImpactsEnum {
+ private static final Impacts DUMMY_IMPACTS = new Impacts() {
+
+ private final List<Impact> impacts = Collections.singletonList(new Impact(Integer.MAX_VALUE, 1L));
+
+ @Override
+ public int numLevels() {
+ return 1;
+ }
+
+ @Override
+ public int getDocIdUpTo(int level) {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ @Override
+ public List<Impact> getImpacts(int level) {
+ return impacts;
+ }
+
+ };
+
private final PostingsEnum delegate;
- private final float maxScore;
/** Wrap the given {@link PostingsEnum}. */
- public SlowImpactsEnum(PostingsEnum delegate, float maxScore) {
+ public SlowImpactsEnum(PostingsEnum delegate) {
this.delegate = delegate;
- this.maxScore = maxScore;
}
@Override
@@ -82,13 +104,10 @@ public final class SlowImpactsEnum extends ImpactsEnum {
}
@Override
- public int advanceShallow(int target) {
- return NO_MORE_DOCS;
- }
+ public void advanceShallow(int target) {}
@Override
- public float getMaxScore(int maxDoc) {
- return maxScore;
+ public Impacts getImpacts() {
+ return DUMMY_IMPACTS;
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java
index 70d4387..5fe9a0d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -111,7 +110,7 @@ class SortedDocValuesTermsEnum extends TermsEnum {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java
index 9099ac8..bbeb5c2 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java
@@ -17,7 +17,6 @@
package org.apache.lucene.index;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -111,7 +110,7 @@ class SortedSetDocValuesTermsEnum extends TermsEnum {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java
index 983bd9c..e1e48ef 100644
--- a/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
@@ -171,10 +170,10 @@ public abstract class TermsEnum implements BytesRefIterator {
public abstract PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException;
/**
- * Return a {@link ImpactsEnum} that computes impacts with {@code scorer}.
+ * Return a {@link ImpactsEnum}.
* @see #postings(PostingsEnum, int)
*/
- public abstract ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException;
+ public abstract ImpactsEnum impacts(int flags) throws IOException;
/**
* Expert: Returns the TermsEnums internal state to position the TermsEnum
@@ -236,7 +235,7 @@ public abstract class TermsEnum implements BytesRefIterator {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
throw new IllegalStateException("this method should never be called");
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionScorer.java b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionScorer.java
index 070b6c4..bb5d99c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionScorer.java
@@ -38,6 +38,9 @@ final class BlockMaxConjunctionScorer extends Scorer {
BlockMaxConjunctionScorer(Weight weight, Collection<Scorer> scorersList) throws IOException {
super(weight);
this.scorers = scorersList.toArray(new Scorer[scorersList.size()]);
+ for (Scorer scorer : scorers) {
+ scorer.advanceShallow(0);
+ }
this.maxScorePropagator = new MaxScoreSumPropagator(scorersList);
// Put scorers with the higher max scores first
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java b/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
index 72f9473..375d3c2 100644
--- a/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
@@ -23,7 +23,6 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeReflector;
@@ -275,8 +274,8 @@ public final class FuzzyTermsEnum extends TermsEnum {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return actualEnum.impacts(scorer, flags);
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return actualEnum.impacts(flags);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/search/MaxScoreCache.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreCache.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreCache.java
new file mode 100644
index 0000000..719bc14
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreCache.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.lucene.index.Impact;
+import org.apache.lucene.index.Impacts;
+import org.apache.lucene.index.ImpactsEnum;
+import org.apache.lucene.search.similarities.Similarity.SimScorer;
+import org.apache.lucene.util.ArrayUtil;
+
+/**
+ * Compute maximum scores based on {@link Impacts} and keep them in a cache in
+ * order not to run expensive similarity score computations multiple times on
+ * the same data.
+ */
+public final class MaxScoreCache {
+
+ private final ImpactsEnum impactsEnum;
+ private final SimScorer scorer;
+ private final float globalMaxScore;
+ private float[] maxScoreCache;
+ private int[] maxScoreCacheUpTo;
+
+ /**
+ * Sole constructor.
+ */
+ public MaxScoreCache(ImpactsEnum impactsEnum, SimScorer scorer) {
+ this.impactsEnum = impactsEnum;
+ this.scorer = scorer;
+ globalMaxScore = scorer.score(Integer.MAX_VALUE, 1L);
+ maxScoreCache = new float[0];
+ maxScoreCacheUpTo = new int[0];
+ }
+
+ private void ensureCacheSize(int size) {
+ if (maxScoreCache.length < size) {
+ int oldLength = maxScoreCache.length;
+ maxScoreCache = ArrayUtil.grow(maxScoreCache, size);
+ maxScoreCacheUpTo = Arrays.copyOf(maxScoreCacheUpTo, maxScoreCache.length);
+ Arrays.fill(maxScoreCacheUpTo, oldLength, maxScoreCacheUpTo.length, -1);
+ }
+ }
+
+ private float computeMaxScore(List<Impact> impacts) {
+ float maxScore = 0;
+ for (Impact impact : impacts) {
+ maxScore = Math.max(scorer.score(impact.freq, impact.norm), maxScore);
+ }
+ return maxScore;
+ }
+
+ /**
+ * Return the first level that includes all doc IDs up to {@code upTo},
+ * or -1 if there is no such level.
+ */
+ private int getLevel(int upTo) throws IOException {
+ final Impacts impacts = impactsEnum.getImpacts();
+ for (int level = 0, numLevels = impacts.numLevels(); level < numLevels; ++level) {
+ final int impactsUpTo = impacts.getDocIdUpTo(level);
+ if (upTo <= impactsUpTo) {
+ return level;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Return the maximum score for the given {@code level}.
+ */
+ float getMaxScoreForLevel(int level) throws IOException {
+ final Impacts impacts = impactsEnum.getImpacts();
+ ensureCacheSize(level + 1);
+ final int levelUpTo = impacts.getDocIdUpTo(level);
+ if (maxScoreCacheUpTo[level] < levelUpTo) {
+ maxScoreCache[level] = computeMaxScore(impacts.getImpacts(level));
+ maxScoreCacheUpTo[level] = levelUpTo;
+ }
+ return maxScoreCache[level];
+ }
+
+ /**
+ * Return the maximum level at which scores are all less than {@code minScore},
+ * or -1 if none.
+ */
+ int getSkipLevel(float minScore) throws IOException {
+ final Impacts impacts = impactsEnum.getImpacts();
+ final int numLevels = impacts.numLevels();
+ for (int level = 0; level < numLevels; ++level) {
+ if (getMaxScoreForLevel(level) >= minScore) {
+ return level - 1;
+ }
+ }
+ return numLevels - 1;
+ }
+
+ /**
+ * Implement the contract of {@link Scorer#advanceShallow(int)} based on the
+ * wrapped {@link ImpactsEnum}.
+ * @see Scorer#advanceShallow(int)
+ */
+ public int advanceShallow(int target) throws IOException {
+ impactsEnum.advanceShallow(target);
+ Impacts impacts = impactsEnum.getImpacts();
+ return impacts.getDocIdUpTo(0);
+ }
+
+ /**
+ * Implement the contract of {@link Scorer#getMaxScore(int)} based on the
+ * wrapped {@link ImpactsEnum} and {@link Scorer}.
+ * @see Scorer#getMaxScore(int)
+ */
+ public float getMaxScore(int upTo) throws IOException {
+ final int level = getLevel(upTo);
+ if (level == -1) {
+ return globalMaxScore;
+ } else {
+ return getMaxScoreForLevel(level);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/search/Scorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/Scorer.java b/lucene/core/src/java/org/apache/lucene/search/Scorer.java
index 81624cc..9a48840 100644
--- a/lucene/core/src/java/org/apache/lucene/search/Scorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/Scorer.java
@@ -161,8 +161,8 @@ public abstract class Scorer {
* Advance to the block of documents that contains {@code target} in order to
* get scoring information about this block. This method is implicitly called
* by {@link DocIdSetIterator#advance(int)} and
- * {@link DocIdSetIterator#nextDoc()}. Calling this method doesn't modify the
- * current {@link DocIdSetIterator#docID()}.
+ * {@link DocIdSetIterator#nextDoc()} on the returned doc ID. Calling this
+ * method doesn't modify the current {@link DocIdSetIterator#docID()}.
* It returns a number that is greater than or equal to all documents
* contained in the current block, but less than any doc IDS of the next block.
* {@code target} must be >= {@link #docID()} as well as all targets that
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
index d51626f..66cf833 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
+import org.apache.lucene.index.Impacts;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SlowImpactsEnum;
@@ -31,6 +32,7 @@ final class TermScorer extends Scorer {
private final ImpactsEnum impactsEnum;
private final DocIdSetIterator iterator;
private final LeafSimScorer docScorer;
+ private final MaxScoreCache maxScoreCache;
private float minCompetitiveScore;
/**
@@ -47,7 +49,8 @@ final class TermScorer extends Scorer {
super(weight);
this.docScorer = docScorer;
if (scoreMode == ScoreMode.TOP_SCORES) {
- impactsEnum = te.impacts(docScorer.getSimScorer(), PostingsEnum.FREQS);
+ impactsEnum = te.impacts(PostingsEnum.FREQS);
+ maxScoreCache = new MaxScoreCache(impactsEnum, docScorer.getSimScorer());
postingsEnum = impactsEnum;
iterator = new DocIdSetIterator() {
@@ -61,8 +64,10 @@ final class TermScorer extends Scorer {
}
if (target > upTo) {
- upTo = impactsEnum.advanceShallow(target);
- maxScore = impactsEnum.getMaxScore(upTo);
+ impactsEnum.advanceShallow(target);
+ Impacts impacts = impactsEnum.getImpacts();
+ upTo = impacts.getDocIdUpTo(0);
+ maxScore = maxScoreCache.getMaxScoreForLevel(0);
}
while (true) {
@@ -76,10 +81,23 @@ final class TermScorer extends Scorer {
return NO_MORE_DOCS;
}
- target = upTo + 1;
-
- upTo = impactsEnum.advanceShallow(target);
- maxScore = impactsEnum.getMaxScore(upTo);
+ impactsEnum.advanceShallow(upTo + 1);
+ Impacts impacts = impactsEnum.getImpacts();
+ final int level = maxScoreCache.getSkipLevel(minCompetitiveScore);
+ if (level >= 0) {
+ // we can skip more docs
+ int newUpTo = impacts.getDocIdUpTo(level);
+ if (newUpTo == NO_MORE_DOCS) {
+ return NO_MORE_DOCS;
+ }
+ target = newUpTo + 1;
+ impactsEnum.advanceShallow(target);
+ impacts = impactsEnum.getImpacts();
+ } else {
+ target = upTo + 1;
+ }
+ upTo = impacts.getDocIdUpTo(0);
+ maxScore = maxScoreCache.getMaxScoreForLevel(0);
}
}
@@ -105,7 +123,8 @@ final class TermScorer extends Scorer {
};
} else {
postingsEnum = te.postings(null, scoreMode.needsScores() ? PostingsEnum.FREQS : PostingsEnum.NONE);
- impactsEnum = new SlowImpactsEnum(postingsEnum, docScorer.getSimScorer().score(Float.MAX_VALUE, 1));
+ impactsEnum = new SlowImpactsEnum(postingsEnum);
+ maxScoreCache = new MaxScoreCache(impactsEnum, docScorer.getSimScorer());
iterator = postingsEnum;
}
}
@@ -132,12 +151,12 @@ final class TermScorer extends Scorer {
@Override
public int advanceShallow(int target) throws IOException {
- return impactsEnum.advanceShallow(target);
+ return maxScoreCache.advanceShallow(target);
}
@Override
public float getMaxScore(int upTo) throws IOException {
- return impactsEnum.getMaxScore(upTo);
+ return maxScoreCache.getMaxScore(upTo);
}
@Override
@@ -148,4 +167,5 @@ final class TermScorer extends Scorer {
/** Returns a string representation of this <code>TermScorer</code>. */
@Override
public String toString() { return "scorer(" + weight + ")[" + super.toString() + "]"; }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/java/org/apache/lucene/search/WANDScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/WANDScorer.java b/lucene/core/src/java/org/apache/lucene/search/WANDScorer.java
index f7a88f1..82b8044 100644
--- a/lucene/core/src/java/org/apache/lucene/search/WANDScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/WANDScorer.java
@@ -139,6 +139,7 @@ final class WANDScorer extends Scorer {
OptionalInt scalingFactor = OptionalInt.empty();
for (Scorer scorer : scorers) {
+ scorer.advanceShallow(0);
float maxScore = scorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS);
if (maxScore != 0 && Float.isFinite(maxScore)) {
// 0 and +Infty should not impact the scale
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java b/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java
index 5743e64..f4d3e69 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java
@@ -20,85 +20,85 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
-import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator.FreqAndNorm;
+import org.apache.lucene.index.Impact;
import org.apache.lucene.util.LuceneTestCase;
public class TestCompetitiveFreqNormAccumulator extends LuceneTestCase {
public void testBasics() {
- CompetitiveFreqNormAccumulator acc = new CompetitiveFreqNormAccumulator();
- Set<FreqAndNorm> expected = new HashSet<>();
+ CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
+ Set<Impact> expected = new HashSet<>();
acc.add(3, 5);
- expected.add(new FreqAndNorm(3, 5));
+ expected.add(new Impact(3, 5));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
acc.add(6, 11);
- expected.add(new FreqAndNorm(6, 11));
+ expected.add(new Impact(6, 11));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
acc.add(10, 13);
- expected.add(new FreqAndNorm(10, 13));
+ expected.add(new Impact(10, 13));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
acc.add(1, 2);
- expected.add(new FreqAndNorm(1, 2));
+ expected.add(new Impact(1, 2));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
acc.add(7, 9);
- expected.remove(new FreqAndNorm(6, 11));
- expected.add(new FreqAndNorm(7, 9));
+ expected.remove(new Impact(6, 11));
+ expected.add(new Impact(7, 9));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
acc.add(8, 2);
expected.clear();
- expected.add(new FreqAndNorm(10, 13));
- expected.add(new FreqAndNorm(8, 2));
+ expected.add(new Impact(10, 13));
+ expected.add(new Impact(8, 2));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
}
public void testExtremeNorms() {
- CompetitiveFreqNormAccumulator acc = new CompetitiveFreqNormAccumulator();
- Set<FreqAndNorm> expected = new HashSet<>();
+ CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
+ Set<Impact> expected = new HashSet<>();
acc.add(3, 5);
- expected.add(new FreqAndNorm(3, 5));
+ expected.add(new Impact(3, 5));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
acc.add(10, 10000);
- expected.add(new FreqAndNorm(10, 10000));
+ expected.add(new Impact(10, 10000));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
acc.add(5, 200);
- expected.add(new FreqAndNorm(5, 200));
+ expected.add(new Impact(5, 200));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
acc.add(20, -100);
- expected.add(new FreqAndNorm(20, -100));
+ expected.add(new Impact(20, -100));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
acc.add(30, -3);
- expected.add(new FreqAndNorm(30, -3));
+ expected.add(new Impact(30, -3));
assertEquals(expected, acc.getCompetitiveFreqNormPairs());
}
public void testOmitFreqs() {
- CompetitiveFreqNormAccumulator acc = new CompetitiveFreqNormAccumulator();
+ CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
acc.add(1, 5);
acc.add(1, 7);
acc.add(1, 4);
- assertEquals(Collections.singleton(new FreqAndNorm(1, 4)), acc.getCompetitiveFreqNormPairs());
+ assertEquals(Collections.singleton(new Impact(1, 4)), acc.getCompetitiveFreqNormPairs());
}
public void testOmitNorms() {
- CompetitiveFreqNormAccumulator acc = new CompetitiveFreqNormAccumulator();
+ CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
acc.add(5, 1);
acc.add(7, 1);
acc.add(4, 1);
- assertEquals(Collections.singleton(new FreqAndNorm(7, 1)), acc.getCompetitiveFreqNormPairs());
+ assertEquals(Collections.singleton(new Impact(7, 1)), acc.getCompetitiveFreqNormPairs());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat.java
index d507b7b..dec5eca 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat.java
@@ -18,19 +18,23 @@ package org.apache.lucene.codecs.lucene50;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator;
+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.blocktree.FieldReader;
import org.apache.lucene.codecs.blocktree.Stats;
+import org.apache.lucene.codecs.lucene50.Lucene50ScoreSkipReader.MutableImpactList;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.BasePostingsFormatTestCase;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.Impact;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -89,33 +93,43 @@ public class TestBlockPostingsFormat extends BasePostingsFormatTestCase {
public void testImpactSerialization() throws IOException {
// omit norms and omit freqs
- doTestImpactSerialization(new int[] { 1 }, new long[] { 1L });
+ doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));
// omit freqs
- doTestImpactSerialization(new int[] { 1 }, new long[] { 42L });
+ doTestImpactSerialization(Collections.singletonList(new Impact(1, 42L)));
// omit freqs with very large norms
- doTestImpactSerialization(new int[] { 1 }, new long[] { -100L });
+ doTestImpactSerialization(Collections.singletonList(new Impact(1, -100L)));
// omit norms
- doTestImpactSerialization(new int[] { 30 }, new long[] { 1L });
+ doTestImpactSerialization(Collections.singletonList(new Impact(30, 1L)));
// omit norms with large freq
- doTestImpactSerialization(new int[] { 500 }, new long[] { 1L });
+ doTestImpactSerialization(Collections.singletonList(new Impact(500, 1L)));
// freqs and norms, basic
doTestImpactSerialization(
- new int[] { 1, 3, 7, 15, 20, 28 },
- new long[] { 7L, 9L, 10L, 11L, 13L, 14L });
+ Arrays.asList(
+ new Impact(1, 7L),
+ new Impact(3, 9L),
+ new Impact(7, 10L),
+ new Impact(15, 11L),
+ new Impact(20, 13L),
+ new Impact(28, 14L)));
// freqs and norms, high values
doTestImpactSerialization(
- new int[] { 2, 10, 12, 50, 1000, 1005 },
- new long[] { 2L, 10L, 50L, -100L, -80L, -3L });
+ Arrays.asList(
+ new Impact(2, 2L),
+ new Impact(10, 10L),
+ new Impact(12, 50L),
+ new Impact(50, -100L),
+ new Impact(1000, -80L),
+ new Impact(1005, -3L)));
}
- private void doTestImpactSerialization(int[] freqs, long[] norms) throws IOException {
- CompetitiveFreqNormAccumulator acc = new CompetitiveFreqNormAccumulator();
- for (int i = 0; i < freqs.length; ++i) {
- acc.add(freqs[i], norms[i]);
+ private void doTestImpactSerialization(List<Impact> impacts) throws IOException {
+ CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
+ for (Impact impact : impacts) {
+ acc.add(impact.freq, impact.norm);
}
try(Directory dir = newDirectory()) {
try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
@@ -124,17 +138,8 @@ public class TestBlockPostingsFormat extends BasePostingsFormatTestCase {
try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
byte[] b = new byte[Math.toIntExact(in.length())];
in.readBytes(b, 0, b.length);
- Lucene50ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new SimScorer("") {
- int i = 0;
-
- @Override
- public float score(float freq, long norm) {
- assert freq == freqs[i];
- assert norm == norms[i];
- i++;
- return 0;
- }
- });
+ List<Impact> impacts2 = Lucene50ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new MutableImpactList());
+ assertEquals(impacts, impacts2);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
index efe4587..acc6506 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
@@ -34,7 +34,6 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.StringField;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -680,7 +679,7 @@ public class TestCodecs extends LuceneTestCase {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index 16cc961..ff248c3 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -42,7 +42,6 @@ import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
@@ -1429,8 +1428,8 @@ public class MemoryIndex {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return new SlowImpactsEnum(postings(null, flags), scorer.score(Float.MAX_VALUE, 1L));
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return new SlowImpactsEnum(postings(null, flags));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
index 4203e07..e7f4c4c 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
@@ -21,11 +21,10 @@ import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.ImpactsEnum;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput;
@@ -90,7 +89,7 @@ final class IDVersionPostingsReader extends PostingsReaderBase {
}
@Override
- public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags) throws IOException {
throw new UnsupportedOperationException("Should never be called, IDVersionSegmentTermsEnum implements impacts directly");
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java
index d5f51e0..b5e96ee 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java
@@ -25,7 +25,6 @@ import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SlowImpactsEnum;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
@@ -1009,10 +1008,10 @@ public final class IDVersionSegmentTermsEnum extends TermsEnum {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
// Only one posting, the slow impl is fine
// We could make this throw UOE but then CheckIndex is angry
- return new SlowImpactsEnum(postings(null, flags), scorer.score(Float.MAX_VALUE, 1));
+ return new SlowImpactsEnum(postings(null, flags));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
index 37c078f..8a2b042 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
@@ -45,7 +45,6 @@ import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SlowImpactsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Accountable;
@@ -477,8 +476,8 @@ public final class RAMOnlyPostingsFormat extends PostingsFormat {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return new SlowImpactsEnum(postings(null, PostingsEnum.FREQS), scorer.score(Float.MAX_VALUE, 1));
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return new SlowImpactsEnum(postings(null, PostingsEnum.FREQS));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
index dfec1db..46beacb 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java
@@ -18,12 +18,12 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Iterator;
+import java.util.List;
import java.util.Objects;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.StringHelper;
@@ -211,12 +211,12 @@ public class AssertingLeafReader extends FilterLeafReader {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
assertThread("Terms enums", creationThread);
assert state == State.POSITIONED: "docs(...) called on unpositioned TermsEnum";
assert (flags & PostingsEnum.FREQS) != 0 : "Freqs should be requested on impacts";
- return new AssertingImpactsEnum(super.impacts(scorer, flags));
+ return new AssertingImpactsEnum(super.impacts(flags));
}
// TODO: we should separately track if we are 'at the end' ?
@@ -454,7 +454,7 @@ public class AssertingLeafReader extends FilterLeafReader {
private final AssertingPostingsEnum assertingPostings;
private final ImpactsEnum in;
- private int lastShallowTarget;
+ private int lastShallowTarget = -1;
AssertingImpactsEnum(ImpactsEnum impacts) {
in = impacts;
@@ -463,20 +463,19 @@ public class AssertingLeafReader extends FilterLeafReader {
}
@Override
- public int advanceShallow(int target) throws IOException {
+ public void advanceShallow(int target) throws IOException {
assert target >= lastShallowTarget : "called on decreasing targets: target = " + target + " < last target = " + lastShallowTarget;
assert target >= docID() : "target = " + target + " < docID = " + docID();
- int upTo = in.advanceShallow(target);
- assert upTo >= target : "upTo = " + upTo + " < target = " + target;
lastShallowTarget = target;
- return upTo;
+ in.advanceShallow(target);
}
@Override
- public float getMaxScore(int upTo) throws IOException {
- assert upTo >= lastShallowTarget : "uTo = " + upTo + " < last shallow target = " + lastShallowTarget;
- float maxScore = in.getMaxScore(upTo);
- return maxScore;
+ public Impacts getImpacts() throws IOException {
+ assert docID() >= 0 || lastShallowTarget >= 0 : "Cannot get impacts until the iterator is positioned or advanceShallow has been called";
+ Impacts impacts = in.getImpacts();
+ CheckIndex.checkImpacts(impacts, Math.max(docID(), lastShallowTarget));
+ return new AssertingImpacts(impacts, this);
}
@Override
@@ -527,6 +526,38 @@ public class AssertingLeafReader extends FilterLeafReader {
}
}
+ static class AssertingImpacts extends Impacts {
+
+ private final Impacts in;
+ private final AssertingImpactsEnum impactsEnum;
+ private final int validFor;
+
+ AssertingImpacts(Impacts in, AssertingImpactsEnum impactsEnum) {
+ this.in = in;
+ this.impactsEnum = impactsEnum;
+ validFor = Math.max(impactsEnum.docID(), impactsEnum.lastShallowTarget);
+ }
+
+ @Override
+ public int numLevels() {
+ assert validFor == Math.max(impactsEnum.docID(), impactsEnum.lastShallowTarget) : "Cannot reuse impacts after advancing the iterator";
+ return in.numLevels();
+ }
+
+ @Override
+ public int getDocIdUpTo(int level) {
+ assert validFor == Math.max(impactsEnum.docID(), impactsEnum.lastShallowTarget) : "Cannot reuse impacts after advancing the iterator";
+ return in.getDocIdUpTo(level);
+ }
+
+ @Override
+ public List<Impact> getImpacts(int level) {
+ assert validFor == Math.max(impactsEnum.docID(), impactsEnum.lastShallowTarget) : "Cannot reuse impacts after advancing the iterator";
+ return in.getImpacts(level);
+ }
+
+ }
+
/** Wraps a NumericDocValues but with additional asserts */
public static class AssertingNumericDocValues extends NumericDocValues {
private final Thread creationThread = Thread.currentThread();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
index 278f4b2..29962e6 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java
@@ -16,11 +16,18 @@
*/
package org.apache.lucene.index;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -33,13 +40,13 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.function.IntToLongFunction;
+import java.util.stream.Collectors;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
@@ -55,12 +62,6 @@ import org.apache.lucene.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.automaton.AutomatonTestUtil.RandomAcceptedStrings;
import org.apache.lucene.util.automaton.CompiledAutomaton;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
/** Helper class extracted from BasePostingsFormatTestCase to exercise a postings format. */
public class RandomPostingsTester {
@@ -608,7 +609,7 @@ public class RandomPostingsTester {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
+ public ImpactsEnum impacts(int flags) throws IOException {
throw new UnsupportedOperationException();
}
}
@@ -1055,126 +1056,146 @@ public class RandomPostingsTester {
} else {
docToNorm = doc -> 1L;
}
- for (int s = 0; s < 3; ++s) {
- final int scoreMode = s;
- SimScorer scorer = new SimScorer(field) {
- @Override
- public float score(float freq, long norm) {
- switch (scoreMode) {
- case 0:
- return freq; // make sure the postings record the best freq
- case 1:
- return 1f / norm; // make sure the postings record the best norm
- default:
- return freq - norm + MAX_NORM; // now a combination that could make intermediate pairs more competitive
- }
- }
- };
-
- // First check max scores and block uptos
- int max = -1;
- float maxScore = 0;
- int flags = PostingsEnum.FREQS;
- if (doCheckPositions) {
- flags |= PostingsEnum.POSITIONS;
+
+ // First check impacts and block uptos
+ int max = -1;
+ List<Impact> impactsCopy = null;
+ int flags = PostingsEnum.FREQS;
+ if (doCheckPositions) {
+ flags |= PostingsEnum.POSITIONS;
+ if (doCheckOffsets) {
+ flags |= PostingsEnum.OFFSETS;
+ }
+ if (doCheckPayloads) {
+ flags |= PostingsEnum.PAYLOADS;
+ }
+ }
+
+ ImpactsEnum impactsEnum = termsEnum.impacts(flags);
+ PostingsEnum postings = termsEnum.postings(null, flags);
+ for (int doc = impactsEnum.nextDoc(); ; doc = impactsEnum.nextDoc()) {
+ assertEquals(postings.nextDoc(), doc);
+ if (doc == DocIdSetIterator.NO_MORE_DOCS) {
+ break;
+ }
+ int freq = postings.freq();
+ assertEquals("freq is wrong", freq, impactsEnum.freq());
+ for (int i = 0; i < freq; ++i) {
+ int pos = postings.nextPosition();
+ assertEquals("position is wrong", pos, impactsEnum.nextPosition());
if (doCheckOffsets) {
- flags |= PostingsEnum.OFFSETS;
+ assertEquals("startOffset is wrong", postings.startOffset(), impactsEnum.startOffset());
+ assertEquals("endOffset is wrong", postings.endOffset(), impactsEnum.endOffset());
}
if (doCheckPayloads) {
- flags |= PostingsEnum.PAYLOADS;
+ assertEquals("payload is wrong", postings.getPayload(), impactsEnum.getPayload());
}
}
+ if (doc > max) {
+ impactsEnum.advanceShallow(doc);
+ Impacts impacts = impactsEnum.getImpacts();
+ CheckIndex.checkImpacts(impacts, doc);
+ impactsCopy = impacts.getImpacts(0)
+ .stream()
+ .map(i -> new Impact(i.freq, i.norm))
+ .collect(Collectors.toList());
+ }
+ freq = impactsEnum.freq();
+ long norm = docToNorm.applyAsLong(doc);
+ int idx = Collections.binarySearch(impactsCopy, new Impact(freq, norm), Comparator.comparing(i -> i.freq));
+ if (idx < 0) {
+ idx = -1 - idx;
+ }
+ assertTrue("Got " + new Impact(freq, norm) + " in postings, but no impact triggers equal or better scores in " + impactsCopy,
+ idx <= impactsCopy.size() && impactsCopy.get(idx).norm <= norm);
+ }
- ImpactsEnum impacts = termsEnum.impacts(scorer, flags);
- PostingsEnum postings = termsEnum.postings(null, flags);
- for (int doc = impacts.nextDoc(); ; doc = impacts.nextDoc()) {
- assertEquals(postings.nextDoc(), doc);
- if (doc == DocIdSetIterator.NO_MORE_DOCS) {
- break;
- }
- int freq = postings.freq();
- assertEquals("freq is wrong", freq, impacts.freq());
- for (int i = 0; i < freq; ++i) {
- int pos = postings.nextPosition();
- assertEquals("position is wrong", pos, impacts.nextPosition());
- if (doCheckOffsets) {
- assertEquals("startOffset is wrong", postings.startOffset(), impacts.startOffset());
- assertEquals("endOffset is wrong", postings.endOffset(), impacts.endOffset());
- }
- if (doCheckPayloads) {
- assertEquals("payload is wrong", postings.getPayload(), impacts.getPayload());
+ // Now check advancing
+ impactsEnum = termsEnum.impacts(flags);
+ postings = termsEnum.postings(postings, flags);
+
+ max = -1;
+ while (true) {
+ int doc = impactsEnum.docID();
+ boolean advance;
+ int target;
+ if (random.nextBoolean()) {
+ advance = false;
+ target = doc + 1;
+ } else {
+ advance = true;
+ int delta = Math.min(1 + random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - doc);
+ target = impactsEnum.docID() + delta;
+ }
+
+ if (target > max && random.nextBoolean()) {
+ int delta = Math.min(random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - target);
+ max = target + delta;
+
+ impactsEnum.advanceShallow(target);
+ Impacts impacts = impactsEnum.getImpacts();
+ CheckIndex.checkImpacts(impacts, target);
+ impactsCopy = Collections.singletonList(new Impact(Integer.MAX_VALUE, 1L));
+ for (int level = 0; level < impacts.numLevels(); ++level) {
+ if (impacts.getDocIdUpTo(level) >= max) {
+ impactsCopy = impacts.getImpacts(level)
+ .stream()
+ .map(i -> new Impact(i.freq, i.norm))
+ .collect(Collectors.toList());
+ break;
}
}
- if (doc > max) {
- max = impacts.advanceShallow(doc);
- assertTrue(max >= doc);
- maxScore = impacts.getMaxScore(max);
- }
- assertEquals(max, impacts.advanceShallow(doc));
- assertTrue(scorer.score(impacts.freq(), docToNorm.applyAsLong(doc)) <= maxScore);
}
- // Now check advancing
- impacts = termsEnum.impacts(scorer, flags);
- postings = termsEnum.postings(postings, flags);
-
- max = -1;
- while (true) {
- int doc = impacts.docID();
- boolean advance;
- int target;
- if (random.nextBoolean()) {
- advance = false;
- target = doc + 1;
- } else {
- advance = true;
- int delta = Math.min(1 + random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - doc);
- target = impacts.docID() + delta;
- }
+ if (advance) {
+ doc = impactsEnum.advance(target);
+ } else {
+ doc = impactsEnum.nextDoc();
+ }
- if (target > max && random.nextBoolean()) {
- int delta = Math.min(random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - target);
- max = target + delta;
- int m = impacts.advanceShallow(target);
- assertTrue(m >= target);
- maxScore = impacts.getMaxScore(max);
+ assertEquals(postings.advance(target), doc);
+ if (doc == DocIdSetIterator.NO_MORE_DOCS) {
+ break;
+ }
+ int freq = postings.freq();
+ assertEquals("freq is wrong", freq, impactsEnum.freq());
+ for (int i = 0; i < postings.freq(); ++i) {
+ int pos = postings.nextPosition();
+ assertEquals("position is wrong", pos, impactsEnum.nextPosition());
+ if (doCheckOffsets) {
+ assertEquals("startOffset is wrong", postings.startOffset(), impactsEnum.startOffset());
+ assertEquals("endOffset is wrong", postings.endOffset(), impactsEnum.endOffset());
}
-
- if (advance) {
- doc = impacts.advance(target);
- } else {
- doc = impacts.nextDoc();
+ if (doCheckPayloads) {
+ assertEquals("payload is wrong", postings.getPayload(), impactsEnum.getPayload());
}
+ }
- assertEquals(postings.advance(target), doc);
- if (doc == DocIdSetIterator.NO_MORE_DOCS) {
- break;
- }
- int freq = postings.freq();
- assertEquals("freq is wrong", freq, impacts.freq());
- for (int i = 0; i < postings.freq(); ++i) {
- int pos = postings.nextPosition();
- assertEquals("position is wrong", pos, impacts.nextPosition());
- if (doCheckOffsets) {
- assertEquals("startOffset is wrong", postings.startOffset(), impacts.startOffset());
- assertEquals("endOffset is wrong", postings.endOffset(), impacts.endOffset());
- }
- if (doCheckPayloads) {
- assertEquals("payload is wrong", postings.getPayload(), impacts.getPayload());
+ if (doc > max) {
+ int delta = Math.min(1 + random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - doc);
+ max = doc + delta;
+ Impacts impacts = impactsEnum.getImpacts();
+ CheckIndex.checkImpacts(impacts, doc);
+ impactsCopy = Collections.singletonList(new Impact(Integer.MAX_VALUE, 1L));
+ for (int level = 0; level < impacts.numLevels(); ++level) {
+ if (impacts.getDocIdUpTo(level) >= max) {
+ impactsCopy = impacts.getImpacts(level)
+ .stream()
+ .map(i -> new Impact(i.freq, i.norm))
+ .collect(Collectors.toList());
+ break;
}
}
+ }
- if (doc > max) {
- int delta = Math.min(1 + random.nextInt(512), DocIdSetIterator.NO_MORE_DOCS - doc);
- max = doc + delta;
- int m = impacts.advanceShallow(doc);
- assertTrue(m >= doc);
- maxScore = impacts.getMaxScore(max);
- }
-
- float score = scorer.score(impacts.freq(), docToNorm.applyAsLong(doc));
- assertTrue(score <= maxScore);
+ freq = impactsEnum.freq();
+ long norm = docToNorm.applyAsLong(doc);
+ int idx = Collections.binarySearch(impactsCopy, new Impact(freq, norm), Comparator.comparing(i -> i.freq));
+ if (idx < 0) {
+ idx = -1 - idx;
}
+ assertTrue("Got " + new Impact(freq, norm) + " in postings, but no impact triggers equal or better scores in " + impactsCopy,
+ idx <= impactsCopy.size() && impactsCopy.get(idx).norm <= norm);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
index 80cd4da..00aee41 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
@@ -88,6 +88,7 @@ public class AssertingScorer extends Scorer {
@Override
public float getMaxScore(int upTo) throws IOException {
assert upTo >= lastShallowTarget : "uTo = " + upTo + " < last target = " + lastShallowTarget;
+ assert docID() >= 0 || lastShallowTarget >= 0 : "Cannot get max scores until the iterator is positioned or advanceShallow has been called";
float maxScore = in.getMaxScore(upTo);
return maxScore;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java
index a918078..784d3bb 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java
@@ -632,7 +632,7 @@ public class CheckHits {
Assert.assertTrue(twoPhase1 == null || twoPhase1.matches());
float score = s2.score();
Assert.assertEquals(s1.score(), score);
- Assert.assertTrue(score <= maxScore);
+ Assert.assertTrue(score + " > " + maxScore + " up to " + upTo, score <= maxScore);
if (score >= minScore && random.nextInt(10) == 0) {
// On some scorers, changing the min score changes the way that docs are iterated
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
index 1b81c7f..53cddc3 100644
--- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
+++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java
@@ -43,7 +43,6 @@ import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.DocIdSetBuilder;
@@ -241,8 +240,8 @@ public final class SolrRangeQuery extends ExtendedQueryBase implements DocSetPro
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return te.impacts(scorer, flags);
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return te.impacts(flags);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af680af7/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java b/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java
index daaf00d..580a971 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java
@@ -35,7 +35,6 @@ import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -609,8 +608,8 @@ public class DocTermOrds implements Accountable {
}
@Override
- public ImpactsEnum impacts(SimScorer scorer, int flags) throws IOException {
- return termsEnum.impacts(scorer, flags);
+ public ImpactsEnum impacts(int flags) throws IOException {
+ return termsEnum.impacts(flags);
}
@Override
[26/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12289: Add more MDC
logging information to collection admin requests
Posted by ab...@apache.org.
SOLR-12289: Add more MDC logging information to collection admin requests
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/84925ba9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/84925ba9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/84925ba9
Branch: refs/heads/jira/solr-11779
Commit: 84925ba9abc7da8485f27fd52d0f80b14caad98f
Parents: 67c13bb
Author: Varun Thacker <va...@apache.org>
Authored: Wed May 2 19:01:07 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed May 2 19:01:07 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../api/collections/OverseerCollectionMessageHandler.java | 9 +++++++--
.../org/apache/solr/handler/admin/CollectionsHandler.java | 4 +++-
.../src/java/org/apache/solr/logging/MDCLoggingContext.java | 6 +++---
4 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84925ba9/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f8261ea..b2235fd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -274,6 +274,8 @@ Other Changes
* SOLR-10036: Upgrade jackson from 2.5.4 to 2.9.5 (Shashank Pedamallu, Shawn Heisey, Varun Thacker)
+* SOLR-12289: Add more MDC logging information to collection admin requests (Varun Thacker)
+
================== 7.3.1 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84925ba9/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
index 6b61455..fed9254 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
@@ -35,11 +35,11 @@ import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.DistributedQueue;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException;
import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException;
-import org.apache.solr.client.solrj.cloud.DistribStateManager;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
@@ -83,6 +83,7 @@ import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.handler.component.ShardRequest;
import org.apache.solr.handler.component.ShardResponse;
+import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.TimeOut;
@@ -103,6 +104,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonParams.NAME;
@@ -242,6 +244,9 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
@Override
@SuppressWarnings("unchecked")
public SolrResponse processMessage(ZkNodeProps message, String operation) {
+ MDCLoggingContext.setCollection(message.getStr(COLLECTION));
+ MDCLoggingContext.setShard(message.getStr(SHARD_ID_PROP));
+ MDCLoggingContext.setReplica(message.getStr(REPLICA_PROP));
log.debug("OverseerCollectionMessageHandler.processMessage : {} , {}", operation, message);
NamedList results = new NamedList();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84925ba9/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index ff47c8b..1c7f81a 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -84,6 +84,7 @@ import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -132,6 +133,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;
+import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_NAME;
import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_VALUE;
@@ -226,7 +228,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
}
CollectionOperation operation = CollectionOperation.get(action);
log.info("Invoked Collection Action :{} with params {} and sendToOCPQueue={}", action.toLower(), req.getParamString(), operation.sendToOCPQueue);
-
+ MDCLoggingContext.setCollection(req.getParams().get(COLLECTION));
invokeAction(req, rsp, cores, action, operation);
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "action is a required param");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84925ba9/solr/core/src/java/org/apache/solr/logging/MDCLoggingContext.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/logging/MDCLoggingContext.java b/solr/core/src/java/org/apache/solr/logging/MDCLoggingContext.java
index 11c8b84..1f5324b 100644
--- a/solr/core/src/java/org/apache/solr/logging/MDCLoggingContext.java
+++ b/solr/core/src/java/org/apache/solr/logging/MDCLoggingContext.java
@@ -45,7 +45,7 @@ public class MDCLoggingContext {
}
});
- private static void setCollection(String collection) {
+ public static void setCollection(String collection) {
if (collection != null) {
MDC.put(COLLECTION_PROP, "c:" + collection);
} else {
@@ -53,7 +53,7 @@ public class MDCLoggingContext {
}
}
- private static void setShard(String shard) {
+ public static void setShard(String shard) {
if (shard != null) {
MDC.put(SHARD_ID_PROP, "s:" + shard);
} else {
@@ -61,7 +61,7 @@ public class MDCLoggingContext {
}
}
- private static void setReplica(String replica) {
+ public static void setReplica(String replica) {
if (replica != null) {
MDC.put(REPLICA_PROP, "r:" + replica);
} else {
[14/50] [abbrv] lucene-solr:jira/solr-11779: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by ab...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/871ffbe1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/871ffbe1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/871ffbe1
Branch: refs/heads/jira/solr-11779
Commit: 871ffbe10edcad6574da4f4b93e9b56c9761c2be
Parents: ff68acf fe018e3
Author: Karl Wright <Da...@gmail.com>
Authored: Mon Apr 30 06:12:46 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Mon Apr 30 06:12:46 2018 -0400
----------------------------------------------------------------------
.../index/BinaryDocValuesFieldUpdates.java | 32 ++--
.../apache/lucene/index/BufferedUpdates.java | 62 ++++----
.../lucene/index/DocValuesFieldUpdates.java | 65 ++++----
.../apache/lucene/index/DocValuesUpdate.java | 92 ++++++++++--
.../index/DocumentsWriterDeleteQueue.java | 7 +-
.../lucene/index/FrozenBufferedUpdates.java | 147 +++++++------------
.../org/apache/lucene/index/IndexWriter.java | 2 +-
.../index/NumericDocValuesFieldUpdates.java | 44 ++++--
.../apache/lucene/index/ReadersAndUpdates.java | 7 +-
.../lucene/index/TestDocValuesFieldUpdates.java | 72 +++++++++
.../lucene/index/TestPendingSoftDeletes.java | 27 +++-
.../directory/TestDirectoryTaxonomyWriter.java | 59 ++++----
12 files changed, 374 insertions(+), 242 deletions(-)
----------------------------------------------------------------------
[28/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12278: Fix typo
errors in ref-guide
Posted by ab...@apache.org.
SOLR-12278: Fix typo errors in ref-guide
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ed948efa
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ed948efa
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ed948efa
Branch: refs/heads/jira/solr-11779
Commit: ed948efabfe54a703587fc01caeed94ce2401946
Parents: 596d0dc
Author: Cao Manh Dat <da...@apache.org>
Authored: Thu May 3 10:40:21 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Thu May 3 10:40:21 2018 +0700
----------------------------------------------------------------------
solr/solr-ref-guide/src/update-request-processors.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ed948efa/solr/solr-ref-guide/src/update-request-processors.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/update-request-processors.adoc b/solr/solr-ref-guide/src/update-request-processors.adoc
index 14e64b6..cf76d3b 100644
--- a/solr/solr-ref-guide/src/update-request-processors.adoc
+++ b/solr/solr-ref-guide/src/update-request-processors.adoc
@@ -275,7 +275,7 @@ What follows are brief descriptions of the currently available update request pr
{solr-javadocs}/solr-core/org/apache/solr/update/processor/IgnoreCommitOptimizeUpdateProcessorFactory.html[IgnoreCommitOptimizeUpdateProcessorFactory]:: Allows you to ignore commit and/or optimize requests from client applications when running in SolrCloud mode, for more information, see: Shards and Indexing Data in SolrCloud
-{solr-javadocs}/solr-core/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.html[IgnoreLargeDocumentProcessorFactory]:: Allows you to filtering out large document with more than `limit` (in KB) to Solr. It can help to prevent unexpected problems on indexing as well as on recovering because of very large documents.
+{solr-javadocs}/solr-core/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.html[IgnoreLargeDocumentProcessorFactory]:: Allows you to prevent large documents with size more than `limit` (in KB) from getting indexed. It can help to prevent unexpected problems on indexing as well as on recovering because of very large documents.
{solr-javadocs}/solr-core/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.html[CloneFieldUpdateProcessorFactory]:: Clones the values found in any matching _source_ field into the configured _dest_ field.
[27/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12278: Adding
ref-guide doc for IgnoreLargeDocumentProcessorFactory
Posted by ab...@apache.org.
SOLR-12278: Adding ref-guide doc for IgnoreLargeDocumentProcessorFactory
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/596d0dc9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/596d0dc9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/596d0dc9
Branch: refs/heads/jira/solr-11779
Commit: 596d0dc9a6ef8633a078bf74ea377d727de4183e
Parents: 84925ba
Author: Cao Manh Dat <da...@apache.org>
Authored: Thu May 3 10:35:03 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Thu May 3 10:35:03 2018 +0700
----------------------------------------------------------------------
solr/solr-ref-guide/src/update-request-processors.adoc | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/596d0dc9/solr/solr-ref-guide/src/update-request-processors.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/update-request-processors.adoc b/solr/solr-ref-guide/src/update-request-processors.adoc
index 6958628..14e64b6 100644
--- a/solr/solr-ref-guide/src/update-request-processors.adoc
+++ b/solr/solr-ref-guide/src/update-request-processors.adoc
@@ -275,6 +275,8 @@ What follows are brief descriptions of the currently available update request pr
{solr-javadocs}/solr-core/org/apache/solr/update/processor/IgnoreCommitOptimizeUpdateProcessorFactory.html[IgnoreCommitOptimizeUpdateProcessorFactory]:: Allows you to ignore commit and/or optimize requests from client applications when running in SolrCloud mode, for more information, see: Shards and Indexing Data in SolrCloud
+{solr-javadocs}/solr-core/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.html[IgnoreLargeDocumentProcessorFactory]:: Allows you to filtering out large document with more than `limit` (in KB) to Solr. It can help to prevent unexpected problems on indexing as well as on recovering because of very large documents.
+
{solr-javadocs}/solr-core/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.html[CloneFieldUpdateProcessorFactory]:: Clones the values found in any matching _source_ field into the configured _dest_ field.
{solr-javadocs}/solr-core/org/apache/solr/update/processor/RegexpBoostProcessorFactory.html[RegexpBoostProcessorFactory]:: A processor which will match content of "inputField" against regular expressions found in "boostFilename", and if it matches will return the corresponding boost value from the file and output this to "boostField" as a double value.
[12/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8275: Fix
TestDirectoryTaxonomyWriter.testRecreateAndRefresh
Posted by ab...@apache.org.
LUCENE-8275: Fix TestDirectoryTaxonomyWriter.testRecreateAndRefresh
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fe018e37
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fe018e37
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fe018e37
Branch: refs/heads/jira/solr-11779
Commit: fe018e37f3a8bc4af52ca98e0e49c1317b159c80
Parents: b43b091
Author: Simon Willnauer <si...@apache.org>
Authored: Mon Apr 30 12:06:06 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon Apr 30 12:08:16 2018 +0200
----------------------------------------------------------------------
.../directory/TestDirectoryTaxonomyWriter.java | 59 +++++++++++---------
1 file changed, 32 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe018e37/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
index 1b5b826..88fb042 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
@@ -44,6 +44,7 @@ import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import org.junit.Test;
@@ -191,37 +192,41 @@ public class TestDirectoryTaxonomyWriter extends FacetTestCase {
// DirTaxoWriter lost the INDEX_EPOCH property if it was opened in
// CREATE_OR_APPEND (or commit(userData) called twice), which could lead to
// DirTaxoReader succeeding to refresh().
- Directory dir = newDirectory();
-
- DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
- touchTaxo(taxoWriter, new FacetLabel("a"));
-
- TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
+ try (Directory dir = newDirectory()) {
- touchTaxo(taxoWriter, new FacetLabel("b"));
-
- TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
- taxoReader.close();
- taxoReader = newtr;
- assertEquals(1, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));
+ DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
+ touchTaxo(taxoWriter, new FacetLabel("a"));
- // now recreate the taxonomy, and check that the epoch is preserved after opening DirTW again.
- taxoWriter.close();
- taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
- touchTaxo(taxoWriter, new FacetLabel("c"));
- taxoWriter.close();
-
- taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
- touchTaxo(taxoWriter, new FacetLabel("d"));
- taxoWriter.close();
+ TaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
- newtr = TaxonomyReader.openIfChanged(taxoReader);
- taxoReader.close();
- taxoReader = newtr;
- assertEquals(2, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));
+ touchTaxo(taxoWriter, new FacetLabel("b"));
- taxoReader.close();
- dir.close();
+ TaxonomyReader newtr = TaxonomyReader.openIfChanged(taxoReader);
+ taxoReader.close();
+ taxoReader = newtr;
+ assertEquals(1, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));
+
+ // now recreate the taxonomy, and check that the epoch is preserved after opening DirTW again.
+ taxoWriter.close();
+
+ assumeFalse("if we are on windows and we have pending deletions we can't execute this test",
+ Constants.WINDOWS && dir.checkPendingDeletions());
+ taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
+ touchTaxo(taxoWriter, new FacetLabel("c"));
+ taxoWriter.close();
+
+ assumeFalse("if we are on windows and we have pending deletions we can't execute this test",
+ Constants.WINDOWS && dir.checkPendingDeletions());
+ taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
+ touchTaxo(taxoWriter, new FacetLabel("d"));
+ taxoWriter.close();
+
+ newtr = TaxonomyReader.openIfChanged(taxoReader);
+ taxoReader.close();
+ taxoReader = newtr;
+ assertEquals(2, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));
+ taxoReader.close();
+ }
}
@Test
[21/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8279: CheckIndex
now cross-checks terms with norms.
Posted by ab...@apache.org.
LUCENE-8279: CheckIndex now cross-checks terms with norms.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e00c4ced
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e00c4ced
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e00c4ced
Branch: refs/heads/jira/solr-11779
Commit: e00c4cede26690a82cf553a22b53a47c675cc01d
Parents: af680af
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed May 2 11:55:48 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed May 2 14:49:32 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 ++
.../org/apache/lucene/index/CheckIndex.java | 34 +++++++++++++++++---
.../lucene/index/TestFilterLeafReader.java | 22 +++++++++++++
3 files changed, 53 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e00c4ced/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index a5d0816..ed6cc26 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -80,6 +80,8 @@ Improvements
* LUCENE-8135: Boolean queries now implement the block-max WAND algorithm in
order to speed up selection of top scored documents. (Adrien Grand)
+* LUCENE-8279: CheckIndex now cross-checks terms with norms. (Adrien Grand)
+
Optimizations
* LUCENE-8040: Optimize IndexSearcher.collectionStatistics, avoiding MultiFields/MultiTerms
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e00c4ced/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index ca27066..0f7871a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -739,7 +739,7 @@ public final class CheckIndex implements Closeable {
// Test Fieldinfos
segInfoStat.fieldInfoStatus = testFieldInfos(reader, infoStream, failFast);
-
+
// Test Field Norms
segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream, failFast);
@@ -1209,7 +1209,8 @@ public final class CheckIndex implements Closeable {
* checks Fields api is consistent with itself.
* searcher is optional, to verify with queries. Can be null.
*/
- private static Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos, boolean doPrint, boolean isVectors, PrintStream infoStream, boolean verbose, boolean doSlowChecks) throws IOException {
+ private static Status.TermIndexStatus checkFields(Fields fields, Bits liveDocs, int maxDoc, FieldInfos fieldInfos,
+ NormsProducer normsProducer, boolean doPrint, boolean isVectors, PrintStream infoStream, boolean verbose, boolean doSlowChecks) throws IOException {
// TODO: we should probably return our own stats thing...?!
long startNS;
if (doPrint) {
@@ -1754,7 +1755,26 @@ public final class CheckIndex implements Closeable {
if (visitedDocs.cardinality() != v) {
throw new RuntimeException("docCount for field " + field + "=" + v + " != recomputed docCount=" + visitedDocs.cardinality());
}
-
+
+ if (fieldInfo.hasNorms() && isVectors == false) {
+ final NumericDocValues norms = normsProducer.getNorms(fieldInfo);
+ // Cross-check terms with norms
+ for (int doc = norms.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = norms.nextDoc()) {
+ if (liveDocs != null && liveDocs.get(doc) == false) {
+ // Norms may only be out of sync with terms on deleted documents.
+ // This happens when a document fails indexing and in that case it
+ // should be immediately marked as deleted by the IndexWriter.
+ continue;
+ }
+ final long norm = norms.longValue();
+ if (norm != 0 && visitedDocs.get(doc) == false) {
+ throw new RuntimeException("Document " + doc + " doesn't have terms according to postings but has a norm value that is not zero: " + norm);
+ } else if (norm == 0 && visitedDocs.get(doc)) {
+ throw new RuntimeException("Document " + doc + " has terms according to postings but its norm value is 0, which may only be used on documents that have no terms");
+ }
+ }
+ }
+
// Test seek to last term:
if (lastTerm != null) {
if (termsEnum.seekCeil(lastTerm.get()) != TermsEnum.SeekStatus.FOUND) {
@@ -1937,7 +1957,11 @@ public final class CheckIndex implements Closeable {
final Fields fields = reader.getPostingsReader().getMergeInstance();
final FieldInfos fieldInfos = reader.getFieldInfos();
- status = checkFields(fields, reader.getLiveDocs(), maxDoc, fieldInfos, true, false, infoStream, verbose, doSlowChecks);
+ NormsProducer normsProducer = reader.getNormsReader();
+ if (normsProducer != null) {
+ normsProducer = normsProducer.getMergeInstance();
+ }
+ status = checkFields(fields, reader.getLiveDocs(), maxDoc, fieldInfos, normsProducer, true, false, infoStream, verbose, doSlowChecks);
} catch (Throwable e) {
if (failFast) {
throw IOUtils.rethrowAlways(e);
@@ -2594,7 +2618,7 @@ public final class CheckIndex implements Closeable {
if (tfv != null) {
// First run with no deletions:
- checkFields(tfv, null, 1, fieldInfos, false, true, infoStream, verbose, doSlowChecks);
+ checkFields(tfv, null, 1, fieldInfos, null, false, true, infoStream, verbose, doSlowChecks);
// Only agg stats if the doc is live:
final boolean doStats = liveDocs == null || liveDocs.get(j);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e00c4ced/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java
index de06f90..f89ef9c 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java
@@ -29,6 +29,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
public class TestFilterLeafReader extends LuceneTestCase {
@@ -97,6 +98,27 @@ public class TestFilterLeafReader extends LuceneTestCase {
}
@Override
+ public NumericDocValues getNormValues(String field) throws IOException {
+ NumericDocValues ndv = super.getNormValues(field);
+ if (ndv == null) {
+ return null;
+ }
+ FixedBitSet docsWithTerms = new FixedBitSet(maxDoc());
+ TermsEnum termsEnum = terms(field).iterator();
+ PostingsEnum postings = null;
+ while (termsEnum.next() != null) {
+ postings = termsEnum.postings(postings, PostingsEnum.NONE);
+ docsWithTerms.or(postings);
+ }
+ return new FilterNumericDocValues(ndv) {
+ @Override
+ public long longValue() throws IOException {
+ return docsWithTerms.get(docID()) ? super.longValue() : 0L;
+ }
+ };
+ }
+
+ @Override
public CacheHelper getCoreCacheHelper() {
return null;
}
[15/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12284: Stop adding
parenthesis to word-break suggestions, unless query uses boolean operators.
Posted by ab...@apache.org.
SOLR-12284: Stop adding parenthesis to word-break suggestions, unless query uses boolean operators.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d92b891f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d92b891f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d92b891f
Branch: refs/heads/jira/solr-11779
Commit: d92b891f95f3185e5d23dc89a23831e633ab64e5
Parents: 871ffbe
Author: jdyer1 <jd...@apache.org>
Authored: Mon Apr 30 13:47:19 2018 -0500
Committer: jdyer1 <jd...@apache.org>
Committed: Mon Apr 30 13:47:19 2018 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../org/apache/solr/spelling/SpellCheckCollator.java | 4 ++--
.../solr/spelling/WordBreakSolrSpellCheckerTest.java | 14 +++++++-------
3 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d92b891f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 08c7c00..ea166fb 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -193,6 +193,9 @@ Bug Fixes
* SOLR-12275: wrong caching for {!filters} as well as for `filters` local param in {!parent} and {!child}
(David Smiley, Mikhail Khluldnev)
+
+* SOLR-12284: WordBreakSolrSpellchecker will no longer add parenthesis in collations when breaking words in
+ non-boolean queries. (James Dyer)
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d92b891f/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
index 859d84f..ad3accf 100644
--- a/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
+++ b/solr/core/src/java/org/apache/solr/spelling/SpellCheckCollator.java
@@ -221,8 +221,7 @@ public class SpellCheckCollator {
//If the correction contains whitespace (because it involved breaking a word in 2+ words),
//then be sure all of the new words have the same optional/required/prohibited status in the query.
- while(indexOfSpace>-1 && indexOfSpace<corr.length()-1) {
- addParenthesis = true;
+ while(indexOfSpace>-1 && indexOfSpace<corr.length()-1) {
char previousChar = tok.startOffset()>0 ? origQuery.charAt(tok.startOffset()-1) : ' ';
if(previousChar=='-' || previousChar=='+') {
corrSb.insert(indexOfSpace + bump, previousChar);
@@ -231,6 +230,7 @@ public class SpellCheckCollator {
}
bump++;
} else if ((tok.getFlags() & QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG) == QueryConverter.TERM_IN_BOOLEAN_QUERY_FLAG) {
+ addParenthesis = true;
corrSb.insert(indexOfSpace + bump, "AND ");
bump += 4;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d92b891f/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java b/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java
index 258594a..e96f499 100644
--- a/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java
+++ b/solr/core/src/test/org/apache/solr/spelling/WordBreakSolrSpellCheckerTest.java
@@ -234,13 +234,13 @@ public class WordBreakSolrSpellCheckerTest extends SolrTestCaseJ4 {
"//lst[@name='collation'][1 ]/str[@name='collationQuery']='lowerfilt:(printable line ample goodness)'",
"//lst[@name='collation'][2 ]/str[@name='collationQuery']='lowerfilt:(paintablepine ample goodness)'",
"//lst[@name='collation'][3 ]/str[@name='collationQuery']='lowerfilt:(printable pineapple goodness)'",
- "//lst[@name='collation'][4 ]/str[@name='collationQuery']='lowerfilt:((paint able) line ample goodness)'",
- "//lst[@name='collation'][5 ]/str[@name='collationQuery']='lowerfilt:(printable (pi ne) ample goodness)'",
- "//lst[@name='collation'][6 ]/str[@name='collationQuery']='lowerfilt:((paint able) pineapple goodness)'",
- "//lst[@name='collation'][7 ]/str[@name='collationQuery']='lowerfilt:((paint able) (pi ne) ample goodness)'",
+ "//lst[@name='collation'][4 ]/str[@name='collationQuery']='lowerfilt:(paint able line ample goodness)'",
+ "//lst[@name='collation'][5 ]/str[@name='collationQuery']='lowerfilt:(printable pi ne ample goodness)'",
+ "//lst[@name='collation'][6 ]/str[@name='collationQuery']='lowerfilt:(paint able pineapple goodness)'",
+ "//lst[@name='collation'][7 ]/str[@name='collationQuery']='lowerfilt:(paint able pi ne ample goodness)'",
"//lst[@name='collation'][8 ]/str[@name='collationQuery']='lowerfilt:(pintable line ample goodness)'",
"//lst[@name='collation'][9 ]/str[@name='collationQuery']='lowerfilt:(pintable pineapple goodness)'",
- "//lst[@name='collation'][10]/str[@name='collationQuery']='lowerfilt:(pintable (pi ne) ample goodness)'",
+ "//lst[@name='collation'][10]/str[@name='collationQuery']='lowerfilt:(pintable pi ne ample goodness)'",
"//lst[@name='collation'][10]/lst[@name='misspellingsAndCorrections']/str[@name='paintable']='pintable'",
"//lst[@name='collation'][10]/lst[@name='misspellingsAndCorrections']/str[@name='pine']='pi ne'",
"//lst[@name='collation'][10]/lst[@name='misspellingsAndCorrections']/str[@name='apple']='ample'",
@@ -297,7 +297,7 @@ public class WordBreakSolrSpellCheckerTest extends SolrTestCaseJ4 {
SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true",
SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "10"),
"//lst[@name='collation'][1 ]/str[@name='collationQuery']='lowerfilt:(+line -ample)'",
- "//lst[@name='collation'][2 ]/str[@name='collationQuery']='lowerfilt:((+pi +ne) -ample)'"
+ "//lst[@name='collation'][2 ]/str[@name='collationQuery']='lowerfilt:(+pi +ne -ample)'"
);
assertQ(req(
"q", "lowerfilt:(+printableinpuntableplantable)",
@@ -309,7 +309,7 @@ public class WordBreakSolrSpellCheckerTest extends SolrTestCaseJ4 {
SpellCheckComponent.SPELLCHECK_COLLATE, "true",
SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true",
SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1"),
- "//lst[@name='collation'][1 ]/str[@name='collationQuery']='lowerfilt:((+printable +in +puntable +plantable))'"
+ "//lst[@name='collation'][1 ]/str[@name='collationQuery']='lowerfilt:(+printable +in +puntable +plantable)'"
);
assertQ(req(
"q", "zxcv AND qwtp AND fghj",
[09/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-10036: Upgrade
jackson from 2.5.4 to 2.9.5
Posted by ab...@apache.org.
SOLR-10036: Upgrade jackson from 2.5.4 to 2.9.5
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f6cbb2db
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f6cbb2db
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f6cbb2db
Branch: refs/heads/jira/solr-11779
Commit: f6cbb2db6be608cacc0c15f34d1db1b53b16d560
Parents: 70abbe74
Author: Varun Thacker <va...@apache.org>
Authored: Sat Apr 28 16:56:06 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Sat Apr 28 16:56:06 2018 -0700
----------------------------------------------------------------------
lucene/ivy-versions.properties | 2 +-
solr/CHANGES.txt | 2 ++
solr/licenses/jackson-annotations-2.5.4.jar.sha1 | 1 -
solr/licenses/jackson-annotations-2.9.5.jar.sha1 | 1 +
solr/licenses/jackson-core-2.5.4.jar.sha1 | 1 -
solr/licenses/jackson-core-2.9.5.jar.sha1 | 1 +
solr/licenses/jackson-databind-2.5.4.jar.sha1 | 1 -
solr/licenses/jackson-databind-2.9.5.jar.sha1 | 1 +
solr/licenses/jackson-dataformat-smile-2.5.4.jar.sha1 | 1 -
solr/licenses/jackson-dataformat-smile-2.9.5.jar.sha1 | 1 +
10 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6cbb2db/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index d930f25..1004e25 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -15,7 +15,7 @@ com.carrotsearch.randomizedtesting.version = 2.5.3
/com.cybozu.labs/langdetect = 1.1-20120112
/com.drewnoakes/metadata-extractor = 2.10.1
-com.fasterxml.jackson.core.version = 2.5.4
+com.fasterxml.jackson.core.version = 2.9.5
/com.fasterxml.jackson.core/jackson-annotations = ${com.fasterxml.jackson.core.version}
/com.fasterxml.jackson.core/jackson-core = ${com.fasterxml.jackson.core.version}
/com.fasterxml.jackson.core/jackson-databind = ${com.fasterxml.jackson.core.version}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6cbb2db/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 021248a..08c7c00 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -262,6 +262,8 @@ Other Changes
toMap, toMultiMap, toFilteredSolrParams, getAll. The latter ones have no direct replacement but are easy to
implement yourself as-needed. (David Smiley)
+* SOLR-10036: Upgrade jackson from 2.5.4 to 2.9.5 (Shashank Pedamallu, Shawn Heisey, Varun Thacker)
+
================== 7.3.1 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6cbb2db/solr/licenses/jackson-annotations-2.5.4.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jackson-annotations-2.5.4.jar.sha1 b/solr/licenses/jackson-annotations-2.5.4.jar.sha1
deleted file mode 100644
index 76bf71f..0000000
--- a/solr/licenses/jackson-annotations-2.5.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7a93b60f5d2d43024f34e15893552ee6defdb971
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6cbb2db/solr/licenses/jackson-annotations-2.9.5.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jackson-annotations-2.9.5.jar.sha1 b/solr/licenses/jackson-annotations-2.9.5.jar.sha1
new file mode 100644
index 0000000..4d7ff74
--- /dev/null
+++ b/solr/licenses/jackson-annotations-2.9.5.jar.sha1
@@ -0,0 +1 @@
+9056ec9db21c57d43219a84bb18c129ae51c6a5d
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6cbb2db/solr/licenses/jackson-core-2.5.4.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jackson-core-2.5.4.jar.sha1 b/solr/licenses/jackson-core-2.5.4.jar.sha1
deleted file mode 100644
index da83a8b..0000000
--- a/solr/licenses/jackson-core-2.5.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0a57a2df1a23ca1ee32f129173ba7f5feaa9ac24
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6cbb2db/solr/licenses/jackson-core-2.9.5.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jackson-core-2.9.5.jar.sha1 b/solr/licenses/jackson-core-2.9.5.jar.sha1
new file mode 100644
index 0000000..b5a7248
--- /dev/null
+++ b/solr/licenses/jackson-core-2.9.5.jar.sha1
@@ -0,0 +1 @@
+a22ac51016944b06fd9ffbc9541c6e7ce5eea117
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6cbb2db/solr/licenses/jackson-databind-2.5.4.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jackson-databind-2.5.4.jar.sha1 b/solr/licenses/jackson-databind-2.5.4.jar.sha1
deleted file mode 100644
index 0fa583e..0000000
--- a/solr/licenses/jackson-databind-2.5.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5dfa42af84584b4a862ea488da84bbbebbb06c35
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6cbb2db/solr/licenses/jackson-databind-2.9.5.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jackson-databind-2.9.5.jar.sha1 b/solr/licenses/jackson-databind-2.9.5.jar.sha1
new file mode 100644
index 0000000..db2ea79
--- /dev/null
+++ b/solr/licenses/jackson-databind-2.9.5.jar.sha1
@@ -0,0 +1 @@
+3490508379d065fe3fcb80042b62f630f7588606
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6cbb2db/solr/licenses/jackson-dataformat-smile-2.5.4.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jackson-dataformat-smile-2.5.4.jar.sha1 b/solr/licenses/jackson-dataformat-smile-2.5.4.jar.sha1
deleted file mode 100644
index e15996c..0000000
--- a/solr/licenses/jackson-dataformat-smile-2.5.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-db0c5f1b6e16cb5f5e0505abfcd4b36f3e8bfdc6
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6cbb2db/solr/licenses/jackson-dataformat-smile-2.9.5.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/jackson-dataformat-smile-2.9.5.jar.sha1 b/solr/licenses/jackson-dataformat-smile-2.9.5.jar.sha1
new file mode 100644
index 0000000..0406c1e
--- /dev/null
+++ b/solr/licenses/jackson-dataformat-smile-2.9.5.jar.sha1
@@ -0,0 +1 @@
+dbc8efaf5c70004dfb4ac5db6fa7e12534ada22d
[32/50] [abbrv] lucene-solr:jira/solr-11779: json-facet-api.adoc typo
fix.
Posted by ab...@apache.org.
json-facet-api.adoc typo fix.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ab118673
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ab118673
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ab118673
Branch: refs/heads/jira/solr-11779
Commit: ab11867364fb93305d3174e99d869c070c57023c
Parents: 9b26108
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu May 3 18:42:56 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu May 3 18:42:56 2018 +0100
----------------------------------------------------------------------
solr/solr-ref-guide/src/json-facet-api.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab118673/solr/solr-ref-guide/src/json-facet-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/json-facet-api.adoc b/solr/solr-ref-guide/src/json-facet-api.adoc
index e71c8bd..4f0ec7d 100644
--- a/solr/solr-ref-guide/src/json-facet-api.adoc
+++ b/solr/solr-ref-guide/src/json-facet-api.adoc
@@ -6,7 +6,7 @@
The new Facet & Analytics Module exposed via the JSON Facet API is a rewrite of Solr's previous faceting capabilities, with the following goals:
* First class native JSON API to control faceting and analytics
-** The structured nature of nested sub-facets are more naturally expressed in JSON rather than the flat nanemspace provided by normal query parameters.
+** The structured nature of nested sub-facets are more naturally expressed in JSON rather than the flat namespace provided by normal query parameters.
* First class integrated analytics support
* Nest any facet type under any other facet type (such as range facet, field facet, query facet)
* Ability to sort facet buckets by any calculated metric
[22/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8142: Fix
AssertingImpactsEnum and add missing javadoc.
Posted by ab...@apache.org.
LUCENE-8142: Fix AssertingImpactsEnum and add missing javadoc.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/46ecb739
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/46ecb739
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/46ecb739
Branch: refs/heads/jira/solr-11779
Commit: 46ecb739766a1a82b458b417e35f9c0936288e65
Parents: e00c4ce
Author: Adrien Grand <jp...@gmail.com>
Authored: Wed May 2 17:20:42 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Wed May 2 17:20:42 2018 +0200
----------------------------------------------------------------------
lucene/core/src/java/org/apache/lucene/index/Impacts.java | 3 +++
.../java/org/apache/lucene/search/DisjunctionMaxScorer.java | 1 +
.../java/org/apache/lucene/search/DisjunctionSumScorer.java | 1 +
.../core/src/java/org/apache/lucene/search/ReqExclScorer.java | 5 +++++
.../src/java/org/apache/lucene/search/ReqOptSumScorer.java | 7 +++++++
.../src/java/org/apache/lucene/search/AssertingScorer.java | 2 +-
6 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46ecb739/lucene/core/src/java/org/apache/lucene/index/Impacts.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/Impacts.java b/lucene/core/src/java/org/apache/lucene/index/Impacts.java
index ce3ece4..f6542b9 100644
--- a/lucene/core/src/java/org/apache/lucene/index/Impacts.java
+++ b/lucene/core/src/java/org/apache/lucene/index/Impacts.java
@@ -23,6 +23,9 @@ import java.util.List;
*/
public abstract class Impacts {
+ /** Sole constructor. Typically invoked by sub classes. */
+ protected Impacts() {}
+
/**
* Return the number of levels on which we have impacts.
* The returned value is always greater than 0 and may not always be the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46ecb739/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
index 3b86068..6d2f65c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
@@ -53,6 +53,7 @@ final class DisjunctionMaxScorer extends DisjunctionScorer {
float scoreMax = 0;
double otherScoreSum = 0;
for (Scorer scorer : subScorers) {
+ scorer.advanceShallow(0);
float subScore = scorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS);
if (subScore >= scoreMax) {
otherScoreSum += scoreMax;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46ecb739/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
index fa92fcd..4f45b07 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
@@ -36,6 +36,7 @@ final class DisjunctionSumScorer extends DisjunctionScorer {
super(weight, subScorers, needsScores);
double maxScore = 0;
for (Scorer scorer : subScorers) {
+ scorer.advanceShallow(0);
maxScore += scorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS);
}
// The error of sums depends on the order in which values are summed up. In
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46ecb739/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java b/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java
index 987293e..cee0a45 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java
@@ -77,6 +77,11 @@ class ReqExclScorer extends Scorer {
}
@Override
+ public int advanceShallow(int target) throws IOException {
+ return reqScorer.advanceShallow(target);
+ }
+
+ @Override
public float getMaxScore(int upTo) throws IOException {
return reqScorer.getMaxScore(upTo);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46ecb739/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java b/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
index 6d93a54..418ad99 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
@@ -52,6 +52,7 @@ class ReqOptSumScorer extends Scorer {
this.reqScorer = reqScorer;
this.optScorer = optScorer;
+ reqScorer.advanceShallow(0);
this.reqMaxScore = reqScorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS);
this.maxScorePropagator = new MaxScoreSumPropagator(Arrays.asList(reqScorer, optScorer));
@@ -210,6 +211,12 @@ class ReqOptSumScorer extends Scorer {
}
@Override
+ public int advanceShallow(int target) throws IOException {
+ optScorer.advanceShallow(target);
+ return reqScorer.advanceShallow(target);
+ }
+
+ @Override
public float getMaxScore(int upTo) throws IOException {
float maxScore = reqScorer.getMaxScore(upTo);
if (optScorer.docID() <= upTo) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46ecb739/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
index 00aee41..952687f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java
@@ -40,7 +40,7 @@ public class AssertingScorer extends Scorer {
IteratorState state = IteratorState.ITERATING;
int doc;
float minCompetitiveScore = 0;
- int lastShallowTarget;
+ int lastShallowTarget = -1;
private AssertingScorer(Random random, Scorer in, ScoreMode scoreMode) {
super(in.weight);
[18/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12278: Add
IgnoreLargeDocumentProcessFactory
Posted by ab...@apache.org.
SOLR-12278: Add IgnoreLargeDocumentProcessFactory
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/555b7ef2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/555b7ef2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/555b7ef2
Branch: refs/heads/jira/solr-11779
Commit: 555b7ef27049633eb91b7c0cf1ec08b24b2f8e41
Parents: ee2198d
Author: Cao Manh Dat <da...@apache.org>
Authored: Wed May 2 17:43:29 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Wed May 2 17:43:29 2018 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../IgnoreLargeDocumentProcessorFactory.java | 174 +++++++++++++++++++
...IgnoreLargeDocumentProcessorFactoryTest.java | 102 +++++++++++
3 files changed, 278 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/555b7ef2/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a3a6c37..f8261ea 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -108,6 +108,8 @@ New Features
* SOLR-8998: introducing uniqueBlock(_root_) aggregation as faster alternative to unique(_root_) for counting
child value facets in parents via json.facet on block index (Dr Oleg Savrasov, Mikhail Khludnev)
+* SOLR-11278: Add IgnoreLargeDocumentProcessFactory (Cao Manh Dat, David Smiley)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/555b7ef2/solr/core/src/java/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.java
new file mode 100644
index 0000000..1782438
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+
+import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
+
+/**
+ * <p>
+ * Gives system administrators a way to ignore very large update from clients.
+ * When an update goes through processors its size can change
+ * therefore this processor should be the last processor of the chain.
+ * </p>
+ * @since 7.4.0
+ */
+public class IgnoreLargeDocumentProcessorFactory extends UpdateRequestProcessorFactory {
+ public static final String LIMIT_SIZE_PARAM = "limit";
+
+ // limit of a SolrInputDocument size (in kb)
+ private long maxDocumentSize = 1024 * 1024;
+
+ @Override
+ public void init(NamedList args) {
+ maxDocumentSize = args.toSolrParams().required().getLong(LIMIT_SIZE_PARAM);
+ args.remove(LIMIT_SIZE_PARAM);
+
+ if (args.size() > 0) {
+ throw new SolrException(SERVER_ERROR,
+ "Unexpected init param(s): '" +
+ args.getName(0) + "'");
+ }
+ }
+
+ @Override
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ return new UpdateRequestProcessor(next) {
+ @Override
+ public void processAdd(AddUpdateCommand cmd) throws IOException {
+ long docSize = ObjectSizeEstimator.fastEstimate(cmd.getSolrInputDocument());
+ if (docSize / 1024 > maxDocumentSize) {
+ throw new SolrException(BAD_REQUEST, "Size of the document "+cmd.getPrintableId()+" is too large, around:"+docSize);
+ }
+ super.processAdd(cmd);
+ }
+ };
+ }
+
+ /**
+ * Util class for quickly estimate size of a {@link org.apache.solr.common.SolrInputDocument}
+ * Compare to {@link org.apache.lucene.util.RamUsageEstimator}, this class have some pros
+ * <ul>
+ * <li>does not use reflection
+ * <li>go as deep as needed to compute size of all {@link org.apache.solr.common.SolrInputField} and
+ * all {@link org.apache.solr.common.SolrInputDocument} children
+ * <li>compute size of String based on its length
+ * <li>fast estimate size of a {@link java.util.Map} or a {@link java.util.Collection}
+ * </ul>
+ */
+ // package private for testing
+ static class ObjectSizeEstimator {
+ /**
+ * Sizes of primitive classes.
+ */
+ private static final Map<Class<?>,Integer> primitiveSizes = new IdentityHashMap<>();
+ static {
+ primitiveSizes.put(boolean.class, 1);
+ primitiveSizes.put(Boolean.class, 1);
+ primitiveSizes.put(byte.class, 1);
+ primitiveSizes.put(Byte.class, 1);
+ primitiveSizes.put(char.class, Character.BYTES);
+ primitiveSizes.put(Character.class, Character.BYTES);
+ primitiveSizes.put(short.class, Short.BYTES);
+ primitiveSizes.put(Short.class, Short.BYTES);
+ primitiveSizes.put(int.class, Integer.BYTES);
+ primitiveSizes.put(Integer.class, Integer.BYTES);
+ primitiveSizes.put(float.class, Float.BYTES);
+ primitiveSizes.put(Float.class, Float.BYTES);
+ primitiveSizes.put(double.class, Double.BYTES);
+ primitiveSizes.put(Double.class, Double.BYTES);
+ primitiveSizes.put(long.class, Long.BYTES);
+ primitiveSizes.put(Long.class, Long.BYTES);
+ }
+
+ static long fastEstimate(SolrInputDocument doc) {
+ if (doc == null) return 0L;
+ long size = 0;
+ if (doc.getFieldNames() != null) {
+ for (String fieldName : doc.getFieldNames()) {
+ size += fastEstimate(fieldName) + fastEstimate(doc.getField(fieldName).getValue());
+ }
+ }
+ if (doc.hasChildDocuments()) {
+ for (SolrInputDocument childDoc : doc.getChildDocuments()) {
+ size += fastEstimate(childDoc);
+ }
+ }
+ return size;
+ }
+
+ static long fastEstimate(Object obj) {
+ if (obj == null) return 0;
+
+ long size = primitiveEstimate(obj, -1);
+ if (size != -1) return size;
+
+ if (obj instanceof Map) {
+ return fastEstimate((Map) obj);
+ }
+
+ if (obj instanceof Collection) {
+ return fastEstimate((Collection) obj);
+ }
+
+ return 0L;
+ }
+
+ private static long primitiveEstimate(Object obj, long def) {
+ Class<?> clazz = obj.getClass();
+ if (clazz.isPrimitive()) {
+ return primitiveSizes.get(clazz);
+ }
+ if (obj instanceof String) {
+ return ((String) obj).length() * Character.BYTES;
+ }
+ return def;
+ }
+
+ private static long fastEstimate(Map<Object, Object> map) {
+ if (map.isEmpty()) return 0;
+ long size = 0;
+ for (Map.Entry<Object, Object> entry : map.entrySet()) {
+ size += primitiveEstimate(entry.getKey(), 0L) + primitiveEstimate(entry.getValue(), 0L);
+ }
+ return size;
+ }
+
+ private static long fastEstimate(Collection collection) {
+ if (collection.isEmpty()) return 0;
+ long size = 0;
+ for (Object obj : collection) {
+ size += primitiveEstimate(obj, 0L);
+ }
+ return size;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/555b7ef2/solr/core/src/test/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactoryTest.java
new file mode 100644
index 0000000..da70fc6
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactoryTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.update.AddUpdateCommand;
+import org.junit.Test;
+
+import static org.apache.solr.update.processor.IgnoreLargeDocumentProcessorFactory.ObjectSizeEstimator.fastEstimate;
+
+public class IgnoreLargeDocumentProcessorFactoryTest extends LuceneTestCase {
+
+ @Test
+ public void testProcessor() throws IOException {
+ NamedList args = new NamedList();
+ args.add(IgnoreLargeDocumentProcessorFactory.LIMIT_SIZE_PARAM, 1);
+
+ IgnoreLargeDocumentProcessorFactory factory = new IgnoreLargeDocumentProcessorFactory();
+ factory.init(args);
+ try {
+ UpdateRequestProcessor processor = factory.getInstance(null, null, null);
+ processor.processAdd(getUpdate(1024));
+ fail("Expected processor to ignore the update");
+ } catch (SolrException e) {
+ //expected
+ }
+
+ args = new NamedList();
+ args.add(IgnoreLargeDocumentProcessorFactory.LIMIT_SIZE_PARAM, 2);
+ factory = new IgnoreLargeDocumentProcessorFactory();
+ factory.init(args);
+ UpdateRequestProcessor processor = factory.getInstance(null, null, null);
+ processor.processAdd(getUpdate(1024));
+
+ }
+
+ public AddUpdateCommand getUpdate(int size) {
+ SolrInputDocument document = new SolrInputDocument();
+ document.addField(new String(new byte[size], Charset.defaultCharset()), 1L);
+ assertTrue(fastEstimate(document) > size);
+
+ AddUpdateCommand cmd = new AddUpdateCommand(null);
+ cmd.solrDoc = document;
+ return cmd;
+ }
+
+ @Test
+ public void testEstimateObjectSize() {
+ assertEquals(fastEstimate("abc"), 6);
+ assertEquals(fastEstimate("abcdefgh"), 16);
+ List<String> keys = Arrays.asList("int", "long", "double", "float", "str");
+ assertEquals(fastEstimate(keys), 42);
+ List<Object> values = Arrays.asList(12, 5L, 12.0, 5.0, "duck");
+ assertEquals(fastEstimate(values), 8);
+
+ Map<String, Object> map = new HashMap<>();
+ map.put("int", 12);
+ map.put("long", 5L);
+ map.put("double", 12.0);
+ map.put("float", 5.0f);
+ map.put("str", "duck");
+ assertEquals(fastEstimate(map), 50);
+
+ SolrInputDocument document = new SolrInputDocument();
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ document.addField(entry.getKey(), entry.getValue());
+ }
+ assertEquals(fastEstimate(document), fastEstimate(map));
+
+ SolrInputDocument childDocument = new SolrInputDocument();
+ for (Map.Entry<String, Object> entry : map.entrySet()) {
+ childDocument.addField(entry.getKey(), entry.getValue());
+ }
+ document.addChildDocument(childDocument);
+ assertEquals(fastEstimate(document), fastEstimate(map) * 2);
+ }
+}
[46/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12279: Reject
invalid 'blockUnknown' values for 'bin/solr auth'
Posted by ab...@apache.org.
SOLR-12279: Reject invalid 'blockUnknown' values for 'bin/solr auth'
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6521d86d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6521d86d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6521d86d
Branch: refs/heads/jira/solr-11779
Commit: 6521d86dded91e6666f8d85c22076a748b6c2525
Parents: 445c0aa
Author: Jason Gerlowski <ge...@apache.org>
Authored: Mon May 7 07:01:01 2018 -0400
Committer: Jason Gerlowski <ge...@apache.org>
Committed: Mon May 7 07:41:13 2018 -0400
----------------------------------------------------------------------
solr/bin-test/test_auth.sh | 40 ++++++++++++++++++++
solr/bin-test/utils/assert.sh | 2 +-
.../src/java/org/apache/solr/util/SolrCLI.java | 15 ++++++++
3 files changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6521d86d/solr/bin-test/test_auth.sh
----------------------------------------------------------------------
diff --git a/solr/bin-test/test_auth.sh b/solr/bin-test/test_auth.sh
new file mode 100644
index 0000000..b31a996
--- /dev/null
+++ b/solr/bin-test/test_auth.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source bin-test/utils/assert.sh
+
+function solr_unit_test_before() {
+ bin/solr auth disable > /dev/null 2>&1
+}
+
+function solr_test_auth_rejects_blockUnknown_option_with_invalid_boolean() {
+ local auth_cmd="bin/solr auth enable -type basicAuth -credentials anyUser:anyPass -blockUnknown ture"
+ local expected_output="Argument [blockUnknown] must be either true or false, but was [ture]"
+ local actual_output; actual_output=$($auth_cmd)
+
+ assert_cmd_failed "$auth_cmd" || return 1
+ assert_output_contains "$actual_output" "$expected_output" || return 1
+}
+
+function solr_test_auth_rejects_updateIncludeFileOnly_option_with_invalid_boolean() {
+ local auth_cmd="bin/solr auth enable -type basicAuth -credentials anyUser:anyPass -updateIncludeFileOnly ture"
+ local expected_output="Argument [updateIncludeFileOnly] must be either true or false, but was [ture]"
+ local actual_output; actual_output=$($auth_cmd)
+
+ assert_cmd_failed "$auth_cmd" || return 1
+ assert_output_contains "$actual_output" "$expected_output" || return 1
+}
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6521d86d/solr/bin-test/utils/assert.sh
----------------------------------------------------------------------
diff --git a/solr/bin-test/utils/assert.sh b/solr/bin-test/utils/assert.sh
index a4bd56d..3f83b43 100644
--- a/solr/bin-test/utils/assert.sh
+++ b/solr/bin-test/utils/assert.sh
@@ -46,7 +46,7 @@ function assert_output_contains() {
local actual_output="$1"
local needle="$2"
- if echo "$actual_output" | grep -q "$needle"; then
+ if [[ "$actual_output" == *"$needle"* ]]; then
return $ASSERT_SUCCESS
fi
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6521d86d/solr/core/src/java/org/apache/solr/util/SolrCLI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
index 784ccbe..9c7970e 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
@@ -72,6 +72,7 @@ import org.apache.commons.exec.Executor;
import org.apache.commons.exec.OS;
import org.apache.commons.exec.environment.EnvironmentUtils;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.SystemUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@@ -3781,6 +3782,17 @@ public class SolrCLI {
};
}
+ private void ensureArgumentIsValidBooleanIfPresent(CommandLine cli, String argName) {
+ if (cli.hasOption(argName)) {
+ final String value = cli.getOptionValue(argName);
+ final Boolean parsedBoolean = BooleanUtils.toBooleanObject(value);
+ if (parsedBoolean == null) {
+ echo("Argument [" + argName + "] must be either true or false, but was [" + value + "]");
+ exit(1);
+ }
+ }
+ }
+
@Override
public int runTool(CommandLine cli) throws Exception {
raiseLogLevelUnlessVerbose(cli);
@@ -3789,6 +3801,9 @@ public class SolrCLI {
return 1;
}
+ ensureArgumentIsValidBooleanIfPresent(cli, "blockUnknown");
+ ensureArgumentIsValidBooleanIfPresent(cli, "updateIncludeFileOnly");
+
String type = cli.getOptionValue("type", "basicAuth");
switch (type) {
case "basicAuth":
[48/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12312:
Replication's IndexFetcher buf size should be initialized to an amount no
greater than the size of the file being transferred.
Posted by ab...@apache.org.
SOLR-12312: Replication's IndexFetcher buf size should be initialized
to an amount no greater than the size of the file being transferred.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/81f61120
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/81f61120
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/81f61120
Branch: refs/heads/jira/solr-11779
Commit: 81f611209c9b71b7538a46d047631ea164dc2a2c
Parents: b0b3293
Author: David Smiley <ds...@apache.org>
Authored: Mon May 7 14:54:11 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Mon May 7 14:54:11 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../org/apache/solr/handler/IndexFetcher.java | 40 ++++++--------------
2 files changed, 14 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/81f61120/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 48f9ad5..abc6d6c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -232,6 +232,9 @@ Optimizations
* SOLR-12066: Cleanup deleted core when node start (Cao Manh Dat)
+* SOLR-12312: Replication IndexFetcher should cap its internal buffer size when the file being transferred is small.
+ (Jeff Miller, David Smiley)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/81f61120/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 93f0edf..d26a0db 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -16,31 +16,6 @@
*/
package org.apache.solr.handler;
-import static org.apache.solr.common.params.CommonParams.JAVABIN;
-import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.handler.ReplicationHandler.ALIAS;
-import static org.apache.solr.handler.ReplicationHandler.CHECKSUM;
-import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS;
-import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE;
-import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST;
-import static org.apache.solr.handler.ReplicationHandler.CMD_INDEX_VERSION;
-import static org.apache.solr.handler.ReplicationHandler.COMMAND;
-import static org.apache.solr.handler.ReplicationHandler.COMPRESSION;
-import static org.apache.solr.handler.ReplicationHandler.CONF_FILES;
-import static org.apache.solr.handler.ReplicationHandler.CONF_FILE_SHORT;
-import static org.apache.solr.handler.ReplicationHandler.EXTERNAL;
-import static org.apache.solr.handler.ReplicationHandler.FETCH_FROM_LEADER;
-import static org.apache.solr.handler.ReplicationHandler.FILE;
-import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
-import static org.apache.solr.handler.ReplicationHandler.GENERATION;
-import static org.apache.solr.handler.ReplicationHandler.INTERNAL;
-import static org.apache.solr.handler.ReplicationHandler.MASTER_URL;
-import static org.apache.solr.handler.ReplicationHandler.OFFSET;
-import static org.apache.solr.handler.ReplicationHandler.SIZE;
-import static org.apache.solr.handler.ReplicationHandler.SKIP_COMMIT_ON_MASTER_VERSION_ZERO;
-import static org.apache.solr.handler.ReplicationHandler.TLOG_FILE;
-import static org.apache.solr.handler.ReplicationHandler.TLOG_FILES;
-
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -79,6 +54,7 @@ import java.util.zip.Adler32;
import java.util.zip.Checksum;
import java.util.zip.InflaterInputStream;
+import com.google.common.base.Strings;
import org.apache.http.client.HttpClient;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.IndexCommit;
@@ -111,7 +87,7 @@ import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.ReplicationHandler.FileInfo;
+import org.apache.solr.handler.ReplicationHandler.*;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher;
@@ -128,7 +104,9 @@ import org.apache.solr.util.TestInjection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Strings;
+import static org.apache.solr.common.params.CommonParams.JAVABIN;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.handler.ReplicationHandler.*;
/**
* <p> Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the
@@ -1539,7 +1517,7 @@ public class IndexFetcher {
private final long size;
private long bytesDownloaded = 0;
- private byte[] buf = new byte[1024 * 1024];
+ private byte[] buf;
private final Checksum checksum;
private int errorCount = 0;
private boolean aborted = false;
@@ -1549,6 +1527,7 @@ public class IndexFetcher {
this.file = file;
this.fileName = (String) fileDetails.get(NAME);
this.size = (Long) fileDetails.get(SIZE);
+ buf = new byte[(int)Math.min(this.size, ReplicationHandler.PACKET_SZ)];
this.solrParamOutput = solrParamOutput;
this.saveAs = saveAs;
indexGen = latestGen;
@@ -1629,8 +1608,11 @@ public class IndexFetcher {
LOG.warn("No content received for file: {}", fileName);
return NO_CONTENT;
}
- if (buf.length < packetSize)
+ //TODO consider recoding the remaining logic to not use/need buf[]; instead use the internal buffer of fis
+ if (buf.length < packetSize) {
+ //This shouldn't happen since sender should use PACKET_SZ and we init the buf based on that too
buf = new byte[packetSize];
+ }
if (checksum != null) {
//read the checksum
fis.readFully(longbytes);
[13/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8280: Reorganize
to allow us to try lots of strategies until we get one.
Posted by ab...@apache.org.
LUCENE-8280: Reorganize to allow us to try lots of strategies until we get one.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ff68acf2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ff68acf2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ff68acf2
Branch: refs/heads/jira/solr-11779
Commit: ff68acf2449f0f705a949e7afb592c4139fd52ad
Parents: 570fff8
Author: Karl Wright <Da...@gmail.com>
Authored: Mon Apr 30 06:12:31 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Mon Apr 30 06:12:31 2018 -0400
----------------------------------------------------------------------
.../spatial3d/geom/GeoComplexPolygon.java | 284 +++++++++----------
1 file changed, 136 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ff68acf2/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
index 4b82897..25e1d67 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
@@ -21,6 +21,7 @@ import java.util.List;
import java.util.ArrayList;
import java.util.Set;
import java.util.HashSet;
+import java.util.Collections;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
@@ -382,20 +383,8 @@ class GeoComplexPolygon extends GeoBasePolygon {
// Find the intersection points for each one of these and the complementary test point planes.
- // There will be multiple intersection points found. We choose the one that has the lowest total distance, as measured in delta X, delta Y, and delta Z.
- double bestDistance = Double.POSITIVE_INFINITY;
- double firstLegValue = 0.0;
- double secondLegValue = 0.0;
- Plane firstLegPlane = null;
- Plane firstLegAbovePlane = null;
- Plane firstLegBelowPlane = null;
- Plane secondLegPlane = null;
- Plane secondLegAbovePlane = null;
- Plane secondLegBelowPlane = null;
- Tree firstLegTree = null;
- Tree secondLegTree = null;
- GeoPoint intersectionPoint = null;
-
+ final List<TraversalStrategy> traversalStrategies = new ArrayList<>(12);
+
if (testPointFixedYAbovePlane != null && testPointFixedYBelowPlane != null && fixedXAbovePlane != null && fixedXBelowPlane != null) {
//check if planes intersects inside world
final double checkAbove = 4.0 * (fixedXAbovePlane.D * fixedXAbovePlane.D * planetModel.inverseAbSquared + testPointFixedYAbovePlane.D * testPointFixedYAbovePlane.D * planetModel.inverseAbSquared - 1.0);
@@ -414,21 +403,10 @@ class GeoComplexPolygon extends GeoBasePolygon {
final double newDistance = tpDelta1 * tpDelta1 + tpDelta2 * tpDelta2 + cpDelta1 * cpDelta1 + cpDelta2 * cpDelta2;
//final double newDistance = (testPoint.x - p.x) * (testPoint.x - p.x) + (testPoint.z - p.z) * (testPoint.z - p.z) + (thePoint.y - p.y) * (thePoint.y - p.y) + (thePoint.z - p.z) * (thePoint.z - p.z);
//final double newDistance = Math.abs(testPoint.x - p.x) + Math.abs(thePoint.y - p.y);
- if (newDistance < bestDistance) {
- //System.out.println(" Picking YZ then XZ");
- bestDistance = newDistance;
- firstLegValue = testPoint.y;
- secondLegValue = x;
- firstLegPlane = testPointFixedYPlane;
- firstLegAbovePlane = testPointFixedYAbovePlane;
- firstLegBelowPlane = testPointFixedYBelowPlane;
- secondLegPlane = travelPlaneFixedX;
- secondLegAbovePlane = fixedXAbovePlane;
- secondLegBelowPlane = fixedXBelowPlane;
- firstLegTree = yTree;
- secondLegTree = xTree;
- intersectionPoint = p;
- }
+ traversalStrategies.add(new TraversalStrategy(newDistance, testPoint.y, x,
+ testPointFixedYPlane, testPointFixedYAbovePlane, testPointFixedYBelowPlane,
+ travelPlaneFixedX, fixedXAbovePlane, fixedXBelowPlane,
+ yTree, xTree, p));
}
}
}
@@ -449,21 +427,10 @@ class GeoComplexPolygon extends GeoBasePolygon {
final double newDistance = tpDelta1 * tpDelta1 + tpDelta2 * tpDelta2 + cpDelta1 * cpDelta1 + cpDelta2 * cpDelta2;
//final double newDistance = (testPoint.x - p.x) * (testPoint.x - p.x) + (testPoint.y - p.y) * (testPoint.y - p.y) + (thePoint.y - p.y) * (thePoint.y - p.y) + (thePoint.z - p.z) * (thePoint.z - p.z);
//final double newDistance = Math.abs(testPoint.x - p.x) + Math.abs(thePoint.z - p.z);
- if (newDistance < bestDistance) {
- //System.out.println(" Picking YZ then XY");
- bestDistance = newDistance;
- firstLegValue = testPoint.z;
- secondLegValue = x;
- firstLegPlane = testPointFixedZPlane;
- firstLegAbovePlane = testPointFixedZAbovePlane;
- firstLegBelowPlane = testPointFixedZBelowPlane;
- secondLegPlane = travelPlaneFixedX;
- secondLegAbovePlane = fixedXAbovePlane;
- secondLegBelowPlane = fixedXBelowPlane;
- firstLegTree = zTree;
- secondLegTree = xTree;
- intersectionPoint = p;
- }
+ traversalStrategies.add(new TraversalStrategy(newDistance, testPoint.z, x,
+ testPointFixedZPlane, testPointFixedZAbovePlane, testPointFixedZBelowPlane,
+ travelPlaneFixedX, fixedXAbovePlane, fixedXBelowPlane,
+ zTree, xTree, p));
}
}
}
@@ -484,21 +451,10 @@ class GeoComplexPolygon extends GeoBasePolygon {
final double newDistance = tpDelta1 * tpDelta1 + tpDelta2 * tpDelta2 + cpDelta1 * cpDelta1 + cpDelta2 * cpDelta2;
//final double newDistance = (testPoint.y - p.y) * (testPoint.y - p.y) + (testPoint.z - p.z) * (testPoint.z - p.z) + (thePoint.x - p.x) * (thePoint.x - p.x) + (thePoint.z - p.z) * (thePoint.z - p.z);
//final double newDistance = Math.abs(testPoint.y - p.y) + Math.abs(thePoint.x - p.x);
- if (newDistance < bestDistance) {
- //System.out.println(" Picking XZ then YZ");
- bestDistance = newDistance;
- firstLegValue = testPoint.x;
- secondLegValue = y;
- firstLegPlane = testPointFixedXPlane;
- firstLegAbovePlane = testPointFixedXAbovePlane;
- firstLegBelowPlane = testPointFixedXBelowPlane;
- secondLegPlane = travelPlaneFixedY;
- secondLegAbovePlane = fixedYAbovePlane;
- secondLegBelowPlane = fixedYBelowPlane;
- firstLegTree = xTree;
- secondLegTree = yTree;
- intersectionPoint = p;
- }
+ traversalStrategies.add(new TraversalStrategy(newDistance, testPoint.x, y,
+ testPointFixedXPlane, testPointFixedXAbovePlane, testPointFixedXBelowPlane,
+ travelPlaneFixedY, fixedYAbovePlane, fixedYBelowPlane,
+ xTree, yTree, p));
}
}
}
@@ -519,21 +475,10 @@ class GeoComplexPolygon extends GeoBasePolygon {
final double newDistance = tpDelta1 * tpDelta1 + tpDelta2 * tpDelta2 + cpDelta1 * cpDelta1 + cpDelta2 * cpDelta2;
//final double newDistance = (testPoint.x - p.x) * (testPoint.x - p.x) + (testPoint.y - p.y) * (testPoint.y - p.y) + (thePoint.x - p.x) * (thePoint.x - p.x) + (thePoint.z - p.z) * (thePoint.z - p.z);
//final double newDistance = Math.abs(testPoint.y - p.y) + Math.abs(thePoint.z - p.z);
- if (newDistance < bestDistance) {
- //System.out.println(" Picking XZ then XY");
- bestDistance = newDistance;
- firstLegValue = testPoint.z;
- secondLegValue = y;
- firstLegPlane = testPointFixedZPlane;
- firstLegAbovePlane = testPointFixedZAbovePlane;
- firstLegBelowPlane = testPointFixedZBelowPlane;
- secondLegPlane = travelPlaneFixedY;
- secondLegAbovePlane = fixedYAbovePlane;
- secondLegBelowPlane = fixedYBelowPlane;
- firstLegTree = zTree;
- secondLegTree = yTree;
- intersectionPoint = p;
- }
+ traversalStrategies.add(new TraversalStrategy(newDistance, testPoint.z, y,
+ testPointFixedZPlane, testPointFixedZAbovePlane, testPointFixedZBelowPlane,
+ travelPlaneFixedY, fixedYAbovePlane, fixedYBelowPlane,
+ zTree, yTree, p));
}
}
}
@@ -554,21 +499,10 @@ class GeoComplexPolygon extends GeoBasePolygon {
final double newDistance = tpDelta1 * tpDelta1 + tpDelta2 * tpDelta2 + cpDelta1 * cpDelta1 + cpDelta2 * cpDelta2;
//final double newDistance = (testPoint.y - p.y) * (testPoint.y - p.y) + (testPoint.z - p.z) * (testPoint.z - p.z) + (thePoint.y - p.y) * (thePoint.y - p.y) + (thePoint.x - p.x) * (thePoint.x - p.x);
//final double newDistance = Math.abs(testPoint.z - p.z) + Math.abs(thePoint.x - p.x);
- if (newDistance < bestDistance) {
- //System.out.println(" Picking XY then YZ");
- bestDistance = newDistance;
- firstLegValue = testPoint.x;
- secondLegValue = z;
- firstLegPlane = testPointFixedXPlane;
- firstLegAbovePlane = testPointFixedXAbovePlane;
- firstLegBelowPlane = testPointFixedXBelowPlane;
- secondLegPlane = travelPlaneFixedZ;
- secondLegAbovePlane = fixedZAbovePlane;
- secondLegBelowPlane = fixedZBelowPlane;
- firstLegTree = xTree;
- secondLegTree = zTree;
- intersectionPoint = p;
- }
+ traversalStrategies.add(new TraversalStrategy(newDistance, testPoint.x, z,
+ testPointFixedXPlane, testPointFixedXAbovePlane, testPointFixedXBelowPlane,
+ travelPlaneFixedZ, fixedZAbovePlane, fixedZBelowPlane,
+ xTree, zTree, p));
}
}
}
@@ -589,74 +523,33 @@ class GeoComplexPolygon extends GeoBasePolygon {
final double newDistance = tpDelta1 * tpDelta1 + tpDelta2 * tpDelta2 + cpDelta1 * cpDelta1 + cpDelta2 * cpDelta2;
//final double newDistance = (testPoint.x - p.x) * (testPoint.x - p.x) + (testPoint.z - p.z) * (testPoint.z - p.z) + (thePoint.y - p.y) * (thePoint.y - p.y) + (thePoint.x - p.x) * (thePoint.x - p.x);
//final double newDistance = Math.abs(testPoint.z - p.z) + Math.abs(thePoint.y - p.y);
- if (newDistance < bestDistance) {
- //System.out.println(" Picking XY then XZ");
- bestDistance = newDistance;
- firstLegValue = testPoint.y;
- secondLegValue = z;
- firstLegPlane = testPointFixedYPlane;
- firstLegAbovePlane = testPointFixedYAbovePlane;
- firstLegBelowPlane = testPointFixedYBelowPlane;
- secondLegPlane = travelPlaneFixedZ;
- secondLegAbovePlane = fixedZAbovePlane;
- secondLegBelowPlane = fixedZBelowPlane;
- firstLegTree = yTree;
- secondLegTree = zTree;
- intersectionPoint = p;
- }
+ traversalStrategies.add(new TraversalStrategy(newDistance, testPoint.y, z,
+ testPointFixedYPlane, testPointFixedYAbovePlane, testPointFixedYBelowPlane,
+ travelPlaneFixedZ, fixedZAbovePlane, fixedZBelowPlane,
+ yTree, zTree, p));
}
}
}
- assert bestDistance > 0.0 : "Best distance should not be zero unless on single plane";
- assert bestDistance < Double.POSITIVE_INFINITY : "Couldn't find an intersection point of any kind";
-
- // First, try with two individual legs. If that doesn't work, try the DualCrossingIterator.
- try {
- // First, we'll determine if the intersection point is in set or not
- //System.out.println(" Finding whether "+intersectionPoint+" is in-set, based on travel from "+testPoint+" along "+firstLegPlane+" (value="+firstLegValue+")");
- final CountingEdgeIterator testPointEdgeIterator = createLinearCrossingEdgeIterator(testPoint,
- firstLegPlane, firstLegAbovePlane, firstLegBelowPlane,
- intersectionPoint.x, intersectionPoint.y, intersectionPoint.z);
- // Traverse our way from the test point to the check point. Use the z tree because that's fixed.
- firstLegTree.traverse(testPointEdgeIterator, firstLegValue);
- final boolean intersectionPointOnEdge = testPointEdgeIterator.isOnEdge();
- // If the intersection point is on the edge, we cannot use this combination of legs, since it's not logically possible to compute in-set or out-of-set
- // with such a starting point.
- if (intersectionPointOnEdge) {
- throw new IllegalArgumentException("Intersection point landed on an edge -- illegal path");
- }
- final boolean intersectionPointInSet = intersectionPointOnEdge || (((testPointEdgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet);
-
- //System.out.println(" Intersection point in-set? "+intersectionPointInSet+" On edge? "+intersectionPointOnEdge);
+ Collections.sort(traversalStrategies);
+
+ if (traversalStrategies.size() == 0) {
+ throw new IllegalArgumentException("No dual-plane travel strategies were found");
+ }
- // Now do the final leg
- //System.out.println(" Finding whether ["+x+","+y+","+z+"] is in-set, based on travel from "+intersectionPoint+" along "+secondLegPlane+" (value="+secondLegValue+")");
- final CountingEdgeIterator travelEdgeIterator = createLinearCrossingEdgeIterator(intersectionPoint,
- secondLegPlane, secondLegAbovePlane, secondLegBelowPlane,
- x, y, z);
- // Traverse our way from the test point to the check point.
- secondLegTree.traverse(travelEdgeIterator, secondLegValue);
- final boolean rval = travelEdgeIterator.isOnEdge() || (((travelEdgeIterator.getCrossingCount() & 1) == 0)?intersectionPointInSet:!intersectionPointInSet);
-
- //System.out.println(" Check point in set? "+rval);
- return rval;
- } catch (IllegalArgumentException e) {
- // Intersection point apparently was on edge, so try another strategy
- final CountingEdgeIterator edgeIterator = new DualCrossingEdgeIterator(testPoint,
- firstLegPlane, firstLegAbovePlane, firstLegBelowPlane,
- secondLegPlane, secondLegAbovePlane, secondLegBelowPlane,
- x, y, z, intersectionPoint);
- firstLegTree.traverse(edgeIterator, firstLegValue);
- if (edgeIterator.isOnEdge()) {
- return true;
+ // Loop through travel strategies, in order, until we find one that works.
+ for (final TraversalStrategy ts : traversalStrategies) {
+ try {
+ return ts.apply(testPoint, testPointInSet, x, y, z);
+ } catch (IllegalArgumentException e) {
+ // Continue
}
- secondLegTree.traverse(edgeIterator, secondLegValue);
- return edgeIterator.isOnEdge() || (((edgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet);
}
+
+ throw new IllegalArgumentException("Exhausted all traversal strategies");
}
}
-
+
@Override
public GeoPoint[] getEdgePoints() {
return edgePoints;
@@ -824,6 +717,101 @@ class GeoComplexPolygon extends GeoBasePolygon {
// Hashcode and equals are system default!!
}
+ /** Strategy class for describing traversals.
+ * Implements Comparable so that these can be ordered by Collections.sort().
+ */
+ private class TraversalStrategy implements Comparable<TraversalStrategy> {
+ private final double traversalDistance;
+ private final double firstLegValue;
+ private final double secondLegValue;
+ private final Plane firstLegPlane;
+ private final Plane firstLegAbovePlane;
+ private final Plane firstLegBelowPlane;
+ private final Plane secondLegPlane;
+ private final Plane secondLegAbovePlane;
+ private final Plane secondLegBelowPlane;
+ private final Tree firstLegTree;
+ private final Tree secondLegTree;
+ private final GeoPoint intersectionPoint;
+
+ public TraversalStrategy(final double traversalDistance, final double firstLegValue, final double secondLegValue,
+ final Plane firstLegPlane, final Plane firstLegAbovePlane, final Plane firstLegBelowPlane,
+ final Plane secondLegPlane, final Plane secondLegAbovePlane, final Plane secondLegBelowPlane,
+ final Tree firstLegTree, final Tree secondLegTree,
+ final GeoPoint intersectionPoint) {
+ this.traversalDistance = traversalDistance;
+ this.firstLegValue = firstLegValue;
+ this.secondLegValue = secondLegValue;
+ this.firstLegPlane = firstLegPlane;
+ this.firstLegAbovePlane = firstLegAbovePlane;
+ this.firstLegBelowPlane = firstLegBelowPlane;
+ this.secondLegPlane = secondLegPlane;
+ this.secondLegAbovePlane = secondLegAbovePlane;
+ this.secondLegBelowPlane = secondLegBelowPlane;
+ this.firstLegTree = firstLegTree;
+ this.secondLegTree = secondLegTree;
+ this.intersectionPoint = intersectionPoint;
+ }
+
+ public boolean apply(final GeoPoint testPoint, final boolean testPointInSet,
+ final double x, final double y, final double z) {
+ // First, try with two individual legs. If that doesn't work, try the DualCrossingIterator.
+ try {
+ // First, we'll determine if the intersection point is in set or not
+ //System.out.println(" Finding whether "+intersectionPoint+" is in-set, based on travel from "+testPoint+" along "+firstLegPlane+" (value="+firstLegValue+")");
+ final CountingEdgeIterator testPointEdgeIterator = createLinearCrossingEdgeIterator(testPoint,
+ firstLegPlane, firstLegAbovePlane, firstLegBelowPlane,
+ intersectionPoint.x, intersectionPoint.y, intersectionPoint.z);
+ // Traverse our way from the test point to the check point. Use the z tree because that's fixed.
+ firstLegTree.traverse(testPointEdgeIterator, firstLegValue);
+ final boolean intersectionPointOnEdge = testPointEdgeIterator.isOnEdge();
+ // If the intersection point is on the edge, we cannot use this combination of legs, since it's not logically possible to compute in-set or out-of-set
+ // with such a starting point.
+ if (intersectionPointOnEdge) {
+ throw new IllegalArgumentException("Intersection point landed on an edge -- illegal path");
+ }
+ final boolean intersectionPointInSet = intersectionPointOnEdge || (((testPointEdgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet);
+
+ //System.out.println(" Intersection point in-set? "+intersectionPointInSet+" On edge? "+intersectionPointOnEdge);
+
+ // Now do the final leg
+ //System.out.println(" Finding whether ["+x+","+y+","+z+"] is in-set, based on travel from "+intersectionPoint+" along "+secondLegPlane+" (value="+secondLegValue+")");
+ final CountingEdgeIterator travelEdgeIterator = createLinearCrossingEdgeIterator(intersectionPoint,
+ secondLegPlane, secondLegAbovePlane, secondLegBelowPlane,
+ x, y, z);
+ // Traverse our way from the test point to the check point.
+ secondLegTree.traverse(travelEdgeIterator, secondLegValue);
+ final boolean rval = travelEdgeIterator.isOnEdge() || (((travelEdgeIterator.getCrossingCount() & 1) == 0)?intersectionPointInSet:!intersectionPointInSet);
+
+ //System.out.println(" Check point in set? "+rval);
+ return rval;
+ } catch (IllegalArgumentException e) {
+ // Intersection point apparently was on edge, so try another strategy
+ final CountingEdgeIterator edgeIterator = new DualCrossingEdgeIterator(testPoint,
+ firstLegPlane, firstLegAbovePlane, firstLegBelowPlane,
+ secondLegPlane, secondLegAbovePlane, secondLegBelowPlane,
+ x, y, z, intersectionPoint);
+ firstLegTree.traverse(edgeIterator, firstLegValue);
+ if (edgeIterator.isOnEdge()) {
+ return true;
+ }
+ secondLegTree.traverse(edgeIterator, secondLegValue);
+ return edgeIterator.isOnEdge() || (((edgeIterator.getCrossingCount() & 1) == 0)?testPointInSet:!testPointInSet);
+ }
+ }
+
+ @Override
+ public int compareTo(final TraversalStrategy other) {
+ if (traversalDistance < other.traversalDistance) {
+ return -1;
+ } else if (traversalDistance > other.traversalDistance) {
+ return 1;
+ }
+ return 0;
+ }
+
+ }
+
/**
* Iterator execution interface, for tree traversal. Pass an object implementing this interface
* into the traversal method of a tree, and each edge that matches will cause this object to be
[39/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12316: Do not allow
to use absolute URIs for including other files in solrconfig.xml and schema
parsing
Posted by ab...@apache.org.
SOLR-12316: Do not allow to use absolute URIs for including other files in solrconfig.xml and schema parsing
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1b760114
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1b760114
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1b760114
Branch: refs/heads/jira/solr-11779
Commit: 1b760114216fcdfae138a8b37f183a9293c49115
Parents: 89fc02a
Author: Uwe Schindler <us...@apache.org>
Authored: Sun May 6 14:21:34 2018 +0200
Committer: Uwe Schindler <us...@apache.org>
Committed: Sun May 6 14:21:34 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../org/apache/solr/util/SystemIdResolver.java | 14 ++++----------
.../apache/solr/util/TestSystemIdResolver.java | 19 +++++++++++++++++--
3 files changed, 24 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1b760114/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index efebe91..48f9ad5 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -212,6 +212,9 @@ Bug Fixes
* SOLR-12293: Updates need to use their own connection pool to maintain connection reuse and prevent spurious
recoveries. (Mark Miller)
+* SOLR-12316: Do not allow to use absolute URIs for including other files in solrconfig.xml and schema parsing.
+ (Ananthesh, Ishan Chattopadhyaya, Uwe Schindler)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1b760114/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java b/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java
index 6fda14f..c208520 100644
--- a/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java
+++ b/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java
@@ -16,9 +16,6 @@
*/
package org.apache.solr.util;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.lucene.analysis.util.ResourceLoader;
import org.xml.sax.InputSource;
@@ -26,7 +23,6 @@ import org.xml.sax.EntityResolver;
import org.xml.sax.ext.EntityResolver2;
import java.io.File;
import java.io.IOException;
-import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.net.URISyntaxException;
import javax.xml.transform.Source;
@@ -55,7 +51,6 @@ import javax.xml.stream.XMLStreamException;
* </pre>
*/
public final class SystemIdResolver implements EntityResolver, EntityResolver2 {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String RESOURCE_LOADER_URI_SCHEME = "solrres";
public static final String RESOURCE_LOADER_AUTHORITY_ABSOLUTE = "@";
@@ -126,8 +121,9 @@ public final class SystemIdResolver implements EntityResolver, EntityResolver2 {
@Override
public InputSource resolveEntity(String name, String publicId, String baseURI, String systemId) throws IOException {
- if (systemId == null)
+ if (systemId == null) {
return null;
+ }
try {
final URI uri = resolveRelativeURI(baseURI, systemId);
@@ -147,12 +143,10 @@ public final class SystemIdResolver implements EntityResolver, EntityResolver2 {
throw new IOException(re.getMessage(), re);
}
} else {
- // resolve all other URIs using the standard resolver
- return null;
+ throw new IOException("Cannot resolve absolute systemIDs / external entities (only relative paths work): " + systemId);
}
} catch (URISyntaxException use) {
- log.warn("An URI systax problem occurred during resolving SystemId, falling back to default resolver", use);
- return null;
+ throw new IOException("An URI syntax problem occurred during resolving systemId: " + systemId, use);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1b760114/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java b/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java
index 7980a59..4c2677d 100644
--- a/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java
+++ b/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java
@@ -17,6 +17,7 @@
package org.apache.solr.util;
import java.io.File;
+import java.io.IOException;
import java.nio.file.Path;
import org.apache.commons.io.IOUtils;
@@ -76,8 +77,22 @@ public class TestSystemIdResolver extends LuceneTestCase {
assertEntityResolving(resolver, SystemIdResolver.createSystemIdFromResourceName(testHome+"/crazy-path-to-schema.xml"),
SystemIdResolver.createSystemIdFromResourceName(testHome+"/crazy-path-to-config.xml"), "crazy-path-to-schema.xml");
- // test, that resolving works if somebody uses an absolute file:-URI in a href attribute, the resolver should return null (default fallback)
- assertNull(resolver.resolveEntity(null, null, "solrres:/solrconfig.xml", fileUri));
+ // if somebody uses an absolute uri (e.g., file://) we should fail resolving:
+ IOException ioe = expectThrows(IOException.class, () -> {
+ resolver.resolveEntity(null, null, "solrres:/solrconfig.xml", fileUri);
+ });
+ assertTrue(ioe.getMessage().startsWith("Cannot resolve absolute"));
+
+ ioe = expectThrows(IOException.class, () -> {
+ resolver.resolveEntity(null, null, "solrres:/solrconfig.xml", "http://lucene.apache.org/test.xml");
+ });
+ assertTrue(ioe.getMessage().startsWith("Cannot resolve absolute"));
+
+ // check that we can't escape with absolute file paths:
+ ioe = expectThrows(IOException.class, () -> {
+ resolver.resolveEntity(null, null, "solrres:/solrconfig.xml", "/etc/passwd");
+ });
+ assertTrue(ioe.getMessage().startsWith("Can't find resource '/etc/passwd' in classpath or"));
}
}
[34/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8293: Ensure only
hard deletes are carried over in a merge
Posted by ab...@apache.org.
LUCENE-8293: Ensure only hard deletes are carried over in a merge
Today we carry over hard deletes based on the SegmentReaders liveDocs.
This is not correct if soft-deletes are used especially with rentention
policies. If a soft delete is added while a segment is merged the document
might end up hard deleted in the target segment. This isn't necessarily a
correctness issue but causes unnecessary writes of hard-deletes. The biggest
issue here is that we assert that previously deleted documents are still deleted
in the live-docs we apply and that might be violated by the retention policy.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3a6f5313
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3a6f5313
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3a6f5313
Branch: refs/heads/jira/solr-11779
Commit: 3a6f5313d6b4a23dea2030cb5d63ad522536f501
Parents: b617489
Author: Simon Willnauer <si...@apache.org>
Authored: Thu May 3 09:29:12 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Fri May 4 12:16:03 2018 +0200
----------------------------------------------------------------------
.../org/apache/lucene/index/IndexWriter.java | 118 +++++++++++--------
.../org/apache/lucene/index/MergePolicy.java | 2 +
.../org/apache/lucene/index/PendingDeletes.java | 5 +-
.../apache/lucene/index/PendingSoftDeletes.java | 11 ++
.../apache/lucene/index/ReadersAndUpdates.java | 21 +++-
.../lucene/index/TestPendingSoftDeletes.java | 5 +
.../TestSoftDeletesRetentionMergePolicy.java | 93 +++++++++++++++
7 files changed, 204 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a6f5313/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index fa631d8..93f5446 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.IntPredicate;
import java.util.stream.Collectors;
import org.apache.lucene.analysis.Analyzer;
@@ -3621,64 +3622,19 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
SegmentCommitInfo info = sourceSegments.get(i);
minGen = Math.min(info.getBufferedDeletesGen(), minGen);
final int maxDoc = info.info.maxDoc();
- final Bits prevLiveDocs = merge.readers.get(i).getLiveDocs();
final ReadersAndUpdates rld = getPooledInstance(info, false);
// We hold a ref, from when we opened the readers during mergeInit, so it better still be in the pool:
assert rld != null: "seg=" + info.info.name;
- final Bits currentLiveDocs = rld.getLiveDocs();
MergeState.DocMap segDocMap = mergeState.docMaps[i];
MergeState.DocMap segLeafDocMap = mergeState.leafDocMaps[i];
- if (prevLiveDocs != null) {
-
- // If we had deletions on starting the merge we must
- // still have deletions now:
- assert currentLiveDocs != null;
- assert prevLiveDocs.length() == maxDoc;
- assert currentLiveDocs.length() == maxDoc;
-
- // There were deletes on this segment when the merge
- // started. The merge has collapsed away those
- // deletes, but, if new deletes were flushed since
- // the merge started, we must now carefully keep any
- // newly flushed deletes but mapping them to the new
- // docIDs.
-
- // Since we copy-on-write, if any new deletes were
- // applied after merging has started, we can just
- // check if the before/after liveDocs have changed.
- // If so, we must carefully merge the liveDocs one
- // doc at a time:
- if (currentLiveDocs != prevLiveDocs) {
- // This means this segment received new deletes
- // since we started the merge, so we
- // must merge them:
- for (int j = 0; j < maxDoc; j++) {
- if (prevLiveDocs.get(j) == false) {
- // if the document was deleted before, it better still be deleted!
- assert currentLiveDocs.get(j) == false;
- } else if (currentLiveDocs.get(j) == false) {
- // the document was deleted while we were merging:
- mergedDeletesAndUpdates.delete(segDocMap.get(segLeafDocMap.get(j)));
- }
- }
- }
- } else if (currentLiveDocs != null) {
- assert currentLiveDocs.length() == maxDoc;
- // This segment had no deletes before but now it
- // does:
- for (int j = 0; j < maxDoc; j++) {
- if (currentLiveDocs.get(j) == false) {
- mergedDeletesAndUpdates.delete(segDocMap.get(segLeafDocMap.get(j)));
- }
- }
- }
+ carryOverHardDeletes(mergedDeletesAndUpdates, maxDoc, mergeState.liveDocs[i], merge.hardLiveDocs.get(i), rld.getHardLiveDocs(),
+ segDocMap, segLeafDocMap);
// Now carry over all doc values updates that were resolved while we were merging, remapping the docIDs to the newly merged docIDs.
// We only carry over packets that finished resolving; if any are still running (concurrently) they will detect that our merge completed
// and re-resolve against the newly merged segment:
-
Map<String,List<DocValuesFieldUpdates>> mergingDVUpdates = rld.getMergingDVUpdates();
for (Map.Entry<String,List<DocValuesFieldUpdates>> ent : mergingDVUpdates.entrySet()) {
@@ -3757,6 +3713,69 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
return mergedDeletesAndUpdates;
}
+ /**
+ * This method carries over hard-deleted documents that are applied to the source segment during a merge.
+ */
+ private static void carryOverHardDeletes(ReadersAndUpdates mergedReadersAndUpdates, int maxDoc,
+ Bits mergeLiveDocs, // the liveDocs used to build the segDocMaps
+ Bits prevHardLiveDocs, // the hard deletes when the merge reader was pulled
+ Bits currentHardLiveDocs, // the current hard deletes
+ MergeState.DocMap segDocMap, MergeState.DocMap segLeafDocMap) throws IOException {
+
+ assert mergeLiveDocs == null || mergeLiveDocs.length() == maxDoc;
+ // if we mix soft and hard deletes we need to make sure that we only carry over deletes
+ // that were not deleted before. Otherwise the segDocMap doesn't contain a mapping.
+ // yet this is also required if any MergePolicy modifies the liveDocs since this is
+ // what the segDocMap is build on.
+ final IntPredicate carryOverDelete = mergeLiveDocs == null || mergeLiveDocs == prevHardLiveDocs
+ ? docId -> currentHardLiveDocs.get(docId) == false
+ : docId -> mergeLiveDocs.get(docId) && currentHardLiveDocs.get(docId) == false;
+ if (prevHardLiveDocs != null) {
+ // If we had deletions on starting the merge we must
+ // still have deletions now:
+ assert currentHardLiveDocs != null;
+ assert mergeLiveDocs != null;
+ assert prevHardLiveDocs.length() == maxDoc;
+ assert currentHardLiveDocs.length() == maxDoc;
+
+ // There were deletes on this segment when the merge
+ // started. The merge has collapsed away those
+ // deletes, but, if new deletes were flushed since
+ // the merge started, we must now carefully keep any
+ // newly flushed deletes but mapping them to the new
+ // docIDs.
+
+ // Since we copy-on-write, if any new deletes were
+ // applied after merging has started, we can just
+ // check if the before/after liveDocs have changed.
+ // If so, we must carefully merge the liveDocs one
+ // doc at a time:
+ if (currentHardLiveDocs != prevHardLiveDocs) {
+ // This means this segment received new deletes
+ // since we started the merge, so we
+ // must merge them:
+ for (int j = 0; j < maxDoc; j++) {
+ if (prevHardLiveDocs.get(j) == false) {
+ // if the document was deleted before, it better still be deleted!
+ assert currentHardLiveDocs.get(j) == false;
+ } else if (carryOverDelete.test(j)) {
+ // the document was deleted while we were merging:
+ mergedReadersAndUpdates.delete(segDocMap.get(segLeafDocMap.get(j)));
+ }
+ }
+ }
+ } else if (currentHardLiveDocs != null) {
+ assert currentHardLiveDocs.length() == maxDoc;
+ // This segment had no deletes before but now it
+ // does:
+ for (int j = 0; j < maxDoc; j++) {
+ if (carryOverDelete.test(j)) {
+ mergedReadersAndUpdates.delete(segDocMap.get(segLeafDocMap.get(j)));
+ }
+ }
+ }
+ }
+
@SuppressWarnings("try")
synchronized private boolean commitMerge(MergePolicy.OneMerge merge, MergeState mergeState) throws IOException {
@@ -4236,6 +4255,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
}
merge.readers = new ArrayList<>(sourceSegments.size());
+ merge.hardLiveDocs = new ArrayList<>(sourceSegments.size());
// This is try/finally to make sure merger's readers are
// closed:
@@ -4251,13 +4271,15 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
final ReadersAndUpdates rld = getPooledInstance(info, true);
rld.setIsMerging();
- SegmentReader reader = rld.getReaderForMerge(context);
+ ReadersAndUpdates.MergeReader mr = rld.getReaderForMerge(context);
+ SegmentReader reader = mr.reader;
int delCount = reader.numDeletedDocs();
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "seg=" + segString(info) + " reader=" + reader);
}
+ merge.hardLiveDocs.add(mr.hardLiveDocs);
merge.readers.add(reader);
assert delCount <= info.info.maxDoc(): "delCount=" + delCount + " info.maxDoc=" + info.info.maxDoc() + " rld.pendingDeleteCount=" + rld.getPendingDeleteCount() + " info.getDelCount()=" + info.getDelCount();
segUpto++;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a6f5313/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
index 029cca9..36ea33b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
@@ -33,6 +33,7 @@ import java.util.stream.Collectors;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MergeInfo;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOSupplier;
/**
@@ -205,6 +206,7 @@ public abstract class MergePolicy {
volatile long totalMergeBytes;
List<SegmentReader> readers; // used by IndexWriter
+ List<Bits> hardLiveDocs; // used by IndexWriter
/** Segments to be merged. */
public final List<SegmentCommitInfo> segments;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a6f5313/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java b/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
index 1878665..2dc0858 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
@@ -145,7 +145,6 @@ class PendingDeletes {
liveDocs = reader.getLiveDocs();
assert liveDocs == null || assertCheckLiveDocs(liveDocs, info.info.maxDoc(), info.getDelCount());
liveDocsShared = true;
-
}
liveDocsInitialized = true;
}
@@ -245,4 +244,8 @@ class PendingDeletes {
int numDeletesToMerge(MergePolicy policy, IOSupplier<CodecReader> readerIOSupplier) throws IOException {
return policy.numDeletesToMerge(info, numPendingDeletes(), readerIOSupplier);
}
+
+ Bits getHardLiveDocs() {
+ return liveDocs;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a6f5313/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java b/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
index eae25e0..41eebd2 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOSupplier;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.MutableBits;
@@ -193,4 +194,14 @@ final class PendingSoftDeletes extends PendingDeletes {
return fisFormat.read(dir, segInfo, segmentSuffix, IOContext.READONCE);
}
}
+
+ Bits getHardLiveDocs() {
+ return hardDeletes.getHardLiveDocs();
+ }
+
+ @Override
+ void liveDocsShared() {
+ super.liveDocsShared();
+ hardDeletes.liveDocsShared();
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a6f5313/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index 6e074ee..b33ac4b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -274,6 +274,13 @@ final class ReadersAndUpdates {
return pendingDeletes.getLiveDocs();
}
+ /**
+ * Returns the live-docs bits excluding documents that are not live due to soft-deletes
+ */
+ public synchronized Bits getHardLiveDocs() {
+ return pendingDeletes.getHardLiveDocs();
+ }
+
public synchronized void dropChanges() {
// Discard (don't save) changes when we are dropping
// the reader; this is used only on the sub-readers
@@ -687,8 +694,18 @@ final class ReadersAndUpdates {
return isMerging;
}
+ final static class MergeReader {
+ final SegmentReader reader;
+ final Bits hardLiveDocs;
+
+ MergeReader(SegmentReader reader, Bits hardLiveDocs) {
+ this.reader = reader;
+ this.hardLiveDocs = hardLiveDocs;
+ }
+ }
+
/** Returns a reader for merge, with the latest doc values updates and deletions. */
- synchronized SegmentReader getReaderForMerge(IOContext context) throws IOException {
+ synchronized MergeReader getReaderForMerge(IOContext context) throws IOException {
// We must carry over any still-pending DV updates because they were not
// successfully written, e.g. because there was a hole in the delGens,
@@ -715,7 +732,7 @@ final class ReadersAndUpdates {
markAsShared();
assert verifyDocCounts();
- return reader;
+ return new MergeReader(reader, pendingDeletes.getHardLiveDocs());
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a6f5313/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
index 7ddf0be..aeb5819 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
@@ -70,6 +70,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
assertTrue(pendingSoftDeletes.getLiveDocs().get(0));
assertFalse(pendingSoftDeletes.getLiveDocs().get(1));
assertTrue(pendingSoftDeletes.getLiveDocs().get(2));
+ assertNull(pendingSoftDeletes.getHardLiveDocs());
// pass reader again
Bits liveDocs = pendingSoftDeletes.getLiveDocs();
pendingSoftDeletes.liveDocsShared();
@@ -91,6 +92,10 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
assertFalse(pendingSoftDeletes.getLiveDocs().get(0));
assertFalse(pendingSoftDeletes.getLiveDocs().get(1));
assertTrue(pendingSoftDeletes.getLiveDocs().get(2));
+ assertNotNull(pendingSoftDeletes.getHardLiveDocs());
+ assertFalse(pendingSoftDeletes.getHardLiveDocs().get(0));
+ assertTrue(pendingSoftDeletes.getHardLiveDocs().get(1));
+ assertTrue(pendingSoftDeletes.getHardLiveDocs().get(2));
IOUtils.close(reader, writer, dir);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a6f5313/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
index 5f1ba6c..c9b22a5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
@@ -445,4 +445,97 @@ public class TestSoftDeletesRetentionMergePolicy extends LuceneTestCase {
writer.close();
dir.close();
}
+
+ public void testSoftDeleteWhileMergeSurvives() throws IOException {
+ Directory dir = newDirectory();
+ String softDelete = "soft_delete";
+ IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField(softDelete);
+ AtomicBoolean update = new AtomicBoolean(true);
+ config.setReaderPooling(true);
+ config.setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete", () -> new DocValuesFieldExistsQuery("keep"),
+ new LogDocMergePolicy()));
+ IndexWriter writer = new IndexWriter(dir, config);
+ writer.getConfig().setMergedSegmentWarmer(sr -> {
+ if (update.compareAndSet(true, false)) {
+ try {
+ writer.softUpdateDocument(new Term("id", "0"), new Document(),
+ new NumericDocValuesField(softDelete, 1), new NumericDocValuesField("keep", 1));
+ writer.commit();
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ }
+ });
+
+ boolean preExistingDeletes = random().nextBoolean();
+ for (int i = 0; i < 2; i++) {
+ Document d = new Document();
+ d.add(new StringField("id", Integer.toString(i), Field.Store.YES));
+ if (preExistingDeletes && random().nextBoolean()) {
+ writer.addDocument(d); // randomly add a preexisting hard-delete that we don't carry over
+ writer.deleteDocuments(new Term("id", Integer.toString(i)));
+ d.add(new NumericDocValuesField("keep", 1));
+ writer.addDocument(d);
+ } else {
+ d.add(new NumericDocValuesField("keep", 1));
+ writer.addDocument(d);
+ }
+ writer.flush();
+ }
+ writer.forceMerge(1);
+ writer.commit();
+ assertFalse(update.get());
+ DirectoryReader open = DirectoryReader.open(dir);
+ assertEquals(0, open.numDeletedDocs());
+ assertEquals(3, open.maxDoc());
+ IOUtils.close(open, writer, dir);
+ }
+
+ /*
+ * This test is trying to hard-delete a particular document while the segment is merged which is already soft-deleted
+ * This requires special logic inside IndexWriter#carryOverHardDeletes since docMaps are not created for this document.
+ */
+ public void testDeleteDocWhileMergeThatIsSoftDeleted() throws IOException {
+ Directory dir = newDirectory();
+ String softDelete = "soft_delete";
+ IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField(softDelete);
+ AtomicBoolean delete = new AtomicBoolean(true);
+ config.setReaderPooling(true);
+ config.setMergePolicy(new LogDocMergePolicy());
+ IndexWriter writer = new IndexWriter(dir, config);
+ Document d = new Document();
+ d.add(new StringField("id", "0", Field.Store.YES));
+ writer.addDocument(d);
+ d = new Document();
+ d.add(new StringField("id", "1", Field.Store.YES));
+ writer.addDocument(d);
+ if (random().nextBoolean()) {
+ // randomly run with a preexisting hard delete
+ d = new Document();
+ d.add(new StringField("id", "2", Field.Store.YES));
+ writer.addDocument(d);
+ writer.deleteDocuments(new Term("id", "2"));
+ }
+
+ writer.flush();
+ DirectoryReader reader = writer.getReader();
+ writer.softUpdateDocument(new Term("id", "0"), new Document(),
+ new NumericDocValuesField(softDelete, 1));
+ writer.flush();
+ writer.getConfig().setMergedSegmentWarmer(sr -> {
+ if (delete.compareAndSet(true, false)) {
+ try {
+ long seqNo = writer.tryDeleteDocument(reader, 0);
+ assertTrue("seqId was -1", seqNo != -1);
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ }
+ });
+ writer.forceMerge(1);
+ assertEquals(2, writer.numDocs());
+ assertEquals(2, writer.maxDoc());
+ assertFalse(delete.get());
+ IOUtils.close(reader, writer, dir);
+ }
}
[44/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8275: Suppress
WindowsFS TestDirectoryTaxonomyWriter
Posted by ab...@apache.org.
LUCENE-8275: Suppress WindowsFS TestDirectoryTaxonomyWriter
TestDirectoryTaxonomyWriter#testRecreateAndRefresh can't deal with pending
files since it creates multiple IW instances on the same directory.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f9942b52
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f9942b52
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f9942b52
Branch: refs/heads/jira/solr-11779
Commit: f9942b525be042982e12b202fda5f4bd29e6c5d1
Parents: 5fc7251
Author: Simon Willnauer <si...@apache.org>
Authored: Mon May 7 12:01:32 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon May 7 12:01:32 2018 +0200
----------------------------------------------------------------------
.../facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f9942b52/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
index 88fb042..4f3e1df 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyWriter.java
@@ -46,10 +46,12 @@ import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
import org.apache.lucene.util.TestUtil;
import org.junit.Test;
+@SuppressFileSystems("WindowsFS") // testRecreateAndRefresh doesn't like pending files
public class TestDirectoryTaxonomyWriter extends FacetTestCase {
// A No-Op TaxonomyWriterCache which always discards all given categories, and
[30/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8290: Keep soft
deletes in sync with on-disk DocValues
Posted by ab...@apache.org.
LUCENE-8290: Keep soft deletes in sync with on-disk DocValues
Today we pass on the doc values update to the PendingDeletes
when it's applied. This might cause issues with a rentention policy
merge policy that will see a deleted document but not it's value on
disk.
This change moves back the PendingDeletes callback to flush time
in order to be consistent with what is actually updated on disk.
This change also makes sure we write values to disk on flush that
are in the reader pool as well as extra best effort checks to drop
fully deleted segments on flush, commit and getReader.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/591fc662
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/591fc662
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/591fc662
Branch: refs/heads/jira/solr-11779
Commit: 591fc6627acffdc75ce88feb8a912b3225b47f9d
Parents: 8b9c2a3
Author: Simon Willnauer <si...@apache.org>
Authored: Wed May 2 16:17:15 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Thu May 3 13:38:35 2018 +0200
----------------------------------------------------------------------
.../org/apache/lucene/index/IndexWriter.java | 51 ++++++++++++++------
.../org/apache/lucene/index/PendingDeletes.java | 13 ++---
.../apache/lucene/index/PendingSoftDeletes.java | 9 +---
.../apache/lucene/index/ReadersAndUpdates.java | 3 +-
.../lucene/index/TestPendingSoftDeletes.java | 9 ++--
.../TestSoftDeletesRetentionMergePolicy.java | 9 ++--
6 files changed, 51 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/591fc662/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index b209146..fa631d8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -510,16 +510,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// TODO: we could instead just clone SIS and pull/incref readers in sync'd block, and then do this w/o IW's lock?
// Must do this sync'd on IW to prevent a merge from completing at the last second and failing to write its DV updates:
- if (readerPool.writeAllDocValuesUpdates()) {
- checkpoint();
- }
-
- if (writeAllDeletes) {
- // Must move the deletes to disk:
- if (readerPool.commit(segmentInfos)) {
- checkpointNoSIS();
- }
- }
+ writeReaderPool(writeAllDeletes);
// Prevent segmentInfos from changing while opening the
// reader; in theory we could instead do similar retry logic,
@@ -3132,11 +3123,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
applyAllDeletesAndUpdates();
synchronized(this) {
-
- if (readerPool.commit(segmentInfos)) {
- checkpointNoSIS();
- }
-
+ writeReaderPool(true);
if (changeCount.get() != lastCommitChangeCount) {
// There are changes to commit, so we will write a new segments_N in startCommit.
// The act of committing is itself an NRT-visible change (an NRT reader that was
@@ -3217,6 +3204,39 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
}
}
}
+
+ /**
+ * Ensures that all changes in the reader-pool are written to disk.
+ * @param writeDeletes if <code>true</code> if deletes should be written to disk too.
+ */
+ private final void writeReaderPool(boolean writeDeletes) throws IOException {
+ assert Thread.holdsLock(this);
+ if (writeDeletes) {
+ if (readerPool.commit(segmentInfos)) {
+ checkpointNoSIS();
+ }
+ } else { // only write the docValues
+ if (readerPool.writeAllDocValuesUpdates()) {
+ checkpoint();
+ }
+ }
+ // now do some best effort to check if a segment is fully deleted
+ List<SegmentCommitInfo> toDrop = new ArrayList<>(); // don't modify segmentInfos in-place
+ for (SegmentCommitInfo info : segmentInfos) {
+ ReadersAndUpdates readersAndUpdates = readerPool.get(info, false);
+ if (readersAndUpdates != null) {
+ if (isFullyDeleted(readersAndUpdates)) {
+ toDrop.add(info);
+ }
+ }
+ }
+ for (SegmentCommitInfo info : toDrop) {
+ dropDeletedSegment(info);
+ }
+ if (toDrop.isEmpty() == false) {
+ checkpoint();
+ }
+ }
/**
* Sets the iterator to provide the commit user data map at commit time. Calling this method
@@ -3503,6 +3523,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
anyChanges |= maybeMerge.getAndSet(false);
synchronized(this) {
+ writeReaderPool(applyAllDeletes);
doAfterFlush();
success = true;
return anyChanges;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/591fc662/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java b/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
index ae91be8..1878665 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PendingDeletes.java
@@ -235,18 +235,11 @@ class PendingDeletes {
}
/**
- * Called before the given DocValuesFieldUpdates are written to disk
- * @param info the field to apply
- */
- void onDocValuesUpdate(FieldInfo info) {
- }
-
- /**
- * Called for every field update for the given field
- * @param field the field that's updated
+ * Called for every field update for the given field at flush time
+ * @param info the field info of the field that's updated
* @param iterator the values to apply
*/
- void onDocValuesUpdate(String field, DocValuesFieldUpdates.Iterator iterator) throws IOException {
+ void onDocValuesUpdate(FieldInfo info, DocValuesFieldUpdates.Iterator iterator) throws IOException {
}
int numDeletesToMerge(MergePolicy policy, IOSupplier<CodecReader> readerIOSupplier) throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/591fc662/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java b/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
index a5e7b14..eae25e0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
+++ b/lucene/core/src/java/org/apache/lucene/index/PendingSoftDeletes.java
@@ -120,14 +120,9 @@ final class PendingSoftDeletes extends PendingDeletes {
}
@Override
- void onDocValuesUpdate(String field, DocValuesFieldUpdates.Iterator iterator) throws IOException {
- if (this.field.equals(field)) {
+ void onDocValuesUpdate(FieldInfo info, DocValuesFieldUpdates.Iterator iterator) throws IOException {
+ if (this.field.equals(info.name)) {
pendingDeleteCount += applySoftDeletes(iterator, getMutableBits());
- }
- }
- @Override
- void onDocValuesUpdate(FieldInfo info) {
- if (field.equals(info.name)) {
assert dvGeneration < info.getDocValuesGen() : "we have seen this generation update already: " + dvGeneration + " vs. " + info.getDocValuesGen();
assert dvGeneration != -2 : "docValues generation is still uninitialized";
dvGeneration = info.getDocValuesGen();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/591fc662/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index b31bc49..6e074ee 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -161,7 +161,6 @@ final class ReadersAndUpdates {
}
fieldUpdates.add(update);
}
- pendingDeletes.onDocValuesUpdate(update.field, update.iterator());
}
public synchronized long getNumDVUpdates() {
@@ -334,7 +333,6 @@ final class ReadersAndUpdates {
final TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);
final SegmentWriteState state = new SegmentWriteState(null, trackingDir, info.info, fieldInfos, null, updatesContext, segmentSuffix);
try (final DocValuesConsumer fieldsConsumer = dvFormat.fieldsConsumer(state)) {
- pendingDeletes.onDocValuesUpdate(fieldInfo);
Function<FieldInfo, DocValuesFieldUpdates.Iterator> updateSupplier = (info) -> {
if (info != fieldInfo) {
throw new IllegalArgumentException("expected field info for field: " + fieldInfo.name + " but got: " + info.name);
@@ -345,6 +343,7 @@ final class ReadersAndUpdates {
}
return DocValuesFieldUpdates.mergedIterator(subs);
};
+ pendingDeletes.onDocValuesUpdate(fieldInfo, updateSupplier.apply(fieldInfo));
if (type == DocValuesType.BINARY) {
fieldsConsumer.addBinaryField(fieldInfo, new EmptyDocValuesProducer() {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/591fc662/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
index 8bd7ca6..7ddf0be 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
@@ -116,9 +116,8 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
List<Integer> docsDeleted = Arrays.asList(1, 3, 7, 8, DocIdSetIterator.NO_MORE_DOCS);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(docsDeleted, 10));
for (DocValuesFieldUpdates update : updates) {
- deletes.onDocValuesUpdate(update.field, update.iterator());
+ deletes.onDocValuesUpdate(fieldInfo, update.iterator());
}
- deletes.onDocValuesUpdate(fieldInfo);
assertEquals(4, deletes.numPendingDeletes());
assertTrue(deletes.getLiveDocs().get(0));
assertFalse(deletes.getLiveDocs().get(1));
@@ -135,9 +134,8 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
updates = Arrays.asList(singleUpdate(docsDeleted, 10));
fieldInfo = new FieldInfo("_soft_deletes", 1, false, false, false, IndexOptions.NONE, DocValuesType.NUMERIC, 1, Collections.emptyMap(), 0, 0);
for (DocValuesFieldUpdates update : updates) {
- deletes.onDocValuesUpdate(update.field, update.iterator());
+ deletes.onDocValuesUpdate(fieldInfo, update.iterator());
}
- deletes.onDocValuesUpdate(fieldInfo);
assertEquals(5, deletes.numPendingDeletes());
assertTrue(deletes.getLiveDocs().get(0));
assertFalse(deletes.getLiveDocs().get(1));
@@ -180,9 +178,8 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
List<Integer> docsDeleted = Arrays.asList(1, DocIdSetIterator.NO_MORE_DOCS);
List<DocValuesFieldUpdates> updates = Arrays.asList(singleUpdate(docsDeleted, 3));
for (DocValuesFieldUpdates update : updates) {
- deletes.onDocValuesUpdate(update.field, update.iterator());
+ deletes.onDocValuesUpdate(fieldInfo, update.iterator());
}
- deletes.onDocValuesUpdate(fieldInfo);
assertEquals(1, deletes.numPendingDeletes());
assertTrue(deletes.getLiveDocs().get(0));
assertFalse(deletes.getLiveDocs().get(1));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/591fc662/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
index b868a2e..5f1ba6c 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesRetentionMergePolicy.java
@@ -289,6 +289,9 @@ public class TestSoftDeletesRetentionMergePolicy extends LuceneTestCase {
writer.softUpdateDocument(new Term("id", id), doc,
new NumericDocValuesField("soft_delete", 1));
}
+ if (rarely()) {
+ writer.flush();
+ }
ids.add(id);
}
} catch (IOException | InterruptedException e) {
@@ -382,13 +385,13 @@ public class TestSoftDeletesRetentionMergePolicy extends LuceneTestCase {
Document tombstone = new Document();
tombstone.add(new NumericDocValuesField("soft_delete", 1));
writer.softUpdateDocument(new Term("id", "1"), tombstone, new NumericDocValuesField("soft_delete", 1));
- // Internally, forceMergeDeletes will call flush to flush pending updates
+ writer.forceMergeDeletes(true); // Internally, forceMergeDeletes will call flush to flush pending updates
// Thus, we will have two segments - both having soft-deleted documents.
// We expect any MP to merge these segments into one segment
// when calling forceMergeDeletes.
- writer.forceMergeDeletes(true);
- assertEquals(1, writer.maxDoc());
assertEquals(1, writer.segmentInfos.asList().size());
+ assertEquals(1, writer.numDocs());
+ assertEquals(1, writer.maxDoc());
writer.close();
dir.close();
}
[10/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8283: Minor
javadoc correction in UH FieldOffsetStrategy
Posted by ab...@apache.org.
LUCENE-8283: Minor javadoc correction in UH FieldOffsetStrategy
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/570fff86
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/570fff86
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/570fff86
Branch: refs/heads/jira/solr-11779
Commit: 570fff86727742360703a7d17f33127f3341edda
Parents: f6cbb2d
Author: David Smiley <ds...@apache.org>
Authored: Sun Apr 29 22:28:56 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Sun Apr 29 22:28:56 2018 -0400
----------------------------------------------------------------------
.../org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/570fff86/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java
index ed47032..cf564f4 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java
@@ -30,7 +30,7 @@ import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
/**
- * Ultimately returns a list of {@link OffsetsEnum} yielding potentially highlightable words in the text. Needs
+ * Ultimately returns an {@link OffsetsEnum} yielding potentially highlightable words in the text. Needs
* information about the query up front.
*
* @lucene.internal
[45/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8261:
InterpolatedProperties.interpolate and recursive property references.
Posted by ab...@apache.org.
LUCENE-8261: InterpolatedProperties.interpolate and recursive property references.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/445c0aa4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/445c0aa4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/445c0aa4
Branch: refs/heads/jira/solr-11779
Commit: 445c0aa47e9962754c93bde1e07a24cd271e7d31
Parents: f9942b5
Author: Dawid Weiss <dw...@apache.org>
Authored: Mon May 7 13:22:11 2018 +0200
Committer: Dawid Weiss <dw...@apache.org>
Committed: Mon May 7 13:22:11 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../dependencies/InterpolatedProperties.java | 127 ++++++++++++++++---
2 files changed, 114 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/445c0aa4/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ed6cc26..5ded39a 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -183,6 +183,9 @@ Bug Fixes
Other
+* LUCENE-8261: InterpolatedProperties.interpolate and recursive property
+ references. (Steve Rowe, Dawid Weiss)
+
* LUCENE-8228: removed obsolete IndexDeletionPolicy clone() requirements from
the javadoc. (Dawid Weiss)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/445c0aa4/lucene/tools/src/java/org/apache/lucene/dependencies/InterpolatedProperties.java
----------------------------------------------------------------------
diff --git a/lucene/tools/src/java/org/apache/lucene/dependencies/InterpolatedProperties.java b/lucene/tools/src/java/org/apache/lucene/dependencies/InterpolatedProperties.java
index 159a803..2d5ca41 100644
--- a/lucene/tools/src/java/org/apache/lucene/dependencies/InterpolatedProperties.java
+++ b/lucene/tools/src/java/org/apache/lucene/dependencies/InterpolatedProperties.java
@@ -19,17 +19,28 @@ package org.apache.lucene.dependencies;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
/**
* Parse a properties file, performing non-recursive Ant-like
* property value interpolation, and return the resulting Properties.
*/
public class InterpolatedProperties extends Properties {
- private static final Pattern PROPERTY_REFERENCE_PATTERN = Pattern.compile("\\$\\{([^}]+)\\}");
+ private static final Pattern PROPERTY_REFERENCE_PATTERN = Pattern.compile("\\$\\{(?<name>[^}]+)\\}");
/**
* Loads the properties file via {@link Properties#load(InputStream)},
@@ -46,26 +57,110 @@ public class InterpolatedProperties extends Properties {
*/
@Override
public void load(Reader reader) throws IOException {
- super.load(reader);
- interpolate();
+ Properties p = new Properties();
+ p.load(reader);
+
+ LinkedHashMap<String, String> props = new LinkedHashMap<>();
+ Enumeration<?> e = p.propertyNames();
+ while (e.hasMoreElements()) {
+ String key = (String) e.nextElement();
+ props.put(key, p.getProperty(key));
+ }
+
+ resolve(props).forEach((k, v) -> this.setProperty(k, v));
}
- /**
- * Perform non-recursive Ant-like property value interpolation
- */
- private void interpolate() {
- StringBuffer buffer = new StringBuffer();
- for (Map.Entry<?,?> entry : entrySet()) {
- buffer.setLength(0);
- Matcher matcher = PROPERTY_REFERENCE_PATTERN.matcher(entry.getValue().toString());
+ private static Map<String,String> resolve(Map<String,String> props) {
+ LinkedHashMap<String, String> resolved = new LinkedHashMap<>();
+ HashSet<String> recursive = new HashSet<>();
+ props.forEach((k, v) -> {
+ resolve(props, resolved, recursive, k, v);
+ });
+ return resolved;
+ }
+
+ private static String resolve(Map<String,String> props,
+ LinkedHashMap<String, String> resolved,
+ Set<String> recursive,
+ String key,
+ String value) {
+ if (value == null) {
+ throw new IllegalArgumentException("Missing replaced property key: " + key);
+ }
+
+ if (recursive.contains(key)) {
+ throw new IllegalArgumentException("Circular recursive property resolution: " + recursive);
+ }
+
+ if (!resolved.containsKey(key)) {
+ recursive.add(key);
+ StringBuffer buffer = new StringBuffer();
+ Matcher matcher = PROPERTY_REFERENCE_PATTERN.matcher(value);
while (matcher.find()) {
- String interpolatedValue = getProperty(matcher.group(1));
- if (null != interpolatedValue) {
- matcher.appendReplacement(buffer, interpolatedValue);
- }
+ String referenced = matcher.group("name");
+ String concrete = resolve(props, resolved, recursive, referenced, props.get(referenced));
+ matcher.appendReplacement(buffer, Matcher.quoteReplacement(concrete));
}
matcher.appendTail(buffer);
- setProperty((String) entry.getKey(), buffer.toString());
+ resolved.put(key, buffer.toString());
+ recursive.remove(key);
+ }
+ assert resolved.get(key).equals(value);
+ return resolved.get(key);
+ }
+
+ public static void main(String [] args) {
+ {
+ Map<String, String> props = new LinkedHashMap<>();
+ props.put("a", "${b}");
+ props.put("b", "${c}");
+ props.put("c", "foo");
+ props.put("d", "${a}/${b}/${c}");
+ assertEquals(resolve(props), "a=foo", "b=foo", "c=foo", "d=foo/foo/foo");
+ }
+
+ {
+ Map<String, String> props = new LinkedHashMap<>();
+ props.put("a", "foo");
+ props.put("b", "${a}");
+ assertEquals(resolve(props), "a=foo", "b=foo");
+ }
+
+ {
+ Map<String, String> props = new LinkedHashMap<>();
+ props.put("a", "${b}");
+ props.put("b", "${c}");
+ props.put("c", "${a}");
+ try {
+ resolve(props);
+ } catch (IllegalArgumentException e) {
+ // Expected, circular reference.
+ if (!e.getMessage().contains("Circular recursive")) {
+ throw new AssertionError();
+ }
+ }
+ }
+
+ {
+ Map<String, String> props = new LinkedHashMap<>();
+ props.put("a", "${b}");
+ try {
+ resolve(props);
+ } catch (IllegalArgumentException e) {
+ // Expected, no referenced value.
+ if (!e.getMessage().contains("Missing replaced")) {
+ throw new AssertionError();
+ }
+ }
+ }
+ }
+
+ private static void assertEquals(Map<String,String> resolved, String... keyValuePairs) {
+ List<String> result = resolved.entrySet().stream().sorted((a, b) -> a.getKey().compareTo(b.getKey()))
+ .map(e -> e.getKey() + "=" + e.getValue())
+ .collect(Collectors.toList());
+ if (!result.equals(Arrays.asList(keyValuePairs))) {
+ throw new AssertionError("Mismatch: \n" + result + "\nExpected: " + Arrays.asList(keyValuePairs));
}
}
}
[47/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8297: Add
IW#tryUpdateDocValues(Reader, int, Fields...)
Posted by ab...@apache.org.
LUCENE-8297: Add IW#tryUpdateDocValues(Reader, int, Fields...)
IndexWriter can update doc values for a specific term but this might
affect all documents containing the term. With tryUpdateDocValues
users can update doc-values fields for individual documents. This allows
for instance to soft-delete individual documents.
The new method shares most of it's code with tryDeleteDocuments.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b0b32931
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b0b32931
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b0b32931
Branch: refs/heads/jira/solr-11779
Commit: b0b32931b28da83d47f0561f4da734d6b9ee6e16
Parents: 6521d86
Author: Simon Willnauer <si...@apache.org>
Authored: Sat May 5 09:55:58 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon May 7 14:35:59 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 6 +
.../apache/lucene/index/DocValuesUpdate.java | 4 +-
.../org/apache/lucene/index/IndexWriter.java | 89 +++++++++--
.../lucene/index/TestMixedDocValuesUpdates.java | 154 ++++++++++++++++++-
4 files changed, 235 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b0b32931/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5ded39a..ec68882 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -155,6 +155,12 @@ New Features
* LUCENE-8265: WordDelimter/GraphFilter now have an option to skip tokens
marked with KeywordAttribute (Mike Sokolov via Mike McCandless)
+* LUCENE-8297: Add IW#tryUpdateDocValues(Reader, int, Fields...) IndexWriter can
+ update doc values for a specific term but this might affect all documents
+ containing the term. With tryUpdateDocValues users can update doc-values
+ fields for individual documents. This allows for instance to soft-delete
+ individual documents. (Simon Willnauer)
+
Bug Fixes
* LUCENE-8266: Detect bogus tiles when creating a standard polygon and
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b0b32931/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
index 8229b60..c8bc8fb 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
@@ -83,7 +83,7 @@ abstract class DocValuesUpdate {
/** An in-place update to a binary DocValues field */
static final class BinaryDocValuesUpdate extends DocValuesUpdate {
- private final BytesRef value;
+ final BytesRef value;
/* Size of BytesRef: 2*INT + ARRAY_HEADER + PTR */
private static final long RAW_VALUE_SIZE_IN_BYTES = NUM_BYTES_ARRAY_HEADER + 2*Integer.BYTES + NUM_BYTES_OBJECT_REF;
@@ -132,7 +132,7 @@ abstract class DocValuesUpdate {
/** An in-place update to a numeric DocValues field */
static final class NumericDocValuesUpdate extends DocValuesUpdate {
- private final long value;
+ final long value;
NumericDocValuesUpdate(Term term, String field, long value) {
this(term, field, value, BufferedUpdates.MAX_INT);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b0b32931/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 93f5446..d8ef5c0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -1347,7 +1347,82 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* to delete documents indexed after opening the NRT
* reader you must use {@link #deleteDocuments(Term...)}). */
public synchronized long tryDeleteDocument(IndexReader readerIn, int docID) throws IOException {
+ // NOTE: DON'T use docID inside the closure
+ return tryModifyDocument(readerIn, docID, (leafDocId, rld) -> {
+ if (rld.delete(leafDocId)) {
+ if (isFullyDeleted(rld)) {
+ dropDeletedSegment(rld.info);
+ checkpoint();
+ }
+
+ // Must bump changeCount so if no other changes
+ // happened, we still commit this change:
+ changed();
+ }
+ });
+ }
+ /** Expert: attempts to update doc values by document ID, as long as
+ * the provided reader is a near-real-time reader (from {@link
+ * DirectoryReader#open(IndexWriter)}). If the
+ * provided reader is an NRT reader obtained from this
+ * writer, and its segment has not been merged away, then
+ * the update succeeds and this method returns a valid (> 0) sequence
+ * number; else, it returns -1 and the caller must then
+ * either retry the update and resolve the document again.
+ *
+ * <b>NOTE</b>: this method can only updates documents
+ * visible to the currently open NRT reader. If you need
+ * to update documents indexed after opening the NRT
+ * reader you must use {@link #updateDocValues(Term, Field...)}. */
+ public synchronized long tryUpdateDocValue(IndexReader readerIn, int docID, Field... fields) throws IOException {
+ // NOTE: DON'T use docID inside the closure
+ final DocValuesUpdate[] dvUpdates = buildDocValuesUpdate(null, fields);
+ return tryModifyDocument(readerIn, docID, (leafDocId, rld) -> {
+ long nextGen = bufferedUpdatesStream.getNextGen();
+ try {
+ Map<String, DocValuesFieldUpdates> fieldUpdatesMap = new HashMap<>();
+ for (DocValuesUpdate update : dvUpdates) {
+ DocValuesFieldUpdates docValuesFieldUpdates = fieldUpdatesMap.computeIfAbsent(update.field, k -> {
+ switch (update.type) {
+ case NUMERIC:
+ return new NumericDocValuesFieldUpdates(nextGen, k, rld.info.info.maxDoc());
+ case BINARY:
+ return new BinaryDocValuesFieldUpdates(nextGen, k, rld.info.info.maxDoc());
+ default:
+ throw new AssertionError("type: " + update.type + " is not supported");
+ }
+ });
+ switch (update.type) {
+ case NUMERIC:
+ docValuesFieldUpdates.add(leafDocId, ((NumericDocValuesUpdate) update).value);
+ break;
+ case BINARY:
+ docValuesFieldUpdates.add(leafDocId, ((BinaryDocValuesUpdate) update).value);
+ break;
+ default:
+ throw new AssertionError("type: " + update.type + " is not supported");
+ }
+ }
+ for (DocValuesFieldUpdates updates : fieldUpdatesMap.values()) {
+ updates.finish();
+ rld.addDVUpdate(updates);
+ }
+ } finally {
+ bufferedUpdatesStream.finishedSegment(nextGen);
+ }
+ // Must bump changeCount so if no other changes
+ // happened, we still commit this change:
+ changed();
+ });
+ }
+
+ @FunctionalInterface
+ private interface DocModifier {
+ void run(int docId, ReadersAndUpdates readersAndUpdates) throws IOException;
+ }
+
+ private synchronized long tryModifyDocument(IndexReader readerIn, int docID, DocModifier toApply) throws IOException {
final LeafReader reader;
if (readerIn instanceof LeafReader) {
// Reader is already atomic: use the incoming docID:
@@ -1365,7 +1440,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
if (!(reader instanceof SegmentReader)) {
throw new IllegalArgumentException("the reader must be a SegmentReader or composite reader containing only SegmentReaders");
}
-
+
final SegmentCommitInfo info = ((SegmentReader) reader).getSegmentInfo();
// TODO: this is a slow linear search, but, number of
@@ -1377,21 +1452,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
ReadersAndUpdates rld = getPooledInstance(info, false);
if (rld != null) {
synchronized(bufferedUpdatesStream) {
- if (rld.delete(docID)) {
- if (isFullyDeleted(rld)) {
- dropDeletedSegment(rld.info);
- checkpoint();
- }
-
- // Must bump changeCount so if no other changes
- // happened, we still commit this change:
- changed();
- }
+ toApply.run(docID, rld);
return docWriter.deleteQueue.getNextSequenceNumber();
}
}
}
-
return -1;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b0b32931/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
index f40379f..401de4d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
@@ -18,17 +18,25 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.HashSet;
+import java.util.List;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -258,7 +266,6 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
writer.close();
DirectoryReader reader = DirectoryReader.open(dir);
- BytesRef scratch = new BytesRef();
for (LeafReaderContext context : reader.leaves()) {
LeafReader r = context.reader();
for (int i = 0; i < numFields; i++) {
@@ -305,8 +312,14 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
int doc = random().nextInt(numDocs);
Term t = new Term("id", "doc" + doc);
long value = random().nextLong();
- writer.updateDocValues(t, new BinaryDocValuesField("f", TestBinaryDocValuesUpdates.toBytes(value)),
- new NumericDocValuesField("cf", value*2));
+ if (random().nextBoolean()) {
+ doUpdate(t, writer, new BinaryDocValuesField("f", TestBinaryDocValuesUpdates.toBytes(value)),
+ new NumericDocValuesField("cf", value*2));
+ } else {
+ writer.updateDocValues(t, new BinaryDocValuesField("f", TestBinaryDocValuesUpdates.toBytes(value)),
+ new NumericDocValuesField("cf", value*2));
+ }
+
DirectoryReader reader = DirectoryReader.open(writer);
for (LeafReaderContext context : reader.leaves()) {
LeafReader r = context.reader();
@@ -394,5 +407,138 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
dir.close();
}
-
+
+ public void testTryUpdateDocValues() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig();
+ IndexWriter writer = new IndexWriter(dir, conf);
+ int numDocs = 1 + random().nextInt(128);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ doc.add(new StringField("id", "" + i, Store.YES));
+ doc.add(new NumericDocValuesField("id", i));
+ doc.add(new BinaryDocValuesField("binaryId", new BytesRef(new byte[] {(byte)i})));
+ writer.addDocument(doc);
+ if (random().nextBoolean()) {
+ writer.flush();
+ }
+ }
+ int doc = random().nextInt(numDocs);
+ doUpdate(new Term("id", "" + doc), writer, new NumericDocValuesField("id", doc + 1),
+ new BinaryDocValuesField("binaryId", new BytesRef(new byte[]{(byte) (doc + 1)})));
+ IndexReader reader = writer.getReader();
+ NumericDocValues idValues = null;
+ BinaryDocValues binaryIdValues = null;
+ for (LeafReaderContext c : reader.leaves()) {
+ TopDocs topDocs = new IndexSearcher(c.reader()).search(new TermQuery(new Term("id", "" + doc)), 10);
+ if (topDocs.totalHits == 1) {
+ assertNull(idValues);
+ assertNull(binaryIdValues);
+ idValues = c.reader().getNumericDocValues("id");
+ assertEquals(topDocs.scoreDocs[0].doc, idValues.advance(topDocs.scoreDocs[0].doc));
+ binaryIdValues = c.reader().getBinaryDocValues("binaryId");
+ assertEquals(topDocs.scoreDocs[0].doc, binaryIdValues.advance(topDocs.scoreDocs[0].doc));
+ } else {
+ assertEquals(0, topDocs.totalHits);
+ }
+ }
+
+ assertNotNull(idValues);
+ assertNotNull(binaryIdValues);
+
+ assertEquals(doc+1, idValues.longValue());
+ assertEquals(new BytesRef(new byte[] {(byte)(doc+1)}), binaryIdValues.binaryValue());
+ IOUtils.close(reader, writer, dir);
+ }
+
+ public void testTryUpdateMultiThreaded() throws IOException, BrokenBarrierException, InterruptedException {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig();
+ IndexWriter writer = new IndexWriter(dir, conf);
+ ReentrantLock[] locks = new ReentrantLock[25 + random().nextInt(50)];
+ int[] values = new int[locks.length];
+ for (int i = 0; i < locks.length; i++) {
+ locks[i] = new ReentrantLock();
+ Document doc = new Document();
+ values[i] = random().nextInt();
+ doc.add(new StringField("id", Integer.toString(i), Store.NO));
+ doc.add(new NumericDocValuesField("value", values[i]));
+ writer.addDocument(doc);
+ }
+
+ Thread[] threads = new Thread[2 + random().nextInt(3)];
+ CyclicBarrier barrier = new CyclicBarrier(threads.length + 1);
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread(() -> {
+ try {
+ barrier.await();
+ for (int doc = 0; doc < 1000; doc++) {
+ int docId = random().nextInt(locks.length);
+ locks[docId].lock();
+ try {
+ int value = random().nextInt();
+ if (random().nextBoolean()) {
+ writer.updateDocValues(new Term("id", docId + ""), new NumericDocValuesField("value", value));
+ } else {
+ doUpdate(new Term("id", docId + ""), writer, new NumericDocValuesField("value", value));
+ }
+ values[docId] = value;
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ } finally {
+ locks[docId].unlock();
+ }
+
+ if (rarely()) {
+ writer.flush();
+ }
+ }
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ });
+ threads[i].start();
+ }
+
+ barrier.await();
+ for (Thread t : threads) {
+ t.join();
+ }
+ try (DirectoryReader reader = writer.getReader()) {
+ for (int i = 0; i < locks.length; i++) {
+ locks[i].lock();
+ try {
+ int value = values[i];
+ TopDocs topDocs = new IndexSearcher(reader).search(new TermQuery(new Term("id", "" + i)), 10);
+ assertEquals(topDocs.totalHits, 1);
+ int docID = topDocs.scoreDocs[0].doc;
+ List<LeafReaderContext> leaves = reader.leaves();
+ int subIndex = ReaderUtil.subIndex(docID, leaves);
+ LeafReader leafReader = leaves.get(subIndex).reader();
+ docID -= leaves.get(subIndex).docBase;
+ NumericDocValues numericDocValues = leafReader.getNumericDocValues("value");
+ assertEquals(docID, numericDocValues.advance(docID));
+ assertEquals(numericDocValues.longValue(), value);
+ } finally {
+ locks[i].unlock();
+ }
+
+ }
+ }
+
+ IOUtils.close(writer, dir);
+ }
+
+ static void doUpdate(Term doc, IndexWriter writer, Field... fields) throws IOException {
+ long seqId = -1;
+ do { // retry if we just committing a merge
+ try (DirectoryReader reader = writer.getReader()) {
+ TopDocs topDocs = new IndexSearcher(reader).search(new TermQuery(doc), 10);
+ assertEquals(1, topDocs.totalHits);
+ int theDoc = topDocs.scoreDocs[0].doc;
+ seqId = writer.tryUpdateDocValue(reader, theDoc, fields);
+ }
+ } while (seqId == -1);
+ }
}
+
[41/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12316: Fix test to
work on linux and test also windows in a better way
Posted by ab...@apache.org.
SOLR-12316: Fix test to work on linux and test also windows in a better way
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fb5b42a0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fb5b42a0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fb5b42a0
Branch: refs/heads/jira/solr-11779
Commit: fb5b42a0e4bb55f82985282a68fff01015c9693f
Parents: 1b76011
Author: Uwe Schindler <us...@apache.org>
Authored: Sun May 6 15:53:07 2018 +0200
Committer: Uwe Schindler <us...@apache.org>
Committed: Sun May 6 15:53:07 2018 +0200
----------------------------------------------------------------------
.../apache/solr/util/TestSystemIdResolver.java | 30 +++++++++++++-------
1 file changed, 19 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fb5b42a0/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java b/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java
index 4c2677d..f87eeb4 100644
--- a/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java
+++ b/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java
@@ -19,6 +19,7 @@ package org.apache.solr.util;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.Arrays;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.analysis.util.ResourceLoader;
@@ -29,11 +30,6 @@ import org.xml.sax.InputSource;
public class TestSystemIdResolver extends LuceneTestCase {
- public void setUp() throws Exception {
- super.setUp();
- System.setProperty("solr.allow.unsafe.resourceloading", "true");
- }
-
public void tearDown() throws Exception {
System.clearProperty("solr.allow.unsafe.resourceloading");
super.tearDown();
@@ -74,8 +70,6 @@ public class TestSystemIdResolver extends LuceneTestCase {
"solrres:/org/apache/solr/util/RTimer.class", "TestSystemIdResolver.class");
assertEntityResolving(resolver, SystemIdResolver.createSystemIdFromResourceName(testHome+"/collection1/conf/schema.xml"),
SystemIdResolver.createSystemIdFromResourceName(testHome+"/collection1/conf/solrconfig.xml"), "schema.xml");
- assertEntityResolving(resolver, SystemIdResolver.createSystemIdFromResourceName(testHome+"/crazy-path-to-schema.xml"),
- SystemIdResolver.createSystemIdFromResourceName(testHome+"/crazy-path-to-config.xml"), "crazy-path-to-schema.xml");
// if somebody uses an absolute uri (e.g., file://) we should fail resolving:
IOException ioe = expectThrows(IOException.class, () -> {
@@ -89,10 +83,24 @@ public class TestSystemIdResolver extends LuceneTestCase {
assertTrue(ioe.getMessage().startsWith("Cannot resolve absolute"));
// check that we can't escape with absolute file paths:
- ioe = expectThrows(IOException.class, () -> {
- resolver.resolveEntity(null, null, "solrres:/solrconfig.xml", "/etc/passwd");
- });
- assertTrue(ioe.getMessage().startsWith("Can't find resource '/etc/passwd' in classpath or"));
+ for (String path : Arrays.asList("/etc/passwd", "/windows/notepad.exe")) {
+ ioe = expectThrows(IOException.class, () -> {
+ resolver.resolveEntity(null, null, "solrres:/solrconfig.xml", path);
+ });
+ assertTrue(ioe.getMessage().startsWith("Can't find resource")
+ || ioe.getMessage().contains("is outside resource loader dir"));
+ }
+ }
+
+ public void testUnsafeResolving() throws Exception {
+ System.setProperty("solr.allow.unsafe.resourceloading", "true");
+
+ final Path testHome = SolrTestCaseJ4.getFile("solr/collection1").getParentFile().toPath();
+ final ResourceLoader loader = new SolrResourceLoader(testHome.resolve("collection1"), this.getClass().getClassLoader());
+ final SystemIdResolver resolver = new SystemIdResolver(loader);
+
+ assertEntityResolving(resolver, SystemIdResolver.createSystemIdFromResourceName(testHome+"/crazy-path-to-schema.xml"),
+ SystemIdResolver.createSystemIdFromResourceName(testHome+"/crazy-path-to-config.xml"), "crazy-path-to-schema.xml");
}
}
[36/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12290: Do not close
any servlet streams and improve our servlet stream closing prevention code
for users and devs.
Posted by ab...@apache.org.
SOLR-12290: Do not close any servlet streams and improve our servlet stream closing prevention code for users and devs.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/29620105
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/29620105
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/29620105
Branch: refs/heads/jira/solr-11779
Commit: 296201055f24f01e1610f2fb87aba7fa90b9dda1
Parents: ad0ad2e
Author: Mark Miller <ma...@apache.org>
Authored: Fri May 4 18:02:06 2018 -0500
Committer: Mark Miller <ma...@apache.org>
Committed: Fri May 4 18:02:06 2018 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../org/apache/solr/handler/BlobHandler.java | 6 +-
.../apache/solr/handler/ReplicationHandler.java | 6 +-
.../solr/handler/loader/CSVLoaderBase.java | 75 +++++-----
.../solr/handler/loader/JavabinLoader.java | 16 +--
.../org/apache/solr/servlet/HttpSolrCall.java | 6 +-
.../apache/solr/servlet/LoadAdminUiServlet.java | 14 +-
.../solr/servlet/ServletInputStreamWrapper.java | 2 +-
.../servlet/ServletOutputStreamWrapper.java | 2 +-
.../apache/solr/servlet/SolrDispatchFilter.java | 139 ++++++++++++-------
.../apache/solr/servlet/SolrRequestParsers.java | 6 +-
11 files changed, 147 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d4c2097..f74e2fd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -206,6 +206,9 @@ Bug Fixes
* SOLR-12202: Fix errors in solr-exporter.cmd. (Minoru Osuka via koji)
+* SOLR-12290: Do not close any servlet streams and improve our servlet stream closing prevention code for users
+ and devs. (Mark Miller)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
index 30301c0..a998657 100644
--- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
@@ -17,7 +17,6 @@
package org.apache.solr.handler;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.math.BigInteger;
@@ -109,9 +108,8 @@ public class BlobHandler extends RequestHandlerBase implements PluginInfoInitial
for (ContentStream stream : req.getContentStreams()) {
ByteBuffer payload;
- try (InputStream is = stream.getStream()) {
- payload = SimplePostTool.inputStreamToByteArray(is, maxSize);
- }
+ payload = SimplePostTool.inputStreamToByteArray(stream.getStream(), maxSize);
+
MessageDigest m = MessageDigest.getInstance("MD5");
m.update(payload.array(), payload.position(), payload.limit());
String md5 = new BigInteger(1, m.digest()).toString(16);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
index 1707c80..6afb8a0 100644
--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
@@ -56,6 +56,7 @@ import java.util.zip.Checksum;
import java.util.zip.DeflaterOutputStream;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.CloseShieldOutputStream;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
@@ -1515,6 +1516,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}
protected void createOutputStream(OutputStream out) {
+ out = new CloseShieldOutputStream(out); // DeflaterOutputStream requires a close call, but don't close the request outputstream
if (Boolean.parseBoolean(compress)) {
fos = new FastOutputStream(new DeflaterOutputStream(out));
} else {
@@ -1579,7 +1581,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}
if (read != buf.length) {
writeNothingAndFlush();
- fos.close();
+ fos.close(); // we close because DeflaterOutputStream requires a close call, but but the request outputstream is protected
break;
}
offset += read;
@@ -1638,7 +1640,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
long bytesRead = channel.read(bb);
if (bytesRead <= 0) {
writeNothingAndFlush();
- fos.close();
+ fos.close(); // we close because DeflaterOutputStream requires a close call, but but the request outputstream is protected
break;
}
fos.writeInt((int) bytesRead);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
index b503fa3..fd8935d 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
@@ -28,7 +28,6 @@ import org.apache.solr.update.*;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.internal.csv.CSVStrategy;
import org.apache.solr.internal.csv.CSVParser;
-import org.apache.commons.io.IOUtils;
import java.util.regex.Pattern;
import java.util.List;
@@ -315,54 +314,48 @@ abstract class CSVLoaderBase extends ContentStreamLoader {
/** load the CSV input */
@Override
- public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor processor) throws IOException {
+ public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor processor)
+ throws IOException {
errHeader = "CSVLoader: input=" + stream.getSourceInfo();
- Reader reader = null;
- try {
- reader = stream.getReader();
- if (skipLines>0) {
- if (!(reader instanceof BufferedReader)) {
- reader = new BufferedReader(reader);
- }
- BufferedReader r = (BufferedReader)reader;
- for (int i=0; i<skipLines; i++) {
- r.readLine();
- }
+ Reader reader = stream.getReader();
+ if (skipLines > 0) {
+ if (!(reader instanceof BufferedReader)) {
+ reader = new BufferedReader(reader);
}
-
- CSVParser parser = new CSVParser(reader, strategy);
-
- // parse the fieldnames from the header of the file
- if (fieldnames==null) {
- fieldnames = parser.getLine();
- if (fieldnames==null) {
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Expected fieldnames in CSV input");
- }
- prepareFields();
+ BufferedReader r = (BufferedReader) reader;
+ for (int i = 0; i < skipLines; i++) {
+ r.readLine();
}
+ }
- // read the rest of the CSV file
- for(;;) {
- int line = parser.getLineNumber(); // for error reporting in MT mode
- String[] vals = null;
- try {
- vals = parser.getLine();
- } catch (IOException e) {
- //Catch the exception and rethrow it with more line information
- input_err("can't read line: " + line, null, line, e);
- }
- if (vals==null) break;
+ CSVParser parser = new CSVParser(reader, strategy);
- if (vals.length != fieldnames.length) {
- input_err("expected "+fieldnames.length+" values but got "+vals.length, vals, line);
- }
+ // parse the fieldnames from the header of the file
+ if (fieldnames == null) {
+ fieldnames = parser.getLine();
+ if (fieldnames == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Expected fieldnames in CSV input");
+ }
+ prepareFields();
+ }
- addDoc(line,vals);
+ // read the rest of the CSV file
+ for (;;) {
+ int line = parser.getLineNumber(); // for error reporting in MT mode
+ String[] vals = null;
+ try {
+ vals = parser.getLine();
+ } catch (IOException e) {
+ // Catch the exception and rethrow it with more line information
+ input_err("can't read line: " + line, null, line, e);
}
- } finally{
- if (reader != null) {
- IOUtils.closeQuietly(reader);
+ if (vals == null) break;
+
+ if (vals.length != fieldnames.length) {
+ input_err("expected " + fieldnames.length + " values but got " + vals.length, vals, line);
}
+
+ addDoc(line, vals);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
index f502a8e..aca3df4 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java
@@ -48,20 +48,8 @@ import org.apache.solr.update.processor.UpdateRequestProcessor;
public class JavabinLoader extends ContentStreamLoader {
@Override
- public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor processor) throws Exception {
- InputStream is = null;
- try {
- is = stream.getStream();
- parseAndLoadDocs(req, rsp, is, processor);
- } finally {
- if(is != null) {
- is.close();
- }
- }
- }
-
- private void parseAndLoadDocs(final SolrQueryRequest req, SolrQueryResponse rsp, InputStream stream,
- final UpdateRequestProcessor processor) throws IOException {
+ public void load(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream cs, UpdateRequestProcessor processor) throws Exception {
+ InputStream stream = cs.getStream();
UpdateRequest update = null;
JavaBinUpdateRequestCodec.StreamingUpdateHandler handler = new JavaBinUpdateRequestCodec.StreamingUpdateHandler() {
private AddUpdateCommand addCmd = null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index 50fa71c..5e4dd78 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -39,8 +39,6 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.input.CloseShieldInputStream;
-import org.apache.commons.io.output.CloseShieldOutputStream;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HeaderIterator;
@@ -590,7 +588,7 @@ public class HttpSolrCall {
} else if (isPostOrPutRequest) {
HttpEntityEnclosingRequestBase entityRequest =
"POST".equals(req.getMethod()) ? new HttpPost(urlstr) : new HttpPut(urlstr);
- InputStream in = new CloseShieldInputStream(req.getInputStream()); // Prevent close of container streams
+ InputStream in = req.getInputStream();
HttpEntity entity = new InputStreamEntity(in, req.getContentLength());
entityRequest.setEntity(entity);
method = entityRequest;
@@ -785,7 +783,7 @@ public class HttpSolrCall {
}
if (Method.HEAD != reqMethod) {
- OutputStream out = new CloseShieldOutputStream(response.getOutputStream()); // Prevent close of container streams, see SOLR-8933
+ OutputStream out = response.getOutputStream();
QueryResponseWriterUtil.writeQueryResponse(out, responseWriter, solrReq, solrRsp, ct);
}
//else http HEAD request, nothing to write out, waited this long just to get ContentType
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java b/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
index c496ce1..1aa1137 100644
--- a/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
+++ b/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
@@ -17,7 +17,6 @@
package org.apache.solr.servlet;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.output.CloseShieldOutputStream;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.common.params.CommonParams;
@@ -41,10 +40,12 @@ import java.nio.charset.StandardCharsets;
public final class LoadAdminUiServlet extends BaseSolrServlet {
@Override
- public void doGet(HttpServletRequest request,
- HttpServletResponse response)
+ public void doGet(HttpServletRequest _request,
+ HttpServletResponse _response)
throws IOException {
-
+ HttpServletRequest request = SolrDispatchFilter.closeShield(_request, false);
+ HttpServletResponse response = SolrDispatchFilter.closeShield(_response, false);
+
response.addHeader("X-Frame-Options", "DENY"); // security: SOLR-7966 - avoid clickjacking for admin interface
// This attribute is set by the SolrDispatchFilter
@@ -57,8 +58,8 @@ public final class LoadAdminUiServlet extends BaseSolrServlet {
response.setCharacterEncoding("UTF-8");
response.setContentType("text/html");
- // Protect container owned streams from being closed by us, see SOLR-8933
- out = new OutputStreamWriter(new CloseShieldOutputStream(response.getOutputStream()), StandardCharsets.UTF_8);
+ // Don't close this! - see SOLR-8933
+ out = new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8);
String html = IOUtils.toString(in, "UTF-8");
Package pack = SolrCore.class.getPackage();
@@ -77,7 +78,6 @@ public final class LoadAdminUiServlet extends BaseSolrServlet {
out.write( StringUtils.replaceEach(html, search, replace) );
} finally {
IOUtils.closeQuietly(in);
- IOUtils.closeQuietly(out);
}
} else {
response.sendError(404);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/core/src/java/org/apache/solr/servlet/ServletInputStreamWrapper.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/ServletInputStreamWrapper.java b/solr/core/src/java/org/apache/solr/servlet/ServletInputStreamWrapper.java
index d229bf7..826dc59 100644
--- a/solr/core/src/java/org/apache/solr/servlet/ServletInputStreamWrapper.java
+++ b/solr/core/src/java/org/apache/solr/servlet/ServletInputStreamWrapper.java
@@ -32,7 +32,7 @@ import org.apache.solr.common.util.SuppressForbidden;
*/
@SuppressForbidden(reason = "delegate methods")
public class ServletInputStreamWrapper extends ServletInputStream {
- final ServletInputStream stream;
+ ServletInputStream stream;
public ServletInputStreamWrapper(ServletInputStream stream) throws IOException {
this.stream = stream;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/core/src/java/org/apache/solr/servlet/ServletOutputStreamWrapper.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/ServletOutputStreamWrapper.java b/solr/core/src/java/org/apache/solr/servlet/ServletOutputStreamWrapper.java
index d12c3bd..1164146 100644
--- a/solr/core/src/java/org/apache/solr/servlet/ServletOutputStreamWrapper.java
+++ b/solr/core/src/java/org/apache/solr/servlet/ServletOutputStreamWrapper.java
@@ -32,7 +32,7 @@ import org.apache.solr.common.util.SuppressForbidden;
*/
@SuppressForbidden(reason = "delegate methods")
public class ServletOutputStreamWrapper extends ServletOutputStream {
- final ServletOutputStream stream;
+ ServletOutputStream stream;
public ServletOutputStreamWrapper(ServletOutputStream stream) {
this.stream = stream;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index fc0c28f..8f32a7d 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@ -37,20 +37,20 @@ import java.util.regex.Pattern;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
+import javax.servlet.ReadListener;
import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
import javax.servlet.ServletOutputStream;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.UnavailableException;
+import javax.servlet.WriteListener;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpServletResponseWrapper;
import org.apache.commons.io.FileCleaningTracker;
-import org.apache.commons.io.input.CloseShieldInputStream;
-import org.apache.commons.io.output.CloseShieldOutputStream;
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.lucene.util.Version;
@@ -98,8 +98,6 @@ public class SolrDispatchFilter extends BaseSolrFilter {
protected HttpClient httpClient;
private ArrayList<Pattern> excludePatterns;
- // Effectively immutable
- private Boolean testMode = null;
private boolean isV2Enabled = !"true".equals(System.getProperty("disable.v2.api", "false"));
private final String metricTag = Integer.toHexString(hashCode());
@@ -119,19 +117,6 @@ public class SolrDispatchFilter extends BaseSolrFilter {
}
public SolrDispatchFilter() {
- // turn on test mode when running tests
- assert testMode = true;
-
- if (testMode == null) {
- testMode = false;
- } else {
- String tm = System.getProperty("solr.tests.doContainerStreamCloseAssert");
- if (tm != null) {
- testMode = Boolean.parseBoolean(tm);
- } else {
- testMode = true;
- }
- }
}
public static final String PROPERTIES_ATTRIBUTE = "solr.properties";
@@ -341,8 +326,8 @@ public class SolrDispatchFilter extends BaseSolrFilter {
public void doFilter(ServletRequest _request, ServletResponse _response, FilterChain chain, boolean retry) throws IOException, ServletException {
if (!(_request instanceof HttpServletRequest)) return;
- HttpServletRequest request = (HttpServletRequest)_request;
- HttpServletResponse response = (HttpServletResponse)_response;
+ HttpServletRequest request = closeShield((HttpServletRequest)_request, retry);
+ HttpServletResponse response = closeShield((HttpServletResponse)_response, retry);
try {
@@ -387,7 +372,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
}
}
- HttpSolrCall call = getHttpSolrCall(closeShield(request, retry), closeShield(response, retry), retry);
+ HttpSolrCall call = getHttpSolrCall(request, response, retry);
ExecutorUtil.setServerThreadFlag(Boolean.TRUE);
try {
Action result = call.call();
@@ -486,31 +471,82 @@ public class SolrDispatchFilter extends BaseSolrFilter {
return true;
}
+ public static class ClosedServletInputStream extends ServletInputStream {
+
+ public static final ClosedServletInputStream CLOSED_SERVLET_INPUT_STREAM = new ClosedServletInputStream();
+
+ @Override
+ public int read() {
+ return -1;
+ }
+
+ @Override
+ public boolean isFinished() {
+ return false;
+ }
+
+ @Override
+ public boolean isReady() {
+ return false;
+ }
+
+ @Override
+ public void setReadListener(ReadListener arg0) {}
+ }
+
+ public static class ClosedServletOutputStream extends ServletOutputStream {
+
+ public static final ClosedServletOutputStream CLOSED_SERVLET_OUTPUT_STREAM = new ClosedServletOutputStream();
+
+ @Override
+ public void write(final int b) throws IOException {
+ throw new IOException("write(" + b + ") failed: stream is closed");
+ }
+
+ @Override
+ public void flush() throws IOException {
+ throw new IOException("flush() failed: stream is closed");
+ }
+
+ @Override
+ public boolean isReady() {
+ return false;
+ }
+
+ @Override
+ public void setWriteListener(WriteListener arg0) {
+ throw new RuntimeException("setWriteListener() failed: stream is closed");
+ }
+ }
+
/**
- * Wrap the request's input stream with a close shield, as if by a {@link CloseShieldInputStream}. If this is a
+ * Wrap the request's input stream with a close shield. If this is a
* retry, we will assume that the stream has already been wrapped and do nothing.
*
+ * Only the container should ever actually close the servlet output stream.
+ *
* @param request The request to wrap.
* @param retry If this is an original request or a retry.
* @return A request object with an {@link InputStream} that will ignore calls to close.
*/
- private HttpServletRequest closeShield(HttpServletRequest request, boolean retry) {
- if (testMode && !retry) {
+ public static HttpServletRequest closeShield(HttpServletRequest request, boolean retry) {
+ if (!retry) {
return new HttpServletRequestWrapper(request) {
- ServletInputStream stream;
-
+
@Override
public ServletInputStream getInputStream() throws IOException {
- // Lazy stream creation
- if (stream == null) {
- stream = new ServletInputStreamWrapper(super.getInputStream()) {
- @Override
- public void close() {
- assert false : "Attempted close of request input stream.";
- }
- };
- }
- return stream;
+
+ return new ServletInputStreamWrapper(super.getInputStream()) {
+ @Override
+ public void close() {
+ // even though we skip closes, we let local tests know not to close so that a full understanding can take
+ // place
+ assert Thread.currentThread().getStackTrace()[2].getClassName().matches(
+ "org\\.apache\\.(?:solr|lucene).*") ? false : true : "Attempted close of request input stream - never do this, you will spoil connection reuse and possibly disrupt a client";
+ this.stream = ClosedServletInputStream.CLOSED_SERVLET_INPUT_STREAM;
+ }
+ };
+
}
};
} else {
@@ -519,31 +555,34 @@ public class SolrDispatchFilter extends BaseSolrFilter {
}
/**
- * Wrap the response's output stream with a close shield, as if by a {@link CloseShieldOutputStream}. If this is a
+ * Wrap the response's output stream with a close shield. If this is a
* retry, we will assume that the stream has already been wrapped and do nothing.
*
+ * Only the container should ever actually close the servlet request stream.
+ *
* @param response The response to wrap.
* @param retry If this response corresponds to an original request or a retry.
* @return A response object with an {@link OutputStream} that will ignore calls to close.
*/
- private HttpServletResponse closeShield(HttpServletResponse response, boolean retry) {
- if (testMode && !retry) {
+ public static HttpServletResponse closeShield(HttpServletResponse response, boolean retry) {
+ if (!retry) {
return new HttpServletResponseWrapper(response) {
- ServletOutputStream stream;
-
+
@Override
public ServletOutputStream getOutputStream() throws IOException {
- // Lazy stream creation
- if (stream == null) {
- stream = new ServletOutputStreamWrapper(super.getOutputStream()) {
- @Override
- public void close() {
- assert false : "Attempted close of response output stream.";
- }
- };
- }
- return stream;
+
+ return new ServletOutputStreamWrapper(super.getOutputStream()) {
+ @Override
+ public void close() {
+ // even though we skip closes, we let local tests know not to close so that a full understanding can take
+ // place
+ assert Thread.currentThread().getStackTrace()[2].getClassName().matches(
+ "org\\.apache\\.(?:solr|lucene).*") ? false : true : "Attempted close of response output stream - never do this, you will spoil connection reuse and possibly disrupt a client";
+ stream = ClosedServletOutputStream.CLOSED_SERVLET_OUTPUT_STREAM;
+ }
+ };
}
+
};
} else {
return response;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/29620105/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
index 60b6d2f..4bcb8d8 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
@@ -519,8 +519,7 @@ public class SolrRequestParsers
@Override
public InputStream getStream() throws IOException {
- // Protect container owned streams from being closed by us, see SOLR-8933
- return new CloseShieldInputStream(req.getInputStream());
+ return req.getInputStream();
}
}
@@ -767,8 +766,7 @@ public class SolrRequestParsers
String userAgent = req.getHeader("User-Agent");
boolean isCurl = userAgent != null && userAgent.startsWith("curl/");
- // Protect container owned streams from being closed by us, see SOLR-8933
- FastInputStream input = FastInputStream.wrap( new CloseShieldInputStream(req.getInputStream()) );
+ FastInputStream input = FastInputStream.wrap(req.getInputStream());
if (isCurl) {
SolrParams params = autodetect(req, streams, input);
[42/50] [abbrv] lucene-solr:jira/solr-11779: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by ab...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0922e58c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0922e58c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0922e58c
Branch: refs/heads/jira/solr-11779
Commit: 0922e58c2c0867815d34c887b182754764cfaa4f
Parents: fb5b42a beaf3a4
Author: Uwe Schindler <us...@apache.org>
Authored: Sun May 6 15:54:04 2018 +0200
Committer: Uwe Schindler <us...@apache.org>
Committed: Sun May 6 15:54:04 2018 +0200
----------------------------------------------------------------------
solr/solr-ref-guide/src/blockjoin-faceting.adoc | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
[05/50] [abbrv] lucene-solr:jira/solr-11779: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
Posted by ab...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/400449f2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/400449f2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/400449f2
Branch: refs/heads/jira/solr-11779
Commit: 400449f2c734afb69c71c5d5732d70e6e8aaab8e
Parents: c94a83f 2c48794
Author: Karl Wright <Da...@gmail.com>
Authored: Sat Apr 28 09:14:05 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Sat Apr 28 09:14:05 2018 -0400
----------------------------------------------------------------------
solr/bin-test/utils/assert.sh | 28 +++++++-----
.../org/apache/solr/client/solrj/io/Lang.java | 4 +-
.../client/solrj/io/eval/OnesEvaluator.java | 47 ++++++++++++++++++++
.../client/solrj/io/eval/ZerosEvaluator.java | 47 ++++++++++++++++++++
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 45 +++++++++++++++++++
6 files changed, 159 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[07/50] [abbrv] lucene-solr:jira/solr-11779: LUCENE-8265:
WordDelimiter*Filter ignores keywords
Posted by ab...@apache.org.
LUCENE-8265: WordDelimiter*Filter ignores keywords
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fc0878cc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fc0878cc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fc0878cc
Branch: refs/heads/jira/solr-11779
Commit: fc0878cc2f97fdaa5206796ca5e0efa4988e7609
Parents: 4fba55c
Author: Michael Sokolov <so...@amazon.com>
Authored: Sun Apr 22 20:41:08 2018 +0000
Committer: Mike McCandless <mi...@apache.org>
Committed: Sat Apr 28 09:47:06 2018 -0400
----------------------------------------------------------------------
.../miscellaneous/WordDelimiterFilter.java | 13 +++++-
.../miscellaneous/WordDelimiterGraphFilter.java | 18 ++++++--
.../miscellaneous/TestWordDelimiterFilter.java | 43 +++++++++++++++-----
.../TestWordDelimiterGraphFilter.java | 32 +++++++++++++++
4 files changed, 90 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc0878cc/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilter.java
index 313386b..16edb3d 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilter.java
@@ -25,6 +25,7 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
@@ -164,7 +165,12 @@ public final class WordDelimiterFilter extends TokenFilter {
* "O'Neil's" => "O", "Neil"
*/
public static final int STEM_ENGLISH_POSSESSIVE = 256;
-
+
+ /**
+ * Suppresses processing terms with {@link KeywordAttribute#isKeyword()}=true.
+ */
+ public static final int IGNORE_KEYWORDS = 512;
+
/**
* If not null is the set of tokens to protect from being delimited
*
@@ -174,6 +180,7 @@ public final class WordDelimiterFilter extends TokenFilter {
private final int flags;
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
+ private final KeywordAttribute keywordAttribute = addAttribute(KeywordAttribute.class);;
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
private final PositionIncrementAttribute posIncAttribute = addAttribute(PositionIncrementAttribute.class);
private final TypeAttribute typeAttribute = addAttribute(TypeAttribute.class);
@@ -243,7 +250,9 @@ public final class WordDelimiterFilter extends TokenFilter {
if (!input.incrementToken()) {
return false;
}
-
+ if (has(IGNORE_KEYWORDS) && keywordAttribute.isKeyword()) {
+ return true;
+ }
int termLength = termAttribute.length();
char[] termBuffer = termAttribute.buffer();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc0878cc/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterGraphFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterGraphFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterGraphFilter.java
index 7949fa2..7d021c5 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterGraphFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterGraphFilter.java
@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
@@ -39,7 +40,7 @@ import org.apache.lucene.util.RamUsageEstimator;
* work correctly when this filter is used in the search-time analyzer. Unlike
* the deprecated {@link WordDelimiterFilter}, this token filter produces a
* correct token graph as output. However, it cannot consume an input token
- * graph correctly.
+ * graph correctly. Processing is suppressed by {@link KeywordAttribute#isKeyword()}=true.
*
* <p>
* Words are split into subwords with the following rules:
@@ -156,7 +157,12 @@ public final class WordDelimiterGraphFilter extends TokenFilter {
* "O'Neil's" => "O", "Neil"
*/
public static final int STEM_ENGLISH_POSSESSIVE = 256;
-
+
+ /**
+ * Suppresses processing terms with {@link KeywordAttribute#isKeyword()}=true.
+ */
+ public static final int IGNORE_KEYWORDS = 512;
+
/**
* If not null is the set of tokens to protect from being delimited
*
@@ -174,6 +180,7 @@ public final class WordDelimiterGraphFilter extends TokenFilter {
private char[][] bufferedTermParts = new char[4][];
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
+ private final KeywordAttribute keywordAttribute = addAttribute(KeywordAttribute.class);;
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
private final PositionIncrementAttribute posIncAttribute = addAttribute(PositionIncrementAttribute.class);
private final PositionLengthAttribute posLenAttribute = addAttribute(PositionLengthAttribute.class);
@@ -225,7 +232,8 @@ public final class WordDelimiterGraphFilter extends TokenFilter {
PRESERVE_ORIGINAL |
SPLIT_ON_CASE_CHANGE |
SPLIT_ON_NUMERICS |
- STEM_ENGLISH_POSSESSIVE)) != 0) {
+ STEM_ENGLISH_POSSESSIVE |
+ IGNORE_KEYWORDS)) != 0) {
throw new IllegalArgumentException("flags contains unrecognized flag: " + configurationFlags);
}
this.flags = configurationFlags;
@@ -335,7 +343,9 @@ public final class WordDelimiterGraphFilter extends TokenFilter {
if (input.incrementToken() == false) {
return false;
}
-
+ if (has(IGNORE_KEYWORDS) && keywordAttribute.isKeyword()) {
+ return true;
+ }
int termLength = termAttribute.length();
char[] termBuffer = termAttribute.buffer();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc0878cc/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
index 2804bfd..f945cd6 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
@@ -27,7 +27,6 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.util.IOUtils;
-import org.junit.Test;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.*;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE;
@@ -57,7 +56,6 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase {
}
***/
- @Test
public void testOffsets() throws IOException {
int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
// test that subwords and catenated subwords have
@@ -77,7 +75,6 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase {
new int[] { 6, 6, 6 });
}
- @Test
public void testOffsetChange() throws Exception {
int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
WordDelimiterFilter wdf = new WordDelimiterFilter(new CannedTokenStream(new Token("übelkeit)", 7, 16)), DEFAULT_WORD_DELIM_TABLE, flags, null);
@@ -88,7 +85,6 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase {
new int[] { 15 });
}
- @Test
public void testOffsetChange2() throws Exception {
int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
WordDelimiterFilter wdf = new WordDelimiterFilter(new CannedTokenStream(new Token("(übelkeit", 7, 17)), DEFAULT_WORD_DELIM_TABLE, flags, null);
@@ -99,7 +95,6 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase {
new int[] { 17 });
}
- @Test
public void testOffsetChange3() throws Exception {
int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
WordDelimiterFilter wdf = new WordDelimiterFilter(new CannedTokenStream(new Token("(übelkeit", 7, 16)), DEFAULT_WORD_DELIM_TABLE, flags, null);
@@ -110,7 +105,6 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase {
new int[] { 16 });
}
- @Test
public void testOffsetChange4() throws Exception {
int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
WordDelimiterFilter wdf = new WordDelimiterFilter(new CannedTokenStream(new Token("(foo,bar)", 7, 16)), DEFAULT_WORD_DELIM_TABLE, flags, null);
@@ -129,7 +123,6 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase {
assertTokenStreamContents(wdf, output);
}
- @Test
public void testSplits() throws Exception {
doSplit("basic-split","basic","split");
doSplit("camelCase","camel","Case");
@@ -175,7 +168,6 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase {
/*
* Test option that allows disabling the special "'s" stemming, instead treating the single quote like other delimiters.
*/
- @Test
public void testPossessives() throws Exception {
doSplitPossessive(1, "ra's", "ra");
doSplitPossessive(0, "ra's", "ra", "s");
@@ -204,7 +196,6 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase {
}
}
- @Test
public void testPositionIncrements() throws Exception {
final int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
final CharArraySet protWords = new CharArraySet(new HashSet<>(Arrays.asList("NUTCH")), false);
@@ -323,6 +314,38 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase {
IOUtils.close(a, a2, a3);
}
+ public void testKeywordFilter() throws Exception {
+ assertAnalyzesTo(keywordTestAnalyzer(GENERATE_WORD_PARTS),
+ "abc-def klm-nop kpop",
+ new String[] {"abc", "def", "klm", "nop", "kpop"});
+ assertAnalyzesTo(keywordTestAnalyzer(GENERATE_WORD_PARTS | IGNORE_KEYWORDS),
+ "abc-def klm-nop kpop",
+ new String[] {"abc", "def", "klm-nop", "kpop"},
+ new int[]{0, 4, 8, 16},
+ new int[]{3, 7, 15, 20},
+ null,
+ new int[]{1, 1, 1, 1},
+ null,
+ false);
+ }
+
+ private Analyzer keywordTestAnalyzer(int flags) throws Exception {
+ return new Analyzer() {
+ @Override
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ KeywordMarkerFilter kFilter = new KeywordMarkerFilter(tokenizer) {
+ private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
+ @Override public boolean isKeyword() {
+ // Marks terms starting with the letter 'k' as keywords
+ return term.toString().charAt(0) == 'k';
+ }
+ };
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(kFilter, flags, null));
+ }
+ };
+ }
+
/** concat numbers + words + all */
public void testLotsOfConcatenating() throws Exception {
final int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_WORDS | CATENATE_NUMBERS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
@@ -346,7 +369,7 @@ public class TestWordDelimiterFilter extends BaseTokenStreamTestCase {
false);
a.close();
}
-
+
/** concat numbers + words + all + preserve original */
public void testLotsOfConcatenating2() throws Exception {
final int flags = PRESERVE_ORIGINAL | GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_WORDS | CATENATE_NUMBERS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fc0878cc/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterGraphFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterGraphFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterGraphFilter.java
index 7516a23..61ae6c0 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterGraphFilter.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterGraphFilter.java
@@ -309,6 +309,38 @@ public class TestWordDelimiterGraphFilter extends BaseTokenStreamTestCase {
IOUtils.close(a, a2, a3);
}
+ public void testKeywordFilter() throws Exception {
+ assertAnalyzesTo(keywordTestAnalyzer(GENERATE_WORD_PARTS),
+ "abc-def klm-nop kpop",
+ new String[] {"abc", "def", "klm", "nop", "kpop"});
+ assertAnalyzesTo(keywordTestAnalyzer(GENERATE_WORD_PARTS | IGNORE_KEYWORDS),
+ "abc-def klm-nop kpop",
+ new String[] {"abc", "def", "klm-nop", "kpop"},
+ new int[]{0, 4, 8, 16},
+ new int[]{3, 7, 15, 20},
+ null,
+ new int[]{1, 1, 1, 1},
+ null,
+ false);
+ }
+
+ private Analyzer keywordTestAnalyzer(int flags) throws Exception {
+ return new Analyzer() {
+ @Override
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ KeywordMarkerFilter kFilter = new KeywordMarkerFilter(tokenizer) {
+ private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
+ @Override public boolean isKeyword() {
+ // Marks terms starting with the letter 'k' as keywords
+ return term.toString().charAt(0) == 'k';
+ }
+ };
+ return new TokenStreamComponents(tokenizer, new WordDelimiterGraphFilter(kFilter, flags, null));
+ }
+ };
+ }
+
/** concat numbers + words + all */
public void testLotsOfConcatenating() throws Exception {
final int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_WORDS | CATENATE_NUMBERS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
[37/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-12293: Updates need
to use their own connection pool to maintain connection reuse and prevent
spurious recoveries.
Posted by ab...@apache.org.
SOLR-12293: Updates need to use their own connection pool to maintain connection reuse and prevent spurious recoveries.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3a2572db
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3a2572db
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3a2572db
Branch: refs/heads/jira/solr-11779
Commit: 3a2572db793b47a6648fae8288a5c8815b689cd1
Parents: 2962010
Author: Mark Miller <ma...@apache.org>
Authored: Fri May 4 20:02:56 2018 -0500
Committer: Mark Miller <ma...@apache.org>
Committed: Fri May 4 20:02:56 2018 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../org/apache/solr/cloud/SyncStrategy.java | 2 +-
.../org/apache/solr/core/BlobRepository.java | 16 +++--
.../org/apache/solr/handler/IndexFetcher.java | 2 +-
.../admin/AutoscalingHistoryHandler.java | 2 +-
.../reporters/solr/SolrClusterReporter.java | 2 +-
.../reporters/solr/SolrShardReporter.java | 2 +-
.../solr/security/PKIAuthenticationPlugin.java | 9 ++-
.../apache/solr/servlet/SolrDispatchFilter.java | 2 +-
.../solr/update/DefaultSolrCoreState.java | 5 +-
.../java/org/apache/solr/update/PeerSync.java | 2 +-
.../apache/solr/update/SolrCmdDistributor.java | 8 +--
.../solr/update/StreamingSolrClients.java | 2 +-
.../apache/solr/update/UpdateShardHandler.java | 68 ++++++++++++--------
.../processor/DistributedUpdateProcessor.java | 2 +-
15 files changed, 73 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f74e2fd..efebe91 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -209,6 +209,9 @@ Bug Fixes
* SOLR-12290: Do not close any servlet streams and improve our servlet stream closing prevention code for users
and devs. (Mark Miller)
+* SOLR-12293: Updates need to use their own connection pool to maintain connection reuse and prevent spurious
+ recoveries. (Mark Miller)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
index a0f00bb2..4368af9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
+++ b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
@@ -69,7 +69,7 @@ public class SyncStrategy {
public SyncStrategy(CoreContainer cc) {
UpdateShardHandler updateShardHandler = cc.getUpdateShardHandler();
- client = updateShardHandler.getHttpClient();
+ client = updateShardHandler.getDefaultHttpClient();
shardHandler = cc.getShardHandlerFactory().getShardHandler();
updateExecutor = updateShardHandler.getUpdateExecutor();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/core/BlobRepository.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
index 48dd70a..0cf6dfb 100644
--- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
@@ -33,6 +33,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
+import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
@@ -43,6 +44,7 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.util.SimplePostTool;
import org.apache.zookeeper.server.ByteBufferInputStream;
import org.slf4j.Logger;
@@ -155,16 +157,20 @@ public class BlobRepository {
Replica replica = getSystemCollReplica();
String url = replica.getStr(BASE_URL_PROP) + "/" + CollectionAdminParams.SYSTEM_COLL + "/blob/" + key + "?wt=filestream";
- HttpClient httpClient = coreContainer.getUpdateShardHandler().getHttpClient();
+ HttpClient httpClient = coreContainer.getUpdateShardHandler().getDefaultHttpClient();
HttpGet httpGet = new HttpGet(url);
ByteBuffer b;
+ HttpResponse response = null;
+ HttpEntity entity = null;
try {
- HttpResponse entity = httpClient.execute(httpGet);
- int statusCode = entity.getStatusLine().getStatusCode();
+ response = httpClient.execute(httpGet);
+ entity = response.getEntity();
+ int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != 200) {
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no such blob or version available: " + key);
}
- try (InputStream is = entity.getEntity().getContent()) {
+
+ try (InputStream is = entity.getContent()) {
b = SimplePostTool.inputStreamToByteArray(is);
}
} catch (Exception e) {
@@ -174,7 +180,7 @@ public class BlobRepository {
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "could not load : " + key, e);
}
} finally {
- httpGet.releaseConnection();
+ Utils.consumeFully(entity);
}
return b;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 9fde19c..93f0edf 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -245,7 +245,7 @@ public class IndexFetcher {
httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword);
httpClientParams.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, useCompression);
- return HttpClientUtil.createClient(httpClientParams, core.getCoreContainer().getUpdateShardHandler().getConnectionManager(), true);
+ return HttpClientUtil.createClient(httpClientParams, core.getCoreContainer().getUpdateShardHandler().getDefaultConnectionManager(), true);
}
public IndexFetcher(final NamedList initArgs, final ReplicationHandler handler, final SolrCore sc) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/handler/admin/AutoscalingHistoryHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/AutoscalingHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/AutoscalingHistoryHandler.java
index 52ad100..69fac0b 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/AutoscalingHistoryHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/AutoscalingHistoryHandler.java
@@ -126,7 +126,7 @@ public class AutoscalingHistoryHandler extends RequestHandlerBase implements Per
}
}
try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(Collections.singletonList(coreContainer.getZkController().getZkServerAddress()), Optional.empty())
- .withHttpClient(coreContainer.getUpdateShardHandler().getHttpClient())
+ .withHttpClient(coreContainer.getUpdateShardHandler().getDefaultHttpClient())
.build()) {
QueryResponse qr = cloudSolrClient.query(collection, params);
rsp.setAllValues(qr.getResponse());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java
index 081dff0..9e8861e 100644
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java
@@ -207,7 +207,7 @@ public class SolrClusterReporter extends SolrCoreContainerReporter {
log.info("Turning off node reporter, period=" + period);
return;
}
- HttpClient httpClient = cc.getUpdateShardHandler().getHttpClient();
+ HttpClient httpClient = cc.getUpdateShardHandler().getDefaultHttpClient();
ZkController zk = cc.getZkController();
String reporterId = zk.getNodeName();
reporter = SolrReporter.Builder.forReports(metricManager, reports)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java
index c477e47..af794fc 100644
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java
@@ -154,7 +154,7 @@ public class SolrShardReporter extends SolrCoreReporter {
.cloudClient(false) // we want to send reports specifically to a selected leader instance
.skipAggregateValues(true) // we don't want to transport details of aggregates
.skipHistograms(true) // we don't want to transport histograms
- .build(core.getCoreContainer().getUpdateShardHandler().getHttpClient(), new LeaderUrlSupplier(core));
+ .build(core.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient(), new LeaderUrlSupplier(core));
reporter.start(period, TimeUnit.SECONDS);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java
index 172659a..877e4f1 100644
--- a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java
+++ b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java
@@ -30,6 +30,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
@@ -195,12 +196,14 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt
PublicKey getRemotePublicKey(String nodename) {
if (!cores.getZkController().getZkStateReader().getClusterState().getLiveNodes().contains(nodename)) return null;
String url = cores.getZkController().getZkStateReader().getBaseUrlForNodeName(nodename);
+ HttpEntity entity = null;
try {
String uri = url + PATH + "?wt=json&omitHeader=true";
log.debug("Fetching fresh public key from : {}",uri);
- HttpResponse rsp = cores.getUpdateShardHandler().getHttpClient()
+ HttpResponse rsp = cores.getUpdateShardHandler().getDefaultHttpClient()
.execute(new HttpGet(uri), HttpClientUtil.createNewHttpClientRequestContext());
- byte[] bytes = EntityUtils.toByteArray(rsp.getEntity());
+ entity = rsp.getEntity();
+ byte[] bytes = EntityUtils.toByteArray(entity);
Map m = (Map) Utils.fromJSON(bytes);
String key = (String) m.get("key");
if (key == null) {
@@ -215,6 +218,8 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt
} catch (Exception e) {
log.error("Exception trying to get public key from : " + url, e);
return null;
+ } finally {
+ Utils.consumeFully(entity);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index 8f32a7d..0419f97 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@ -172,7 +172,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
coresInit = createCoreContainer(solrHome == null ? SolrResourceLoader.locateSolrHome() : Paths.get(solrHome),
extraProperties);
- this.httpClient = coresInit.getUpdateShardHandler().getHttpClient();
+ this.httpClient = coresInit.getUpdateShardHandler().getDefaultHttpClient();
setupJvmMetrics(coresInit);
log.debug("user.dir=" + System.getProperty("user.dir"));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
index 739604f..cc79e3c 100644
--- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
+++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
@@ -360,11 +360,8 @@ public final class DefaultSolrCoreState extends SolrCoreState implements Recover
// have to 'wait in line' a bit or bail if a recovery is
// already queued up - the recovery execution itself is run
// in another thread on another 'recovery' executor.
- // The update executor is interrupted on shutdown and should
- // not do disk IO.
- // The recovery executor is not interrupted on shutdown.
//
- // avoid deadlock: we can't use the recovery executor here
+ // avoid deadlock: we can't use the recovery executor here!
cc.getUpdateShardHandler().getUpdateExecutor().submit(recoveryTask);
} catch (RejectedExecutionException e) {
// fine, we are shutting down
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/update/PeerSync.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java
index 3511a6c..8931e63 100644
--- a/solr/core/src/java/org/apache/solr/update/PeerSync.java
+++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java
@@ -156,7 +156,7 @@ public class PeerSync implements SolrMetricProducer {
this.cantReachIsSuccess = cantReachIsSuccess;
this.getNoVersionsIsSuccess = getNoVersionsIsSuccess;
this.doFingerprint = doFingerprint && !("true".equals(System.getProperty("solr.disableFingerprint")));
- this.client = core.getCoreContainer().getUpdateShardHandler().getHttpClient();
+ this.client = core.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient();
this.onlyIfActive = onlyIfActive;
uhandler = core.getUpdateHandler();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 08b397f..80e2253 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -49,7 +49,6 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
/**
@@ -67,7 +66,6 @@ public class SolrCmdDistributor implements Closeable {
private final List<Error> allErrors = new ArrayList<>();
private final List<Error> errors = Collections.synchronizedList(new ArrayList<Error>());
- private final ExecutorService updateExecutor;
private final CompletionService<Object> completionService;
private final Set<Future<Object>> pending = new HashSet<>();
@@ -78,16 +76,14 @@ public class SolrCmdDistributor implements Closeable {
public SolrCmdDistributor(UpdateShardHandler updateShardHandler) {
this.clients = new StreamingSolrClients(updateShardHandler);
- this.updateExecutor = updateShardHandler.getUpdateExecutor();
- this.completionService = new ExecutorCompletionService<>(updateExecutor);
+ this.completionService = new ExecutorCompletionService<>(updateShardHandler.getUpdateExecutor());
}
public SolrCmdDistributor(StreamingSolrClients clients, int maxRetriesOnForward, int retryPause) {
this.clients = clients;
this.maxRetriesOnForward = maxRetriesOnForward;
this.retryPause = retryPause;
- this.updateExecutor = clients.getUpdateExecutor();
- completionService = new ExecutorCompletionService<>(updateExecutor);
+ completionService = new ExecutorCompletionService<>(clients.getUpdateExecutor());
}
public void finish() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java b/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
index 316aa45..cc1f755 100644
--- a/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
+++ b/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
@@ -54,7 +54,7 @@ public class StreamingSolrClients {
public StreamingSolrClients(UpdateShardHandler updateShardHandler) {
this.updateExecutor = updateShardHandler.getUpdateExecutor();
- httpClient = updateShardHandler.getHttpClient();
+ httpClient = updateShardHandler.getUpdateOnlyHttpClient();
}
public List<Error> getErrors() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
index fa752a0..ecc843f 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
@@ -27,7 +27,6 @@ import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.cloud.RecoveryStrategy;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
@@ -51,8 +50,6 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
/*
* A downside to configuring an upper bound will be big update reorders (when that upper bound is hit)
* and then undetected shard inconsistency as a result.
- * This update executor is used for different things too... both update streams (which may be very long lived)
- * and control messages (peersync? LIR?) and could lead to starvation if limited.
* Therefore this thread pool is left unbounded. See SOLR-8205
*/
private ExecutorService updateExecutor = ExecutorUtil.newMDCAwareCachedThreadPool(
@@ -60,20 +57,28 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
private ExecutorService recoveryExecutor;
- private final CloseableHttpClient client;
+ private final CloseableHttpClient updateOnlyClient;
+
+ private final CloseableHttpClient defaultClient;
- private final InstrumentedPoolingHttpClientConnectionManager clientConnectionManager;
+ private final InstrumentedPoolingHttpClientConnectionManager updateOnlyConnectionManager;
+
+ private final InstrumentedPoolingHttpClientConnectionManager defaultConnectionManager;
private final InstrumentedHttpRequestExecutor httpRequestExecutor;
+
private final Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricRegistry registry;
public UpdateShardHandler(UpdateShardHandlerConfig cfg) {
- clientConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
+ updateOnlyConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
+ defaultConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
if (cfg != null ) {
- clientConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
- clientConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
+ updateOnlyConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
+ updateOnlyConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
+ defaultConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
+ defaultConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
}
ModifiableSolrParams clientParams = new ModifiableSolrParams();
@@ -90,8 +95,10 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
}
}
+
httpRequestExecutor = new InstrumentedHttpRequestExecutor(metricNameStrategy);
- client = HttpClientUtil.createClient(clientParams, clientConnectionManager, false, httpRequestExecutor);
+ updateOnlyClient = HttpClientUtil.createClient(clientParams, updateOnlyConnectionManager, false, httpRequestExecutor);
+ defaultClient = HttpClientUtil.createClient(clientParams, defaultConnectionManager, false, httpRequestExecutor);
// following is done only for logging complete configuration.
// The maxConnections and maxConnectionsPerHost have already been specified on the connection manager
@@ -99,7 +106,8 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, cfg.getMaxUpdateConnections());
clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, cfg.getMaxUpdateConnectionsPerHost());
}
- log.debug("Created UpdateShardHandler HTTP client with params: {}", clientParams);
+ log.debug("Created default UpdateShardHandler HTTP client with params: {}", clientParams);
+ log.debug("Created update only UpdateShardHandler HTTP client with params: {}", clientParams);
ThreadFactory recoveryThreadFactory = new SolrjNamedThreadFactory("recoveryExecutor");
if (cfg != null && cfg.getMaxRecoveryThreads() > 0) {
@@ -120,10 +128,10 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) {
registry = manager.registry(registryName);
String expandedScope = SolrMetricManager.mkName(scope, getCategory().name());
- clientConnectionManager.initializeMetrics(manager, registryName, tag, expandedScope);
- httpRequestExecutor.initializeMetrics(manager, registryName, tag, expandedScope);
+ updateOnlyConnectionManager.initializeMetrics(manager, registryName, tag, expandedScope);
+ defaultConnectionManager.initializeMetrics(manager, registryName, tag, expandedScope);
updateExecutor = MetricUtils.instrumentedExecutorService(updateExecutor, this, registry,
- SolrMetricManager.mkName("updateExecutor", expandedScope, "threadPool"));
+ SolrMetricManager.mkName("updateOnlyExecutor", expandedScope, "threadPool"));
recoveryExecutor = MetricUtils.instrumentedExecutorService(recoveryExecutor, this, registry,
SolrMetricManager.mkName("recoveryExecutor", expandedScope, "threadPool"));
}
@@ -148,31 +156,33 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
return registry;
}
- public HttpClient getHttpClient() {
- return client;
+ // if you are looking for a client to use, it's probably this one.
+ public HttpClient getDefaultHttpClient() {
+ return defaultClient;
}
- /**
- * This method returns an executor that is not meant for disk IO and that will
- * be interrupted on shutdown.
+ // don't introduce a bug, this client is for sending updates only!
+ public HttpClient getUpdateOnlyHttpClient() {
+ return updateOnlyClient;
+ }
+
+
+ /**
+ * This method returns an executor that is meant for non search related tasks.
*
- * @return an executor for update related activities that do not do disk IO.
+ * @return an executor for update side related activities.
*/
public ExecutorService getUpdateExecutor() {
return updateExecutor;
}
-
- public PoolingHttpClientConnectionManager getConnectionManager() {
- return clientConnectionManager;
+ public PoolingHttpClientConnectionManager getDefaultConnectionManager() {
+ return defaultConnectionManager;
}
/**
- * In general, RecoveryStrategy threads do not do disk IO, but they open and close SolrCores
- * in async threads, among other things, and can trigger disk IO, so we use this alternate
- * executor rather than the 'updateExecutor', which is interrupted on shutdown.
*
- * @return executor for {@link RecoveryStrategy} thread which will not be interrupted on close.
+ * @return executor for recovery operations
*/
public ExecutorService getRecoveryExecutor() {
return recoveryExecutor;
@@ -186,8 +196,10 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
} catch (Exception e) {
SolrException.log(log, e);
} finally {
- HttpClientUtil.close(client);
- clientConnectionManager.close();
+ HttpClientUtil.close(updateOnlyClient);
+ HttpClientUtil.close(defaultClient);
+ updateOnlyConnectionManager.close();
+ defaultConnectionManager.close();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3a2572db/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index d5e4194..d9cceee 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -1285,7 +1285,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
NamedList<Object> rsp = null;
try (HttpSolrClient hsc = new HttpSolrClient.Builder(leaderUrl).
- withHttpClient(updateShardHandler.getHttpClient()).build()) {
+ withHttpClient(updateShardHandler.getUpdateOnlyHttpClient()).build()) {
rsp = hsc.request(ur);
} catch (SolrServerException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Error during fetching [" + id +
[02/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-11749: Fix logic
errors in some assert funcs
Posted by ab...@apache.org.
SOLR-11749: Fix logic errors in some assert funcs
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/43c086a0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/43c086a0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/43c086a0
Branch: refs/heads/jira/solr-11779
Commit: 43c086a002b3488a6fdd6a71ce1879a2538cce4e
Parents: e263ae30
Author: Jason Gerlowski <ge...@apache.org>
Authored: Fri Apr 27 09:46:11 2018 -0400
Committer: Jason Gerlowski <ge...@apache.org>
Committed: Fri Apr 27 09:46:11 2018 -0400
----------------------------------------------------------------------
solr/bin-test/utils/assert.sh | 28 ++++++++++++++++------------
1 file changed, 16 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/43c086a0/solr/bin-test/utils/assert.sh
----------------------------------------------------------------------
diff --git a/solr/bin-test/utils/assert.sh b/solr/bin-test/utils/assert.sh
index 7c0f323..a4bd56d 100644
--- a/solr/bin-test/utils/assert.sh
+++ b/solr/bin-test/utils/assert.sh
@@ -14,13 +14,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-ASSERT_SUCCEEDED=1
-ASSERT_FAILURE=0
+ASSERT_SUCCESS=0
+ASSERT_FAILURE=1
+
+TEST_SUCCESS=0
+TEST_FAILURE=1
function assert_cmd_succeeded() {
retval=$?
- if [[ $? -ne 0 ]]; then
+ if [[ $retval -ne 0 ]]; then
echo "Expected command $1 to succeed, but exited with $retval"
return $ASSERT_FAILURE
fi
@@ -31,7 +34,7 @@ function assert_cmd_succeeded() {
function assert_cmd_failed() {
retval=$?
- if [[ $? -eq 0 ]]; then
+ if [[ $retval -eq 0 ]]; then
echo "Expected command $1 to fail, but exited with $retval"
return $ASSERT_FAILURE
fi
@@ -67,9 +70,9 @@ function assert_collection_exists() {
local coll_name=$1
local coll_list=$(bin/solr zk ls /collections -z localhost:9983)
- for coll in "$coll_list";
+ for coll in $coll_list;
do
- if [[ $(echo $coll | tr -d " ") -eq $coll_name ]]; then
+ if [[ $(echo $coll | tr -d " ") == $coll_name ]]; then
return $ASSERT_SUCCESS
fi
done
@@ -81,9 +84,10 @@ function assert_collection_exists() {
function assert_collection_doesnt_exist() {
local coll_name=$1
local coll_list=$(bin/solr zk ls /collections -z localhost:9983)
- for coll in "$coll_list";
+ for coll in $coll_list;
do
- if [[ $(echo $coll | tr -d " ") -eq $coll_name ]]; then
+ echo "Comparing $coll to $coll_name"
+ if [[ $(echo $coll | tr -d " ") == "$coll_name" ]]; then
echo "Expected not to find collection [$coll_name], but it exists"
return $ASSERT_FAILURE
fi
@@ -96,9 +100,9 @@ function assert_config_exists() {
local config_name=$1
local config_list=$(bin/solr zk ls /configs -z localhost:9983)
- for config in "$config_list";
+ for config in $config_list;
do
- if [[ $(echo $config | tr -d " ") -eq $config_name ]]; then
+ if [[ $(echo $config | tr -d " ") == $config_name ]]; then
return $ASSERT_SUCCESS
fi
done
@@ -111,9 +115,9 @@ function assert_config_doesnt_exist() {
local config_name=$1
local config_list=$(bin/solr zk ls /configs -z localhost:9983)
- for config in "$config_list";
+ for config in $config_list;
do
- if [[ $(echo $config | tr -d " ") -eq $config_name ]]; then
+ if [[ $(echo $config | tr -d " ") == $config_name ]]; then
echo "Expected not to find config [$config_name], but it exists"
return $ASSERT_FAILURE
fi