You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 09:49:30 UTC
[lucene] branch jira/solr-12730 created (now e12aeba)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a change to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git.
at e12aeba SOLR-12730: Cleanup.
This branch includes the following new commits:
new 3d91b8e Initial patch.
new b47df35 Merge branch 'master' into jira/solr-12730
new efb73a3 SOLR-12730: add more debug logging.
new 514fef4 Added more debug logging on how and why commits are reserved and deleted
new 328b913 Enable info stream logging including ref counts Add debug printlns in directory factory Pass through expunge and related params in DUPF
new 2bff401 Adding more logging around when files are reference counted Added custom CMS impl which sets thread names with MDC context information
new 2138b83 Added custom merge scheduler which prints MDC logging context in merge scheduler threads Adding more docs before commit so that old commit points can be cleared
new 2ebe7d9 Merge branch 'master' into jira/solr-12730
new e12aeba SOLR-12730: Cleanup.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[lucene] 08/09: Merge branch 'master' into jira/solr-12730
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 2ebe7d9a4511327990179e995cfd896f7ddba73c
Merge: 2138b83 45b772f
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Mon Nov 5 12:12:02 2018 +0100
Merge branch 'master' into jira/solr-12730
lucene/CHANGES.txt | 16 +
.../lucene/geo/{Polygon2D.java => EdgeTree.java} | 330 ++++----------
.../org/apache/lucene/geo/GeoEncodingUtils.java | 4 +
.../src/java/org/apache/lucene/geo/GeoUtils.java | 14 +
.../src/java/org/apache/lucene/geo/Polygon2D.java | 478 +++------------------
.../src/java/org/apache/lucene/geo/Rectangle.java | 7 +
.../org/apache/lucene/document/LatLonShape.java | 24 +-
.../document/LatLonShapeBoundingBoxQuery.java | 155 ++++++-
...PolygonQuery.java => LatLonShapeLineQuery.java} | 76 ++--
.../lucene/document/LatLonShapePolygonQuery.java | 2 +-
.../apache/lucene/document/LatLonShapeQuery.java | 4 +-
.../src/java/org/apache/lucene/geo/Line.java | 10 +
.../src/java/org/apache/lucene/geo/Line2D.java | 40 ++
.../java/org/apache/lucene/geo/Tessellator.java | 55 +--
.../lucene/document/BaseLatLonShapeTestCase.java | 165 ++++++-
.../document/TestLatLonLineShapeQueries.java | 77 +++-
.../document/TestLatLonPointShapeQueries.java | 56 ++-
.../document/TestLatLonPolygonShapeQueries.java | 44 +-
.../org/apache/lucene/geo/TestTessellator.java | 32 ++
.../org/apache/lucene/spatial3d/Geo3DUtil.java | 9 +-
.../apache/lucene/spatial3d/TestGeo3DPoint.java | 38 +-
solr/CHANGES.txt | 128 ++----
.../solr/cloud/autoscaling/IndexSizeTrigger.java | 4 +-
.../solr/core/ByteBuffersDirectoryFactory.java | 46 ++
.../apache/solr/handler/admin/ClusterStatus.java | 12 +
.../solr/handler/component/FacetComponent.java | 3 +-
.../component/PivotFacetFieldValueCollection.java | 34 +-
.../search/facet/FacetFieldProcessorByHashDV.java | 16 +-
.../apache/solr/search/facet/UniqueBlockAgg.java | 8 +-
.../apache/solr/search/facet/UniqueSlotAcc.java | 3 +
.../CloneFieldUpdateProcessorFactory.java | 1 +
.../apache/solr/cloud/DistribCursorPagingTest.java | 2 +
.../org/apache/solr/cloud/TestCloudPivotFacet.java | 182 +-------
.../cloud/api/collections/TestCollectionAPI.java | 37 +-
.../cloud/autoscaling/IndexSizeTriggerTest.java | 81 ++++
.../solr/core/ByteBuffersDirectoryFactoryTest.java | 72 ++++
.../solr/core/CachingDirectoryFactoryTest.java | 7 +-
.../org/apache/solr/core/DirectoryFactoryTest.java | 45 +-
.../component/DistributedFacetPivotLargeTest.java | 88 ++++
.../org/apache/solr/search/QueryEqualityTest.java | 63 ++-
.../solr/search/TestCollapseQParserPlugin.java | 11 +-
.../solr/search/TestExtendedDismaxParser.java | 68 ++-
.../solr/search/TestFoldingMultitermQuery.java | 6 +-
.../test/org/apache/solr/search/TestLRUCache.java | 9 +-
.../org/apache/solr/search/TestQueryTypes.java | 14 +-
.../solr/search/TestReRankQParserPlugin.java | 11 +-
.../org/apache/solr/search/TestRealTimeGet.java | 167 +++----
.../org/apache/solr/search/TestSolr4Spatial.java | 10 +-
.../apache/solr/search/TestSolrQueryParser.java | 19 +-
.../apache/solr/search/facet/TestJsonFacets.java | 65 +++
solr/solr-ref-guide/src/collections-api.adoc | 2 +-
solr/solr-ref-guide/src/scalar-math.adoc | 4 +-
.../src/solrcloud-autoscaling-triggers.adoc | 7 +
solr/solr-ref-guide/src/statistics.adoc | 81 +++-
.../client/solrj/cloud/autoscaling/Suggester.java | 8 +-
.../solrj/impl/HttpClusterStateProvider.java | 34 +-
.../java/org/apache/solr/client/solrj/io/Lang.java | 1 +
.../client/solrj/io/eval/PercentileEvaluator.java | 25 +-
.../solr/client/solrj/io/eval/RecipEvaluator.java | 52 +++
.../org/apache/solr/common/NavigableObject.java | 2 +-
.../client/solrj/cloud/autoscaling/TestPolicy.java | 25 +-
.../org/apache/solr/client/solrj/io/TestLang.java | 2 +-
.../client/solrj/io/stream/MathExpressionTest.java | 42 ++
.../solrj/io/stream/StreamDecoratorTest.java | 2 +
.../solrj/io/stream/StreamExpressionTest.java | 2 +
65 files changed, 1773 insertions(+), 1364 deletions(-)
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
index e41297e,31322c0..27daec2
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
@@@ -91,7 -89,7 +91,7 @@@ public class IndexSizeTrigger extends T
super(TriggerEventType.INDEXSIZE, name);
TriggerUtils.validProperties(validProperties,
ABOVE_BYTES_PROP, ABOVE_DOCS_PROP, BELOW_BYTES_PROP, BELOW_DOCS_PROP,
- COLLECTIONS_PROP, MAX_OPS_PROP, SPLIT_FUZZ_PROP);
- COLLECTIONS_PROP, MAX_OPS_PROP, SPLIT_METHOD_PROP);
++ COLLECTIONS_PROP, MAX_OPS_PROP, SPLIT_METHOD_PROP, SPLIT_FUZZ_PROP);
}
@Override
[lucene] 07/09: Added custom merge scheduler which prints MDC
logging context in merge scheduler threads Adding more docs before commit
so that old commit points can be cleared
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 2138b8392d3e989a3ab513a631dac0993b385fad
Author: Shalin Shekhar Mangar <sh...@apache.org>
AuthorDate: Thu Nov 1 19:41:51 2018 +0530
Added custom merge scheduler which prints MDC logging context in merge scheduler threads
Adding more docs before commit so that old commit points can be cleared
---
.../org/apache/solr/update/SolrIndexConfig.java | 50 +++++++++++-----------
.../cloud/autoscaling/IndexSizeTriggerTest.java | 13 +++++-
2 files changed, 36 insertions(+), 27 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
index 38f16c2..0df2b16 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
@@ -286,7 +286,30 @@ public class SolrIndexConfig implements MapSerializable {
private MergeScheduler buildMergeScheduler(IndexSchema schema) {
String msClassName = mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME : mergeSchedulerInfo.className;
- MergeScheduler scheduler = new TestCMS(); // todo nocommit
+ // todo nocommit -- remove this scheduler instance with proper MDC logging support inside merge scheduler threads
+ MergeScheduler scheduler = new ConcurrentMergeScheduler() {
+ @Override
+ protected synchronized MergeThread getMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) throws IOException {
+ MergeThread mergeThread = super.getMergeThread(writer, merge);
+ final Map<String, String> submitterContext = MDC.getCopyOfContextMap();
+ StringBuilder contextString = new StringBuilder();
+ if (submitterContext != null) {
+ Collection<String> values = submitterContext.values();
+
+ for (String value : values) {
+ contextString.append(value + " ");
+ }
+ if (contextString.length() > 1) {
+ contextString.setLength(contextString.length() - 1);
+ }
+ }
+
+ String ctxStr = contextString.toString().replace("/", "//");
+ final String submitterContextStr = ctxStr.length() <= 512 ? ctxStr : ctxStr.substring(0, 512);
+ mergeThread.setName(mergeThread.getName() + "-processing-" + submitterContextStr);
+ return mergeThread;
+ }
+ };
if (mergeSchedulerInfo != null) {
// LUCENE-5080: these two setters are removed, so we have to invoke setMaxMergesAndThreads
@@ -314,29 +337,4 @@ public class SolrIndexConfig implements MapSerializable {
return scheduler;
}
-
- static class TestCMS extends ConcurrentMergeScheduler {
- @Override
- protected synchronized MergeThread getMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) throws IOException {
- MergeThread mergeThread = super.getMergeThread(writer, merge);
- final Map<String, String> submitterContext = MDC.getCopyOfContextMap();
- StringBuilder contextString = new StringBuilder();
- if (submitterContext != null) {
- Collection<String> values = submitterContext.values();
-
- for (String value : values) {
- contextString.append(value + " ");
- }
- if (contextString.length() > 1) {
- contextString.setLength(contextString.length() - 1);
- }
- }
-
- String ctxStr = contextString.toString().replace("/", "//");
- final String submitterContextStr = ctxStr.length() <= 512 ? ctxStr : ctxStr.substring(0, 512);
- mergeThread.setName(mergeThread.getName() + "-processing-" + submitterContextStr);
- return mergeThread;
- }
- }
-
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index 84833dc..76b1a55 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -67,7 +67,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_P
/**
*
*/
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.handler.admin.MetricsHandler=DEBUG;org.apache.solr.core.SolrDeletionPolicy=DEBUG;org.apache.solr.core.IndexDeletionPolicyWrapper=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUGorg.apache.solr.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.O [...]
+//@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.handler.admin.MetricsHandler=DEBUG;org.apache.solr.core.SolrDeletionPolicy=DEBUG;org.apache.solr.core.IndexDeletionPolicyWrapper=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUGorg.apache.solr.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud [...]
public class IndexSizeTriggerTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -652,6 +652,17 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
// wait for the segments to merge to reduce the index size
cloudManager.getTimeSource().sleep(50000);
+ // add some docs so that every shard gets an update
+ // we can reduce the number of docs here but this also works
+ for (int j = 0; j < 1; j++) {
+ UpdateRequest ureq = new UpdateRequest();
+ ureq.setParam("collection", collectionName);
+ for (int i = 0; i < 98; i++) {
+ ureq.add("id", "id-" + (i * 100) + "-" + j);
+ }
+ solrClient.request(ureq);
+ }
+
log.info("-- requesting commit");
solrClient.commit(collectionName, true, true);
[lucene] 05/09: Enable info stream logging including ref counts Add
debug printlns in directory factory Pass through expunge and related params
in DUPF
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 328b913b4697f15353969bbf60a839dda2fe98ea
Author: Shalin Shekhar Mangar <sh...@apache.org>
AuthorDate: Tue Oct 30 17:11:15 2018 +0530
Enable info stream logging including ref counts
Add debug printlns in directory factory
Pass through expunge and related params in DUPF
---
lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java | 2 +-
lucene/core/src/java/org/apache/lucene/store/FSDirectory.java | 1 +
solr/core/src/java/org/apache/solr/core/DirectoryFactory.java | 2 ++
solr/core/src/java/org/apache/solr/core/SolrCore.java | 1 +
.../apache/solr/update/processor/DistributedUpdateProcessor.java | 6 ++++--
.../test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml | 1 +
.../org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java | 2 +-
7 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
index cc9d2e0..f76cb98 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
@@ -108,7 +108,7 @@ final class IndexFileDeleter implements Closeable {
/** Change to true to see details of reference counts when
* infoStream is enabled */
- public static boolean VERBOSE_REF_COUNTS = false;
+ public static boolean VERBOSE_REF_COUNTS = true;
private final IndexWriter writer;
diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
index fb03f85..db4e1f9 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
@@ -339,6 +339,7 @@ public abstract class FSDirectory extends BaseDirectory {
}
privateDeleteFile(name, false);
maybeDeletePendingFiles();
+ System.out.println("Deleted file: " + name);
}
/** Try to delete any pending files that we had previously tried to delete but failed
diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
index fab3300..a05a36c 100644
--- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
@@ -280,6 +280,8 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
break;
}
}
+
+ System.out.println(Arrays.toString(files) + " size=" + size + " on path: " + directory.toString());
return size;
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index abc4af6..1af7771 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -433,6 +433,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
dir = directoryFactory.get(getIndexDir(), DirContext.DEFAULT, solrConfig.indexConfig.lockType);
try {
size = DirectoryFactory.sizeOfDirectory(dir);
+ System.out.println("Found size=" + size + " for indexDir=" + getIndexDir());
} finally {
directoryFactory.release(dir);
}
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index 004f4f7..cc0f8fb 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -235,7 +235,8 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// this should always be used - see filterParams
DistributedUpdateProcessorFactory.addParamToDistributedRequestWhitelist
- (this.req, UpdateParams.UPDATE_CHAIN, TEST_DISTRIB_SKIP_SERVERS, CommonParams.VERSION_FIELD);
+ (this.req, UpdateParams.UPDATE_CHAIN, TEST_DISTRIB_SKIP_SERVERS, CommonParams.VERSION_FIELD,
+ UpdateParams.EXPUNGE_DELETES, UpdateParams.OPTIMIZE, UpdateParams.MAX_OPTIMIZE_SEGMENTS);
CoreContainer cc = req.getCore().getCoreContainer();
@@ -1387,10 +1388,11 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionDelete(cmd);
+ System.out.println("Doing versionDelete on " + req.getCore().getName() + " dropCmd=" + dropCmd);
}
if (dropCmd) {
- // TODO: do we need to add anything to the response?
+ // TODO: do we need to add anytprocehing to the response?
return;
}
diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml
index f671842..d98ba99 100644
--- a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml
@@ -46,6 +46,7 @@
</requestHandler>
<indexConfig>
<mergeScheduler class="${solr.mscheduler:org.apache.lucene.index.ConcurrentMergeScheduler}"/>
+ <infoStream>true</infoStream>
</indexConfig>
</config>
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index a3f75dc..84833dc 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -669,7 +669,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
- System.exit(-1);
+// System.exit(-1);
assertEquals(1, listenerEvents.size());
events = listenerEvents.get("capturing4");
[lucene] 02/09: Merge branch 'master' into jira/solr-12730
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit b47df35d526073142a32cf9f7fc57c0ea8e4b1b0
Merge: 3d91b8e 5fc4d51
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Mon Oct 29 12:41:15 2018 +0100
Merge branch 'master' into jira/solr-12730
lucene/CHANGES.txt | 10 +
lucene/analysis/common/README.txt | 2 +
lucene/analysis/common/build.xml | 20 +-
.../org/tartarus/snowball/ext/ArabicStemmer.java | 1912 ++++++++++++++++++++
.../analysis/snowball/TestSnowballVocab.java | 1 +
.../analysis/snowball/TestSnowballVocabData.zip | Bin 3128133 -> 3568843 bytes
.../apache/lucene/analysis/ko/KoreanTokenizer.java | 4 +
.../ko/dict/TokenInfoDictionary$buffer.dat | Bin 7245625 -> 7245613 bytes
.../analysis/ko/dict/TokenInfoDictionary$fst.dat | Bin 5640925 -> 5640903 bytes
.../ko/dict/TokenInfoDictionary$targetMap.dat | Bin 811783 -> 811783 bytes
.../lucene/analysis/ko/TestKoreanTokenizer.java | 8 +
.../analysis/ko/dict/TestTokenInfoDictionary.java | 4 +
.../analysis/ko/util/BinaryDictionaryWriter.java | 29 +-
.../ko/util/TokenInfoDictionaryBuilder.java | 17 +-
lucene/common-build.xml | 2 +-
.../apache/lucene/geo/SimpleWKTShapeParser.java | 406 +++++
.../lucene/geo/TestSimpleWKTShapeParsing.java | 206 +++
solr/CHANGES.txt | 29 +-
.../src/java/org/apache/solr/api/V2HttpCall.java | 5 +-
.../org/apache/solr/cloud/ElectionContext.java | 3 +
.../solr/highlight/UnifiedSolrHighlighter.java | 33 +-
.../java/org/apache/solr/servlet/HttpSolrCall.java | 4 +-
.../test-files/solr/collection1/conf/schema15.xml | 10 +
.../solrconfig-distrib-update-processor-chains.xml | 2 +-
.../conf/solrconfig-doctransformers.xml | 2 +-
.../solr/collection1/conf/solrconfig-hash.xml | 2 +-
.../conf/solrconfig-master-throttled.xml | 2 +-
.../solr/collection1/conf/solrconfig-paramset.xml | 2 +-
.../solr/collection1/conf/solrconfig-sql.xml | 2 +-
.../solr/collection1/conf/solrconfig-tagger.xml | 2 +-
.../handler/TestSolrConfigHandlerConcurrent.java | 4 +-
.../solr/highlight/TestUnifiedSolrHighlighter.java | 14 +-
.../org/apache/solr/search/QueryEqualityTest.java | 12 +
.../org/apache/solr/search/facet/DebugAgg.java | 2 +-
.../facet}/TestCloudJSONFacetJoinDomain.java | 6 +-
.../facet}/TestCloudJSONFacetSKG.java | 7 +-
solr/solr-ref-guide/src/collections-api.adoc | 36 +-
.../solr-ref-guide/src/computational-geometry.adoc | 188 ++
solr/solr-ref-guide/src/curve-fitting.adoc | 101 ++
solr/solr-ref-guide/src/dsp.adoc | 124 +-
solr/solr-ref-guide/src/highlighting.adoc | 31 +-
.../src/images/math-expressions/sinewave.png | Bin 0 -> 273253 bytes
.../src/images/math-expressions/sinewave256.png | Bin 0 -> 369866 bytes
solr/solr-ref-guide/src/math-expressions.adoc | 15 +-
solr/solr-ref-guide/src/matrix-math.adoc | 53 +
solr/solr-ref-guide/src/statistics.adoc | 87 +
.../src/stream-decorator-reference.adoc | 50 +-
.../src/stream-evaluator-reference.adoc | 5 +-
solr/solr-ref-guide/src/vector-math.adoc | 38 +
.../solrj/impl/StreamingBinaryResponseParser.java | 14 +-
.../apache/solr/common/params/HighlightParams.java | 1 +
.../solrj/solr/collection1/conf/solrconfig-sql.xml | 2 +-
.../solrj/solr/configsets/ml/conf/solrconfig.xml | 2 +-
.../solr/configsets/streaming/conf/solrconfig.xml | 2 +-
.../embedded/SolrExampleStreamingBinaryTest.java | 56 +
.../solr/client/solrj/request/TestV2Request.java | 46 +-
.../apache/solr/cloud/MultiSolrCloudTestCase.java | 1 +
57 files changed, 3482 insertions(+), 134 deletions(-)
[lucene] 04/09: Added more debug logging on how and why commits are
reserved and deleted
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 514fef4f29c5f6c7a274f674577d3f4fda41342a
Author: Shalin Shekhar Mangar <sh...@apache.org>
AuthorDate: Tue Oct 30 15:43:12 2018 +0530
Added more debug logging on how and why commits are reserved and deleted
---
.../org/apache/solr/cloud/overseer/ReplicaMutator.java | 4 ++--
.../org/apache/solr/core/IndexDeletionPolicyWrapper.java | 15 +++++++++++++--
.../solr/cloud/autoscaling/IndexSizeTriggerTest.java | 2 +-
3 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
index 68a42b9..82a0610 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
@@ -394,7 +394,7 @@ public class ReplicaMutator {
}
}
if (allActive) {
- log.info("Shard: {} - all replicas are active. Finding status of fellow sub-shards", sliceName);
+ log.info("Shard: {} - all {} replicas are active. Finding status of fellow sub-shards", sliceName, slice.getReplicasMap().size());
// find out about other sub shards
Map<String, Slice> allSlicesCopy = new HashMap<>(collection.getSlicesMap());
List<Slice> subShardSlices = new ArrayList<>();
@@ -413,7 +413,7 @@ public class ReplicaMutator {
break outer;
}
}
- log.info("Shard: {} - Fellow sub-shard: {} has all replicas active", sliceName, otherSlice.getName());
+ log.info("Shard: {} - Fellow sub-shard: {} has all {} replicas active", sliceName, otherSlice.getName(), otherSlice.getReplicasMap().size());
subShardSlices.add(otherSlice);
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java b/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java
index 40e65b7..ee7ed36 100644
--- a/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java
+++ b/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java
@@ -130,6 +130,7 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy {
if (reserveCount == null) reserveCount = new AtomicInteger();
reserveCount.incrementAndGet();
savedCommits.put(indexCommitGen, reserveCount);
+ log.debug("Saving commit point for generation {}", indexCommitGen);
}
/** Release a previously saved commit point */
@@ -138,6 +139,7 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy {
if (reserveCount == null) return;// this should not happen
if (reserveCount.decrementAndGet() <= 0) {
savedCommits.remove(indexCommitGen);
+ log.debug("Releasing commit point for generation {}", indexCommitGen);
}
}
@@ -189,11 +191,20 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy {
@Override
public void delete() {
Long gen = delegate.getGeneration();
+ log.debug("Checking whether we can delete commit point with generation: {}", gen);
Long reserve = reserves.get(gen);
- if (reserve != null && System.nanoTime() < reserve) return;
- if (savedCommits.containsKey(gen)) return;
+ long currentTime = System.nanoTime();
+ if (reserve != null && currentTime < reserve) {
+ log.debug("Commit point with generation: {} not deleted because its reserve {} is less than current time {}", gen, reserve, currentTime);
+ return;
+ }
+ if (savedCommits.containsKey(gen)) {
+ log.debug("Commit point with generation: {} not deleted because it is saved");
+ return;
+ }
if (snapshotMgr.isSnapshotted(gen)) return;
delegate.delete();
+ log.debug("Commit point with generation: {} deleted", gen);
}
@Override
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index 934ec20..a3f75dc 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -67,7 +67,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_P
/**
*
*/
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.handler.admin.MetricsHandler=DEBUG")
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.handler.admin.MetricsHandler=DEBUG;org.apache.solr.core.SolrDeletionPolicy=DEBUG;org.apache.solr.core.IndexDeletionPolicyWrapper=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUGorg.apache.solr.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.api.collections=DEBUG;org.apache.solr.cloud.O [...]
public class IndexSizeTriggerTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
[lucene] 03/09: SOLR-12730: add more debug logging.
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit efb73a39b03b3f4ac2fa139d4e0816a35c5d8b8d
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Mon Oct 29 20:06:31 2018 +0100
SOLR-12730: add more debug logging.
---
.../solr/cloud/autoscaling/IndexSizeTrigger.java | 19 +++++++++++++---
.../src/java/org/apache/solr/core/SolrCore.java | 26 ++++++++++++++++++++++
.../apache/solr/handler/admin/MetricsHandler.java | 8 +++++++
.../org/apache/solr/update/SolrIndexSplitter.java | 4 ++--
.../cloud/autoscaling/IndexSizeTriggerTest.java | 19 ++++++++++++++--
5 files changed, 69 insertions(+), 7 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
index 533ec53..e41297e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
@@ -252,7 +252,7 @@ public class IndexSizeTrigger extends TriggerBase {
DocCollection docCollection = clusterState.getCollection(coll);
shards.forEach((sh, replicas) -> {
- // check only the leader of a replica in active shard
+ // check only the leader replica in an active shard
Slice s = docCollection.getSlice(sh);
if (s.getState() != Slice.State.ACTIVE) {
return;
@@ -262,6 +262,10 @@ public class IndexSizeTrigger extends TriggerBase {
if (r == null) {
return;
}
+ // not on this node
+ if (!r.getNodeName().equals(node)) {
+ return;
+ }
// find ReplicaInfo
ReplicaInfo info = null;
for (ReplicaInfo ri : replicas) {
@@ -282,6 +286,7 @@ public class IndexSizeTrigger extends TriggerBase {
String registry = SolrCoreMetricManager.createRegistryName(true, coll, sh, replicaName, null);
String tag = "metrics:" + registry + ":INDEX.sizeInBytes";
metricTags.put(tag, info);
+ metricTags.put("metrics:" + registry + ":INDEX.sizeDetails", info);
tag = "metrics:" + registry + ":SEARCHER.searcher.numDocs";
metricTags.put(tag, info);
});
@@ -297,14 +302,14 @@ public class IndexSizeTrigger extends TriggerBase {
} else {
// verify that it's a Number
if (!(size instanceof Number)) {
- log.warn("invalid size value - not a number: '" + size + "' is " + size.getClass().getName());
+ log.warn("invalid size value for tag " + tag + " - not a number: '" + size + "' is " + size.getClass().getName());
return;
}
ReplicaInfo currentInfo = currentSizes.computeIfAbsent(info.getCore(), k -> (ReplicaInfo)info.clone());
if (tag.contains("INDEX")) {
currentInfo.getVariables().put(BYTES_SIZE_PROP, ((Number) size).longValue());
- } else {
+ } else if (tag.contains("SEARCHER")) {
currentInfo.getVariables().put(DOCS_SIZE_PROP, ((Number) size).longValue());
}
}
@@ -458,6 +463,14 @@ public class IndexSizeTrigger extends TriggerBase {
if (ops.isEmpty()) {
return;
}
+ try {
+ ClusterState cs = cloudManager.getClusterStateProvider().getClusterState();
+ cs.forEachCollection(coll -> {
+ log.debug("##== Collection: {}", coll);
+ });
+ } catch (IOException e) {
+ throw new RuntimeException("oops: ", e);
+ }
if (processor.process(new IndexSizeEvent(getName(), eventTime.get(), ops, aboveSize, belowSize))) {
// update last event times
aboveSize.forEach((coll, replicas) -> {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 6e13039..abc4af6 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -443,6 +443,31 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return size;
}
+ String getIndexSizeDetails() {
+ Directory dir;
+ StringBuilder sb = new StringBuilder();
+ try {
+ if (directoryFactory.exists(getIndexDir())) {
+ dir = directoryFactory.get(getIndexDir(), DirContext.DEFAULT, solrConfig.indexConfig.lockType);
+ try {
+ String[] files = dir.listAll();
+ Arrays.sort(files);
+ for (String file : files) {
+ sb.append('\n');
+ sb.append(file);
+ sb.append('\t');
+ sb.append(String.valueOf(dir.fileLength(file)));
+ }
+ } finally {
+ directoryFactory.release(dir);
+ }
+ }
+ } catch (IOException e) {
+ SolrException.log(log, "IO error while trying to get the details of the Directory", e);
+ }
+ return sb.toString();
+ }
+
@Override
public String getName() {
return name;
@@ -1161,6 +1186,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
manager.registerGauge(this, registry, () -> resourceLoader.getInstancePath().toString(), getMetricTag(), true, "instanceDir", Category.CORE.toString());
manager.registerGauge(this, registry, () -> isClosed() ? "(closed)" : getIndexDir(), getMetricTag(), true, "indexDir", Category.CORE.toString());
manager.registerGauge(this, registry, () -> isClosed() ? 0 : getIndexSize(), getMetricTag(), true, "sizeInBytes", Category.INDEX.toString());
+ manager.registerGauge(this, registry, () -> isClosed() ? "" : getIndexSizeDetails(), getMetricTag(), true, "sizeDetails", Category.INDEX.toString());
manager.registerGauge(this, registry, () -> isClosed() ? "(closed)" : NumberUtils.readableSize(getIndexSize()), getMetricTag(), true, "size", Category.INDEX.toString());
if (coreContainer != null) {
manager.registerGauge(this, registry, () -> coreContainer.getNamesForCore(this), getMetricTag(), true, "aliases", Category.CORE.toString());
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index 752e021..9b9f948 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -17,6 +17,7 @@
package org.apache.solr.handler.admin;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
@@ -41,6 +42,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.metrics.SolrMetricManager;
@@ -49,11 +51,15 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
import org.apache.solr.util.stats.MetricUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Request handler to return metrics
*/
public class MetricsHandler extends RequestHandlerBase implements PermissionNameProvider {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
final SolrMetricManager metricManager;
public static final String COMPACT_PARAM = "compact";
@@ -99,6 +105,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
handleRequest(req.getParams(), (k, v) -> rsp.add(k, v));
+ log.debug("##== Req: {}", req);
+ log.debug("##== Rsp: {}", Utils.toJSONString(rsp.getValues()));
}
public void handleRequest(SolrParams params, BiConsumer<String, Object> consumer) throws Exception {
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
index 334a29d..6ed0b1f 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
@@ -174,7 +174,7 @@ public class SolrIndexSplitter {
log.error("Original error closing IndexWriter:", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reopening IndexWriter after failed close", e1);
}
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error closing current IndexWriter, aborting offline split...", e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error closing current IndexWriter, aborting 'link' split...", e);
}
}
boolean success = false;
@@ -195,7 +195,7 @@ public class SolrIndexSplitter {
t = timings.sub("parentApplyBufferedUpdates");
ulog.applyBufferedUpdates();
t.stop();
- log.info("Splitting in 'offline' mode " + (success? "finished" : "FAILED") +
+ log.info("Splitting in 'link' mode " + (success? "finished" : "FAILED") +
": re-opened parent IndexWriter.");
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index c933c0a..934ec20 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -67,7 +67,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_P
/**
*
*/
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.handler.admin.MetricsHandler=DEBUG")
public class IndexSizeTriggerTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -90,6 +90,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
+ System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
@@ -503,7 +504,8 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
int aboveBytes = maxSize * 9 / 10;
- long waitForSeconds = 3 + random().nextInt(5);
+ // need to wait for recovery after splitting
+ long waitForSeconds = 10 + random().nextInt(5);
// the trigger is initially disabled so that we have time to add listeners
// and have them capture all events once the trigger is enabled
@@ -562,6 +564,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
"'name' : 'index_size_trigger4'" +
"}" +
"}";
+ log.info("-- resuming trigger");
req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -570,6 +573,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
boolean await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS);
assertTrue("did not finish processing in time", await);
+ log.info("-- suspending trigger");
// suspend the trigger to avoid generating more events
String suspendTriggerCommand = "{" +
"'suspend-trigger' : {" +
@@ -624,6 +628,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
finished = new CountDownLatch(1);
+ log.info("-- deleting documents");
for (int j = 0; j < 10; j++) {
UpdateRequest ureq = new UpdateRequest();
ureq.setParam("collection", collectionName);
@@ -632,6 +637,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
}
solrClient.request(ureq);
}
+ cloudManager.getTimeSource().sleep(5000);
// make sure the actual index size is reduced by deletions, otherwise we may still violate aboveBytes
UpdateRequest ur = new UpdateRequest();
ur.setParam(UpdateParams.COMMIT, "true");
@@ -640,14 +646,17 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
ur.setParam(UpdateParams.MAX_OPTIMIZE_SEGMENTS, "1");
ur.setParam(UpdateParams.WAIT_SEARCHER, "true");
ur.setParam(UpdateParams.OPEN_SEARCHER, "true");
+ log.info("-- requesting optimize / expungeDeletes / commit");
solrClient.request(ur, collectionName);
// wait for the segments to merge to reduce the index size
cloudManager.getTimeSource().sleep(50000);
+ log.info("-- requesting commit");
solrClient.commit(collectionName, true, true);
// resume the trigger
+ log.info("-- resuming trigger");
req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -656,6 +665,12 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS);
assertTrue("did not finish processing in time", await);
+ log.info("-- suspending trigger");
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ System.exit(-1);
+
assertEquals(1, listenerEvents.size());
events = listenerEvents.get("capturing4");
assertNotNull("'capturing4' events not found", events);
[lucene] 06/09: Adding more logging around when files are reference
counted Added custom CMS impl which sets thread names with MDC context
information
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 2bff40112b91eeec3d3a7b2abfe7d92aa84f5586
Author: Shalin Shekhar Mangar <sh...@apache.org>
AuthorDate: Thu Nov 1 11:09:16 2018 +0530
Adding more logging around when files are reference counted
Added custom CMS impl which sets thread names with MDC context information
---
.../org/apache/lucene/index/IndexFileDeleter.java | 23 +++++++++++++++--
.../org/apache/solr/update/SolrIndexConfig.java | 29 +++++++++++++++++++++-
.../processor/DistributedUpdateProcessor.java | 1 -
3 files changed, 49 insertions(+), 4 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
index f76cb98..cd5cf7f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
@@ -20,6 +20,8 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Collection;
@@ -564,7 +566,16 @@ final class IndexFileDeleter implements Closeable {
RefCount rc = getRefCount(fileName);
if (infoStream.isEnabled("IFD")) {
if (VERBOSE_REF_COUNTS) {
- infoStream.message("IFD", " IncRef \"" + fileName + "\": pre-incr count is " + rc.count);
+ // todo nocommit remove the extra logging
+ String extra = null;
+ if ("_0.fdt".equals(fileName)) {
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter writer = new PrintWriter(stringWriter);
+ new Exception().printStackTrace(writer);
+ extra = stringWriter.toString();
+ }
+
+ infoStream.message("IFD", " IncRef \"" + fileName + "\": pre-incr count is " + rc.count + (extra != null ? " and stack is " + extra : ""));
}
}
rc.IncRef();
@@ -602,7 +613,15 @@ final class IndexFileDeleter implements Closeable {
RefCount rc = getRefCount(fileName);
if (infoStream.isEnabled("IFD")) {
if (VERBOSE_REF_COUNTS) {
- infoStream.message("IFD", " DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
+ String extra = null;
+ // todo nocommit remove the extra logging
+ if ("_0.fdt".equals(fileName)) {
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter writer = new PrintWriter(stringWriter);
+ new Exception().printStackTrace(writer);
+ extra = stringWriter.toString();
+ }
+ infoStream.message("IFD", " DecRef \"" + fileName + "\": pre-decr count is " + rc.count + (extra != null ? " and stack is " + extra : ""));
}
}
if (rc.DecRef() == 0) {
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
index 48b2417..38f16c2 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
@@ -18,6 +18,7 @@ package org.apache.solr.update;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -25,6 +26,7 @@ import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MergePolicy;
@@ -48,6 +50,7 @@ import org.apache.solr.schema.IndexSchema;
import org.apache.solr.util.SolrPluginUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
import static org.apache.solr.core.Config.assertWarnOrFail;
@@ -283,7 +286,7 @@ public class SolrIndexConfig implements MapSerializable {
private MergeScheduler buildMergeScheduler(IndexSchema schema) {
String msClassName = mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME : mergeSchedulerInfo.className;
- MergeScheduler scheduler = schema.getResourceLoader().newInstance(msClassName, MergeScheduler.class);
+ MergeScheduler scheduler = new TestCMS(); // todo nocommit
if (mergeSchedulerInfo != null) {
// LUCENE-5080: these two setters are removed, so we have to invoke setMaxMergesAndThreads
@@ -312,4 +315,28 @@ public class SolrIndexConfig implements MapSerializable {
return scheduler;
}
+ static class TestCMS extends ConcurrentMergeScheduler {
+ @Override
+ protected synchronized MergeThread getMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) throws IOException {
+ MergeThread mergeThread = super.getMergeThread(writer, merge);
+ final Map<String, String> submitterContext = MDC.getCopyOfContextMap();
+ StringBuilder contextString = new StringBuilder();
+ if (submitterContext != null) {
+ Collection<String> values = submitterContext.values();
+
+ for (String value : values) {
+ contextString.append(value + " ");
+ }
+ if (contextString.length() > 1) {
+ contextString.setLength(contextString.length() - 1);
+ }
+ }
+
+ String ctxStr = contextString.toString().replace("/", "//");
+ final String submitterContextStr = ctxStr.length() <= 512 ? ctxStr : ctxStr.substring(0, 512);
+ mergeThread.setName(mergeThread.getName() + "-processing-" + submitterContextStr);
+ return mergeThread;
+ }
+ }
+
}
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index cc0f8fb..8bdc8bc 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -1388,7 +1388,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionDelete(cmd);
- System.out.println("Doing versionDelete on " + req.getCore().getName() + " dropCmd=" + dropCmd);
}
if (dropCmd) {
[lucene] 01/09: Initial patch.
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 3d91b8e27736b6a7456d710a467a7c723205d47a
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Wed Oct 24 13:28:21 2018 +0200
Initial patch.
---
.../solr/cloud/api/collections/SplitShardCmd.java | 9 ++++-
.../solr/cloud/autoscaling/IndexSizeTrigger.java | 26 ++++++++++++++-
.../solr/cloud/autoscaling/TriggerEvent.java | 2 ++
.../solr/handler/admin/CollectionsHandler.java | 9 ++++-
.../cloud/autoscaling/IndexSizeTriggerTest.java | 30 ++++++++++-------
.../cloud/autoscaling/SplitShardSuggester.java | 14 +++++++-
.../client/solrj/cloud/autoscaling/Suggester.java | 15 ++++++++-
.../solrj/request/CollectionAdminRequest.java | 10 ++++++
.../org/apache/solr/common/cloud/DocRouter.java | 38 +++++++++++++++++++++-
.../solr/common/params/CommonAdminParams.java | 4 ++-
10 files changed, 138 insertions(+), 19 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index aa4909d..b9370ac 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -686,6 +686,13 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
boolean firstReplicaNrt) {
String splitKey = message.getStr("split.key");
String rangesStr = message.getStr(CoreAdminParams.RANGES);
+ String fuzzStr = message.getStr(CommonAdminParams.SPLIT_FUZZ, "0");
+ float fuzz = 0.0f;
+ try {
+ fuzz = Float.parseFloat(fuzzStr);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "?Invalid numeric value of 'fuzz': " + fuzzStr);
+ }
DocRouter.Range range = parentSlice.getRange();
if (range == null) {
@@ -752,7 +759,7 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"A shard can only be split into "+MIN_NUM_SUB_SHARDS+" to " + MAX_NUM_SUB_SHARDS
+ " subshards in one split request. Provided "+NUM_SUB_SHARDS+"=" + numSubShards);
- subRanges.addAll(router.partitionRange(numSubShards, range));
+ subRanges.addAll(router.partitionRange(numSubShards, range, fuzz));
}
for (int i = 0; i < subRanges.size(); i++) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
index 25083ae..533ec53 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
@@ -41,11 +41,13 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.metrics.SolrCoreMetricManager;
+import org.apache.solr.update.SolrIndexSplitter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,6 +65,8 @@ public class IndexSizeTrigger extends TriggerBase {
public static final String BELOW_OP_PROP = "belowOp";
public static final String COLLECTIONS_PROP = "collections";
public static final String MAX_OPS_PROP = "maxOps";
+ public static final String SPLIT_FUZZ_PROP = CommonAdminParams.SPLIT_FUZZ;
+ public static final String SPLIT_METHOD_PROP = CommonAdminParams.SPLIT_METHOD;
public static final String BYTES_SIZE_PROP = "__bytes__";
public static final String DOCS_SIZE_PROP = "__docs__";
@@ -76,6 +80,8 @@ public class IndexSizeTrigger extends TriggerBase {
private long aboveBytes, aboveDocs, belowBytes, belowDocs;
private int maxOps;
+ private SolrIndexSplitter.SplitMethod splitMethod;
+ private float splitFuzz;
private CollectionParams.CollectionAction aboveOp, belowOp;
private final Set<String> collections = new HashSet<>();
private final Map<String, Long> lastAboveEventMap = new ConcurrentHashMap<>();
@@ -85,7 +91,7 @@ public class IndexSizeTrigger extends TriggerBase {
super(TriggerEventType.INDEXSIZE, name);
TriggerUtils.validProperties(validProperties,
ABOVE_BYTES_PROP, ABOVE_DOCS_PROP, BELOW_BYTES_PROP, BELOW_DOCS_PROP,
- COLLECTIONS_PROP, MAX_OPS_PROP);
+ COLLECTIONS_PROP, MAX_OPS_PROP, SPLIT_FUZZ_PROP);
}
@Override
@@ -165,6 +171,18 @@ public class IndexSizeTrigger extends TriggerBase {
} catch (Exception e) {
throw new TriggerValidationException(getName(), MAX_OPS_PROP, "invalid value: '" + maxOpsStr + "': " + e.getMessage());
}
+ String methodStr = (String)properties.getOrDefault(CommonAdminParams.SPLIT_METHOD, SolrIndexSplitter.SplitMethod.LINK.toLower());
+ splitMethod = SolrIndexSplitter.SplitMethod.get(methodStr);
+ if (splitMethod == null) {
+ throw new TriggerValidationException(getName(), SPLIT_METHOD_PROP, "Unknown value '" + CommonAdminParams.SPLIT_METHOD +
+ ": " + methodStr);
+ }
+ String fuzzStr = String.valueOf(properties.getOrDefault(SPLIT_FUZZ_PROP, 0.0f));
+ try {
+ splitFuzz = Float.parseFloat(fuzzStr);
+ } catch (Exception e) {
+ throw new TriggerValidationException(getName(), SPLIT_FUZZ_PROP, "invalid value: '" + fuzzStr + "': " + e.getMessage());
+ }
}
@Override
@@ -383,6 +401,12 @@ public class IndexSizeTrigger extends TriggerBase {
}
TriggerEvent.Op op = new TriggerEvent.Op(aboveOp);
op.addHint(Suggester.Hint.COLL_SHARD, new Pair<>(coll, r.getShard()));
+ Map<String, Object> params = new HashMap<>();
+ params.put(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower());
+ if (splitFuzz > 0) {
+ params.put(CommonAdminParams.SPLIT_FUZZ, splitFuzz);
+ }
+ op.addHint(Suggester.Hint.PARAMS, params);
ops.add(op);
Long time = lastAboveEventMap.get(r.getCore());
if (time != null && eventTime.get() > time) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
index 8e3a348..c61556c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
@@ -64,6 +64,8 @@ public class TriggerEvent implements MapWriter {
if (hint.multiValued) {
Collection<?> values = value instanceof Collection ? (Collection) value : Collections.singletonList(value);
((Set) hints.computeIfAbsent(hint, h -> new LinkedHashSet<>())).addAll(values);
+ } else if (value instanceof Map) {
+ hints.put(hint, value);
} else {
hints.put(hint, value == null ? null : String.valueOf(value));
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index dfb3c6b..462da77 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -144,6 +144,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE;
import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS;
+import static org.apache.solr.common.params.CommonAdminParams.SPLIT_FUZZ;
import static org.apache.solr.common.params.CommonAdminParams.SPLIT_METHOD;
import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE;
import static org.apache.solr.common.params.CommonParams.NAME;
@@ -640,6 +641,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
String rangesStr = req.getParams().get(CoreAdminParams.RANGES);
String splitKey = req.getParams().get("split.key");
String numSubShards = req.getParams().get(NUM_SUB_SHARDS);
+ String fuzz = req.getParams().get(SPLIT_FUZZ);
if (splitKey == null && shard == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "At least one of shard, or split.key should be specified.");
@@ -656,6 +658,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
throw new SolrException(ErrorCode.BAD_REQUEST,
"numSubShards can not be specified with split.key or ranges parameters");
}
+ if (fuzz != null && (splitKey != null || rangesStr != null)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "fuzz can not be specified with split.key or ranges parameters");
+ }
Map<String, Object> map = copy(req.getParams(), null,
COLLECTION_PROP,
@@ -665,7 +671,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
WAIT_FOR_FINAL_STATE,
TIMING,
SPLIT_METHOD,
- NUM_SUB_SHARDS);
+ NUM_SUB_SHARDS,
+ SPLIT_FUZZ);
return copyPropertiesWithPrefix(req.getParams(), map, COLL_PROP_PREFIX);
}),
DELETESHARD_OP(DELETESHARD, (req, rsp, h) -> {
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index 996532a..c933c0a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -93,7 +93,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
- if (random().nextBoolean()) {
+ if (random().nextBoolean() || true) {
cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
solrClient = cluster.getSolrClient();
loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader();
@@ -501,7 +501,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
}
assertTrue("maxSize should be non-zero", maxSize > 0);
- int aboveBytes = maxSize * 2 / 3;
+ int aboveBytes = maxSize * 9 / 10;
long waitForSeconds = 3 + random().nextInt(5);
@@ -570,6 +570,16 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
boolean await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS);
assertTrue("did not finish processing in time", await);
+ // suspend the trigger to avoid generating more events
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {" +
+ "'name' : 'index_size_trigger4'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
assertEquals(1, listenerEvents.size());
List<CapturedEvent> events = listenerEvents.get("capturing4");
assertNotNull("'capturing4' events not found", events);
@@ -613,15 +623,6 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
listenerEvents.clear();
finished = new CountDownLatch(1);
- // suspend the trigger first so that we can safely delete all docs
- String suspendTriggerCommand = "{" +
- "'suspend-trigger' : {" +
- "'name' : 'index_size_trigger4'" +
- "}" +
- "}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
for (int j = 0; j < 10; j++) {
UpdateRequest ureq = new UpdateRequest();
@@ -641,7 +642,12 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
ur.setParam(UpdateParams.OPEN_SEARCHER, "true");
solrClient.request(ur, collectionName);
- // resume trigger
+ // wait for the segments to merge to reduce the index size
+ cloudManager.getTimeSource().sleep(50000);
+
+ solrClient.commit(collectionName, true, true);
+
+ // resume the trigger
req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
index 2c1d7df..8aafef8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
@@ -17,11 +17,13 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.util.Collections;
+import java.util.Map;
import java.util.Set;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.util.Pair;
/**
@@ -44,6 +46,16 @@ class SplitShardSuggester extends Suggester {
throw new RuntimeException("split-shard requires exactly one pair of 'collection' and 'shard'");
}
Pair<String, String> collShard = shards.iterator().next();
- return CollectionAdminRequest.splitShard(collShard.first()).setShardName(collShard.second());
+ Map<String, Object> params = (Map<String, Object>)hints.getOrDefault(Hint.PARAMS, Collections.emptyMap());
+ Float splitFuzz = (Float)params.get(CommonAdminParams.SPLIT_FUZZ);
+ CollectionAdminRequest.SplitShard req = CollectionAdminRequest.splitShard(collShard.first()).setShardName(collShard.second());
+ if (splitFuzz != null) {
+ req.setSplitFuzz(splitFuzz);
+ }
+ String splitMethod = (String)params.get(CommonAdminParams.SPLIT_METHOD);
+ if (splitMethod != null) {
+ req.setSplitMethod(splitMethod);
+ }
+ return req;
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index bba5906..39ad8bf 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -95,7 +95,15 @@ public abstract class Suggester implements MapWriter {
Collection<?> values = value instanceof Collection ? (Collection) value : Collections.singletonList(value);
((Set) hints.computeIfAbsent(hint, h -> new HashSet<>())).addAll(values);
} else {
- hints.put(hint, value == null ? null : String.valueOf(value));
+ if (value == null) {
+ hints.put(hint, null);
+ } else {
+ if ((value instanceof Map) || (value instanceof Number)) {
+ hints.put(hint, value);
+ } else {
+ hints.put(hint, String.valueOf(value));
+ }
+ }
}
return this;
}
@@ -372,6 +380,11 @@ public abstract class Suggester implements MapWriter {
NUMBER(true, o -> {
if (!(o instanceof Number)) throw new RuntimeException("NUMBER hint must be a number");
}),
+ PARAMS(false, o -> {
+ if (!(o instanceof Map)) {
+ throw new RuntimeException("PARAMS hint must be a Map<String, Object>");
+ }
+ }),
REPLICA(true);
public final boolean multiValued;
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 4f26984..3769c2e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -1154,6 +1154,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
protected String shard;
protected String splitMethod;
protected Integer numSubShards;
+ protected Float splitFuzz;
private Properties properties;
@@ -1183,6 +1184,15 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return splitMethod;
}
+ public SplitShard setSplitFuzz(float splitFuzz) {
+ this.splitFuzz = splitFuzz;
+ return this;
+ }
+
+ public Float getSplitFuzz() {
+ return splitFuzz;
+ }
+
public SplitShard setSplitKey(String splitKey) {
this.splitKey = splitKey;
return this;
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java
index 846c25e..c1471b5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java
@@ -30,6 +30,7 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
@@ -153,24 +154,59 @@ public abstract class DocRouter {
}
/**
- * Returns the range for each partition
+ * Split the range into partitions.
+ * @param partitions number of partitions
+ * @param range range to split
*/
public List<Range> partitionRange(int partitions, Range range) {
+ return partitionRange(partitions, range, 0.0f);
+ }
+
+ /**
+ * Split the range into partitions with inexact sizes.
+ * @param partitions number of partitions
+ * @param range range to split
+ * @param fuzz value between 0 (inclusive) and 0.5 indicating inexact split, i.e. percentage
+ * of variation in resulting ranges - odd ranges will be larger and even ranges will be smaller
+ * by up to that percentage.
+ */
+ public List<Range> partitionRange(int partitions, Range range, float fuzz) {
int min = range.min;
int max = range.max;
assert max >= min;
+ if (fuzz > 0.5f) {
+ throw new IllegalArgumentException("'fuzz' parameter must be <= 0.5f but was " + fuzz);
+ }
if (partitions == 0) return Collections.EMPTY_LIST;
long rangeSize = (long)max - (long)min;
long rangeStep = Math.max(1, rangeSize / partitions);
+ long fuzzStep = Math.round(rangeStep * fuzz / 2.0f);
+ // use a predictable pseudo-random
+ Random r = null;
+ if (fuzzStep > 2) {
+ r = new Random(0);
+ // limit randomness to half the range
+ fuzzStep = fuzzStep / 2;
+ }
List<Range> ranges = new ArrayList<>(partitions);
long start = min;
long end = start;
+ boolean odd = true;
while (end < max) {
end = start + rangeStep;
+ if (fuzzStep > 0) {
+ long currentFuzz = r != null ? fuzzStep + r.nextLong() % fuzzStep : fuzzStep;
+ if (odd) {
+ end = end + currentFuzz;
+ } else {
+ end = end - currentFuzz;
+ }
+ odd = !odd;
+ }
// make last range always end exactly on MAX_VALUE
if (ranges.size() == partitions - 1) {
end = max;
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonAdminParams.java
index c080342..13d304c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonAdminParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonAdminParams.java
@@ -27,8 +27,10 @@ public interface CommonAdminParams
String IN_PLACE_MOVE = "inPlaceMove";
/** Method to use for shard splitting. */
String SPLIT_METHOD = "splitMethod";
- /** **/
+ /** Number of sub-shards to create. **/
String NUM_SUB_SHARDS = "numSubShards";
/** Timeout for replicas to become active. */
String TIMEOUT = "timeout";
+ /** Inexact shard splitting factor. */
+ String SPLIT_FUZZ = "splitFuzz";
}
[lucene] 09/09: SOLR-12730: Cleanup.
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit e12aebac31e0747fcea92ad8581f2b9aa1a4b66b
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Mon Nov 5 19:04:34 2018 +0100
SOLR-12730: Cleanup.
---
.../org/apache/lucene/index/IndexFileDeleter.java | 25 ++--------
.../java/org/apache/lucene/store/FSDirectory.java | 1 -
.../solr/cloud/api/collections/SplitShardCmd.java | 55 ++++++++++++++++------
.../solr/cloud/autoscaling/IndexSizeTrigger.java | 11 +----
.../org/apache/solr/core/DirectoryFactory.java | 2 -
.../solr/core/IndexDeletionPolicyWrapper.java | 15 +-----
.../src/java/org/apache/solr/core/SolrCore.java | 27 -----------
.../apache/solr/handler/admin/MetricsHandler.java | 8 ----
.../org/apache/solr/update/SolrIndexConfig.java | 29 +-----------
.../processor/DistributedUpdateProcessor.java | 2 +-
.../src/java/org/apache/solr/util/TimeOut.java | 2 +-
.../configsets/cloud-minimal/conf/solrconfig.xml | 3 +-
.../cloud/autoscaling/IndexSizeTriggerTest.java | 14 +++---
.../autoscaling/sim/SimClusterStateProvider.java | 5 ++
.../autoscaling/sim/SimSolrCloudTestCase.java | 5 +-
.../solrj/cloud/autoscaling/AutoScalingConfig.java | 2 +-
16 files changed, 69 insertions(+), 137 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
index cd5cf7f..cc9d2e0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
@@ -20,8 +20,6 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Collection;
@@ -110,7 +108,7 @@ final class IndexFileDeleter implements Closeable {
/** Change to true to see details of reference counts when
* infoStream is enabled */
- public static boolean VERBOSE_REF_COUNTS = true;
+ public static boolean VERBOSE_REF_COUNTS = false;
private final IndexWriter writer;
@@ -566,16 +564,7 @@ final class IndexFileDeleter implements Closeable {
RefCount rc = getRefCount(fileName);
if (infoStream.isEnabled("IFD")) {
if (VERBOSE_REF_COUNTS) {
- // todo nocommit remove the extra logging
- String extra = null;
- if ("_0.fdt".equals(fileName)) {
- StringWriter stringWriter = new StringWriter();
- PrintWriter writer = new PrintWriter(stringWriter);
- new Exception().printStackTrace(writer);
- extra = stringWriter.toString();
- }
-
- infoStream.message("IFD", " IncRef \"" + fileName + "\": pre-incr count is " + rc.count + (extra != null ? " and stack is " + extra : ""));
+ infoStream.message("IFD", " IncRef \"" + fileName + "\": pre-incr count is " + rc.count);
}
}
rc.IncRef();
@@ -613,15 +602,7 @@ final class IndexFileDeleter implements Closeable {
RefCount rc = getRefCount(fileName);
if (infoStream.isEnabled("IFD")) {
if (VERBOSE_REF_COUNTS) {
- String extra = null;
- // todo nocommit remove the extra logging
- if ("_0.fdt".equals(fileName)) {
- StringWriter stringWriter = new StringWriter();
- PrintWriter writer = new PrintWriter(stringWriter);
- new Exception().printStackTrace(writer);
- extra = stringWriter.toString();
- }
- infoStream.message("IFD", " DecRef \"" + fileName + "\": pre-decr count is " + rc.count + (extra != null ? " and stack is " + extra : ""));
+ infoStream.message("IFD", " DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
}
}
if (rc.DecRef() == 0) {
diff --git a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
index db4e1f9..fb03f85 100644
--- a/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
@@ -339,7 +339,6 @@ public abstract class FSDirectory extends BaseDirectory {
}
privateDeleteFile(name, false);
maybeDeletePendingFiles();
- System.out.println("Deleted file: " + name);
}
/** Try to delete any pending files that we had previously tried to delete but failed
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index b9370ac..f9ac520 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -26,16 +26,19 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.DistributedQueue;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
+import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.overseer.OverseerAction;
@@ -774,23 +777,47 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
public static boolean lockForSplit(SolrCloudManager cloudManager, String collection, String shard) throws Exception {
String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/" + shard + "-splitting";
- if (cloudManager.getDistribStateManager().hasData(path)) {
- return false;
- }
- Map<String, Object> map = new HashMap<>();
- map.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs()));
- byte[] data = Utils.toJSON(map);
- try {
- cloudManager.getDistribStateManager().makePath(path, data, CreateMode.EPHEMERAL, true);
- } catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Can't lock parent slice for splitting (another split operation running?): " +
- collection + "/" + shard, e);
+ log.info("===locking " + path, new Exception());
+ final DistribStateManager stateManager = cloudManager.getDistribStateManager();
+ synchronized (stateManager) {
+ if (stateManager.hasData(path)) {
+ VersionedData vd = stateManager.getData(path);
+ log.info("=== already locked! {}", Utils.fromJSON(vd.getData()));
+ return false;
+ }
+ Map<String, Object> map = new HashMap<>();
+ map.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs()));
+ byte[] data = Utils.toJSON(map);
+ try {
+ cloudManager.getDistribStateManager().makePath(path, data, CreateMode.EPHEMERAL, true);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Can't lock parent slice for splitting (another split operation running?): " +
+ collection + "/" + shard, e);
+ }
+ return true;
}
- return true;
}
public static void unlockForSplit(SolrCloudManager cloudManager, String collection, String shard) throws Exception {
- String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/" + shard + "-splitting";
- cloudManager.getDistribStateManager().removeRecursively(path, true, true);
+ if (shard != null) {
+ String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/" + shard + "-splitting";
+ cloudManager.getDistribStateManager().removeRecursively(path, true, true);
+ } else {
+ String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;
+ try {
+ List<String> names = cloudManager.getDistribStateManager().listData(path);
+ for (String name : cloudManager.getDistribStateManager().listData(path)) {
+ if (name.endsWith("-splitting")) {
+ try {
+ cloudManager.getDistribStateManager().removeData(path + "/" + name, -1);
+ } catch (NoSuchElementException nse) {
+ // ignore
+ }
+ }
+ }
+ } catch (NoSuchElementException nse) {
+ // ignore
+ }
+ }
}
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
index 27daec2..7483501 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
@@ -171,7 +171,7 @@ public class IndexSizeTrigger extends TriggerBase {
} catch (Exception e) {
throw new TriggerValidationException(getName(), MAX_OPS_PROP, "invalid value: '" + maxOpsStr + "': " + e.getMessage());
}
- String methodStr = (String)properties.getOrDefault(CommonAdminParams.SPLIT_METHOD, SolrIndexSplitter.SplitMethod.REWRITE.toLower());
+ String methodStr = (String)properties.getOrDefault(CommonAdminParams.SPLIT_METHOD, SolrIndexSplitter.SplitMethod.LINK.toLower());
splitMethod = SolrIndexSplitter.SplitMethod.get(methodStr);
if (splitMethod == null) {
throw new TriggerValidationException(getName(), SPLIT_METHOD_PROP, "Unknown value '" + CommonAdminParams.SPLIT_METHOD +
@@ -286,7 +286,6 @@ public class IndexSizeTrigger extends TriggerBase {
String registry = SolrCoreMetricManager.createRegistryName(true, coll, sh, replicaName, null);
String tag = "metrics:" + registry + ":INDEX.sizeInBytes";
metricTags.put(tag, info);
- metricTags.put("metrics:" + registry + ":INDEX.sizeDetails", info);
tag = "metrics:" + registry + ":SEARCHER.searcher.numDocs";
metricTags.put(tag, info);
});
@@ -463,14 +462,6 @@ public class IndexSizeTrigger extends TriggerBase {
if (ops.isEmpty()) {
return;
}
- try {
- ClusterState cs = cloudManager.getClusterStateProvider().getClusterState();
- cs.forEachCollection(coll -> {
- log.debug("##== Collection: {}", coll);
- });
- } catch (IOException e) {
- throw new RuntimeException("oops: ", e);
- }
if (processor.process(new IndexSizeEvent(getName(), eventTime.get(), ops, aboveSize, belowSize))) {
// update last event times
aboveSize.forEach((coll, replicas) -> {
diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
index a05a36c..fab3300 100644
--- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
@@ -280,8 +280,6 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
break;
}
}
-
- System.out.println(Arrays.toString(files) + " size=" + size + " on path: " + directory.toString());
return size;
}
diff --git a/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java b/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java
index ee7ed36..40e65b7 100644
--- a/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java
+++ b/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java
@@ -130,7 +130,6 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy {
if (reserveCount == null) reserveCount = new AtomicInteger();
reserveCount.incrementAndGet();
savedCommits.put(indexCommitGen, reserveCount);
- log.debug("Saving commit point for generation {}", indexCommitGen);
}
/** Release a previously saved commit point */
@@ -139,7 +138,6 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy {
if (reserveCount == null) return;// this should not happen
if (reserveCount.decrementAndGet() <= 0) {
savedCommits.remove(indexCommitGen);
- log.debug("Releasing commit point for generation {}", indexCommitGen);
}
}
@@ -191,20 +189,11 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy {
@Override
public void delete() {
Long gen = delegate.getGeneration();
- log.debug("Checking whether we can delete commit point with generation: {}", gen);
Long reserve = reserves.get(gen);
- long currentTime = System.nanoTime();
- if (reserve != null && currentTime < reserve) {
- log.debug("Commit point with generation: {} not deleted because its reserve {} is less than current time {}", gen, reserve, currentTime);
- return;
- }
- if (savedCommits.containsKey(gen)) {
- log.debug("Commit point with generation: {} not deleted because it is saved");
- return;
- }
+ if (reserve != null && System.nanoTime() < reserve) return;
+ if (savedCommits.containsKey(gen)) return;
if (snapshotMgr.isSnapshotted(gen)) return;
delegate.delete();
- log.debug("Commit point with generation: {} deleted", gen);
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 1af7771..6e13039 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -433,7 +433,6 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
dir = directoryFactory.get(getIndexDir(), DirContext.DEFAULT, solrConfig.indexConfig.lockType);
try {
size = DirectoryFactory.sizeOfDirectory(dir);
- System.out.println("Found size=" + size + " for indexDir=" + getIndexDir());
} finally {
directoryFactory.release(dir);
}
@@ -444,31 +443,6 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return size;
}
- String getIndexSizeDetails() {
- Directory dir;
- StringBuilder sb = new StringBuilder();
- try {
- if (directoryFactory.exists(getIndexDir())) {
- dir = directoryFactory.get(getIndexDir(), DirContext.DEFAULT, solrConfig.indexConfig.lockType);
- try {
- String[] files = dir.listAll();
- Arrays.sort(files);
- for (String file : files) {
- sb.append('\n');
- sb.append(file);
- sb.append('\t');
- sb.append(String.valueOf(dir.fileLength(file)));
- }
- } finally {
- directoryFactory.release(dir);
- }
- }
- } catch (IOException e) {
- SolrException.log(log, "IO error while trying to get the details of the Directory", e);
- }
- return sb.toString();
- }
-
@Override
public String getName() {
return name;
@@ -1187,7 +1161,6 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
manager.registerGauge(this, registry, () -> resourceLoader.getInstancePath().toString(), getMetricTag(), true, "instanceDir", Category.CORE.toString());
manager.registerGauge(this, registry, () -> isClosed() ? "(closed)" : getIndexDir(), getMetricTag(), true, "indexDir", Category.CORE.toString());
manager.registerGauge(this, registry, () -> isClosed() ? 0 : getIndexSize(), getMetricTag(), true, "sizeInBytes", Category.INDEX.toString());
- manager.registerGauge(this, registry, () -> isClosed() ? "" : getIndexSizeDetails(), getMetricTag(), true, "sizeDetails", Category.INDEX.toString());
manager.registerGauge(this, registry, () -> isClosed() ? "(closed)" : NumberUtils.readableSize(getIndexSize()), getMetricTag(), true, "size", Category.INDEX.toString());
if (coreContainer != null) {
manager.registerGauge(this, registry, () -> coreContainer.getNamesForCore(this), getMetricTag(), true, "aliases", Category.CORE.toString());
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index 9b9f948..752e021 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -17,7 +17,6 @@
package org.apache.solr.handler.admin;
-import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
@@ -42,7 +41,6 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.metrics.SolrMetricManager;
@@ -51,15 +49,11 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
import org.apache.solr.util.stats.MetricUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Request handler to return metrics
*/
public class MetricsHandler extends RequestHandlerBase implements PermissionNameProvider {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
final SolrMetricManager metricManager;
public static final String COMPACT_PARAM = "compact";
@@ -105,8 +99,6 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
handleRequest(req.getParams(), (k, v) -> rsp.add(k, v));
- log.debug("##== Req: {}", req);
- log.debug("##== Rsp: {}", Utils.toJSONString(rsp.getValues()));
}
public void handleRequest(SolrParams params, BiConsumer<String, Object> consumer) throws Exception {
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
index 0df2b16..48b2417 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
@@ -18,7 +18,6 @@ package org.apache.solr.update;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -26,7 +25,6 @@ import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
import org.apache.lucene.index.ConcurrentMergeScheduler;
-import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MergePolicy;
@@ -50,7 +48,6 @@ import org.apache.solr.schema.IndexSchema;
import org.apache.solr.util.SolrPluginUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
import static org.apache.solr.core.Config.assertWarnOrFail;
@@ -286,30 +283,7 @@ public class SolrIndexConfig implements MapSerializable {
private MergeScheduler buildMergeScheduler(IndexSchema schema) {
String msClassName = mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME : mergeSchedulerInfo.className;
- // todo nocommit -- remove this scheduler instance with proper MDC logging support inside merge scheduler threads
- MergeScheduler scheduler = new ConcurrentMergeScheduler() {
- @Override
- protected synchronized MergeThread getMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) throws IOException {
- MergeThread mergeThread = super.getMergeThread(writer, merge);
- final Map<String, String> submitterContext = MDC.getCopyOfContextMap();
- StringBuilder contextString = new StringBuilder();
- if (submitterContext != null) {
- Collection<String> values = submitterContext.values();
-
- for (String value : values) {
- contextString.append(value + " ");
- }
- if (contextString.length() > 1) {
- contextString.setLength(contextString.length() - 1);
- }
- }
-
- String ctxStr = contextString.toString().replace("/", "//");
- final String submitterContextStr = ctxStr.length() <= 512 ? ctxStr : ctxStr.substring(0, 512);
- mergeThread.setName(mergeThread.getName() + "-processing-" + submitterContextStr);
- return mergeThread;
- }
- };
+ MergeScheduler scheduler = schema.getResourceLoader().newInstance(msClassName, MergeScheduler.class);
if (mergeSchedulerInfo != null) {
// LUCENE-5080: these two setters are removed, so we have to invoke setMaxMergesAndThreads
@@ -337,4 +311,5 @@ public class SolrIndexConfig implements MapSerializable {
return scheduler;
}
+
}
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index 8bdc8bc..c0ca0dc 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -1391,7 +1391,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
}
if (dropCmd) {
- // TODO: do we need to add anytprocehing to the response?
+ // TODO: do we need to add anything to the response?
return;
}
diff --git a/solr/core/src/java/org/apache/solr/util/TimeOut.java b/solr/core/src/java/org/apache/solr/util/TimeOut.java
index ce996f4..a0220a7 100644
--- a/solr/core/src/java/org/apache/solr/util/TimeOut.java
+++ b/solr/core/src/java/org/apache/solr/util/TimeOut.java
@@ -61,7 +61,7 @@ public class TimeOut {
public void waitFor(String messageOnTimeOut, Supplier<Boolean> supplier)
throws InterruptedException, TimeoutException {
while (!supplier.get() && !hasTimedOut()) {
- Thread.sleep(500);
+ timeSource.sleep(500);
}
if (hasTimedOut()) throw new TimeoutException(messageOnTimeOut);
}
diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml
index d98ba99..853ba65 100644
--- a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml
@@ -46,7 +46,6 @@
</requestHandler>
<indexConfig>
<mergeScheduler class="${solr.mscheduler:org.apache.lucene.index.ConcurrentMergeScheduler}"/>
- <infoStream>true</infoStream>
- </indexConfig>
+: </indexConfig>
</config>
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index 314ddbd..f06a57d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -35,6 +35,7 @@ import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -44,7 +45,6 @@ import org.apache.solr.cloud.CloudTestUtils;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.cloud.autoscaling.sim.SimCloudManager;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.params.CommonParams;
@@ -54,7 +54,6 @@ import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.util.LogLevel;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -95,7 +94,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
- if (random().nextBoolean() || true) {
+ if (random().nextBoolean() && false) {
cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
solrClient = cluster.getSolrClient();
loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader();
@@ -126,7 +125,8 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
} else {
cluster.deleteAllCollections();
}
- cloudManager.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1);
+ AutoScalingConfig autoScalingConfig = new AutoScalingConfig(new Policy(), null, null, null, 0);
+ cloudManager.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(autoScalingConfig), -1);
cloudManager.getTimeSource().sleep(5000);
listenerEvents.clear();
listenerCreated = new CountDownLatch(1);
@@ -211,7 +211,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
}
Map<String, Object> params = (Map<String, Object>)op.getHints().get(Suggester.Hint.PARAMS);
assertNotNull("params are null: " + op, params);
- assertEquals("splitMethod: " + op, "rewrite", params.get(CommonAdminParams.SPLIT_METHOD));
+ assertEquals("splitMethod: " + op, "link", params.get(CommonAdminParams.SPLIT_METHOD));
}
assertTrue("shard1 should be split", shard1);
assertTrue("shard2 should be split", shard2);
@@ -317,7 +317,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
- boolean await = finished.await(60000 / SPEED, TimeUnit.MILLISECONDS);
+ boolean await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS);
assertTrue("did not finish processing in time", await);
CloudTestUtils.waitForState(cloudManager, collectionName, 20, TimeUnit.SECONDS, CloudTestUtils.clusterShape(6, 2, true, true));
assertEquals(1, listenerEvents.size());
@@ -506,7 +506,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
}
assertTrue("maxSize should be non-zero", maxSize > 0);
- int aboveBytes = maxSize * 9 / 10;
+ int aboveBytes = maxSize * 8 / 10;
// need to wait for recovery after splitting
long waitForSeconds = 10 + random().nextInt(5);
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index a7471eb..ef7e2ef 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -954,6 +954,7 @@ public class SimClusterStateProvider implements ClusterStateProvider {
sliceProperties.remove(collection);
leaderThrottles.remove(collection);
colShardReplicaMap.remove(collection);
+ SplitShardCmd.unlockForSplit(cloudManager, collection, null);
opDelay(collection, CollectionParams.CollectionAction.DELETE.name());
@@ -1007,6 +1008,7 @@ public class SimClusterStateProvider implements ClusterStateProvider {
values.put(ImplicitSnitch.SYSLOADAVG, 1.0);
values.put(ImplicitSnitch.HEAPUSAGE, 123450000);
});
+ cloudManager.getDistribStateManager().removeRecursively(ZkStateReader.COLLECTIONS_ZKNODE, true, false);
} finally {
lock.unlock();
}
@@ -1260,6 +1262,9 @@ public class SimClusterStateProvider implements ClusterStateProvider {
success = true;
} finally {
if (!success) {
+ Map<String, Object> sProps = sliceProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>())
+ .computeIfAbsent(sliceName.get(), s -> new ConcurrentHashMap<>());
+ sProps.remove(BUFFERED_UPDATES);
SplitShardCmd.unlockForSplit(cloudManager, collectionName, sliceName.get());
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
index 3d41ea4..3c9bd26 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
@@ -25,6 +25,8 @@ import java.util.List;
import java.util.function.Predicate;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
@@ -87,7 +89,8 @@ public class SimSolrCloudTestCase extends SolrTestCaseJ4 {
if (cluster == null)
throw new RuntimeException("SimCloudManager not configured - have you called configureCluster()?");
// clear any persisted configuration
- cluster.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1);
+ AutoScalingConfig autoScalingConfig = new AutoScalingConfig(new Policy(), null, null, null, 0);
+ cluster.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(autoScalingConfig), -1);
cluster.getDistribStateManager().setData(ZkStateReader.ROLES, Utils.toJSON(new HashMap<>()), -1);
cluster.getSimNodeStateProvider().simRemoveDeadNodes();
cluster.getSimClusterStateProvider().simRemoveDeadNodes();
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java
index ccd02eb..8a52598 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java
@@ -324,7 +324,7 @@ public class AutoScalingConfig implements MapWriter {
empty = jsonMap.isEmpty();
}
- private AutoScalingConfig(Policy policy, Map<String, TriggerConfig> triggerConfigs, Map<String,
+ public AutoScalingConfig(Policy policy, Map<String, TriggerConfig> triggerConfigs, Map<String,
TriggerListenerConfig> listenerConfigs, Map<String, Object> properties, int zkVersion) {
this.policy = policy;
this.triggers = triggerConfigs != null ? Collections.unmodifiableMap(new LinkedHashMap<>(triggerConfigs)) : null;