You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by th...@apache.org on 2015/05/20 17:04:48 UTC
svn commit: r1680593 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/update/ solr/server/
solr/server/solr/configsets/basic_configs/conf/
solr/server/solr/configsets/data_driven_schema_configs/conf/ solr/serv...
Author: thelabdude
Date: Wed May 20 15:04:47 2015
New Revision: 1680593
URL: http://svn.apache.org/r1680593
Log:
SOLR-6820: Make the number of version buckets used by the UpdateLog configurable as increasing beyond the default 256 has been shown to help with high volume indexing performance in SolrCloud
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/UpdateLog.java
lucene/dev/branches/branch_5x/solr/server/ (props changed)
lucene/dev/branches/branch_5x/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml
lucene/dev/branches/branch_5x/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
lucene/dev/branches/branch_5x/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1680593&r1=1680592&r2=1680593&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Wed May 20 15:04:47 2015
@@ -349,6 +349,11 @@ Other Changes
ZkCmdExecutor#ensureExists as they were doing the same thing. Also ZkCmdExecutor#ensureExists now respects the
CreateMode passed to it. (Varun Thacker)
+* SOLR-6820: Make the number of version buckets used by the UpdateLog configurable as
+ increasing beyond the default 256 has been shown to help with high volume indexing
+ performance in SolrCloud; helps overcome a limitation where Lucene uses the request
+ thread to perform expensive index housekeeping work. (Mark Miller, yonik, Timothy Potter)
+
================== 5.1.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/UpdateLog.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/UpdateLog.java?rev=1680593&r1=1680592&r2=1680593&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/UpdateLog.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/update/UpdateLog.java Wed May 20 15:04:47 2015
@@ -144,6 +144,7 @@ public class UpdateLog implements Plugin
protected final int numDeletesByQueryToKeep = 100;
protected int numRecordsToKeep;
protected int maxNumLogsToKeep;
+ protected int numVersionBuckets; // This should only be used to initialize VersionInfo... the actual number of buckets may be rounded up to a power of two.
// keep track of deletes only... this is not updated on an add
protected LinkedHashMap<BytesRef, LogPtr> oldDeletes = new LinkedHashMap<BytesRef, LogPtr>(numDeletesToKeep) {
@@ -224,6 +225,10 @@ public class UpdateLog implements Plugin
return maxNumLogsToKeep;
}
+ public int getNumVersionBuckets() {
+ return numVersionBuckets;
+ }
+
protected static int objToInt(Object obj, int def) {
if (obj != null) {
return Integer.parseInt(obj.toString());
@@ -238,9 +243,13 @@ public class UpdateLog implements Plugin
numRecordsToKeep = objToInt(info.initArgs.get("numRecordsToKeep"), 100);
maxNumLogsToKeep = objToInt(info.initArgs.get("maxNumLogsToKeep"), 10);
+ numVersionBuckets = objToInt(info.initArgs.get("numVersionBuckets"), 256);
+ if (numVersionBuckets <= 0)
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Number of version buckets must be greater than 0!");
- log.info("Initializing UpdateLog: dataDir={} defaultSyncLevel={} numRecordsToKeep={} maxNumLogsToKeep={}",
- dataDir, defaultSyncLevel, numRecordsToKeep, maxNumLogsToKeep);
+ log.info("Initializing UpdateLog: dataDir={} defaultSyncLevel={} numRecordsToKeep={} maxNumLogsToKeep={} numVersionBuckets={}",
+ dataDir, defaultSyncLevel, numRecordsToKeep, maxNumLogsToKeep, numVersionBuckets);
}
/* Note, when this is called, uhandler is not completely constructed.
@@ -292,7 +301,7 @@ public class UpdateLog implements Plugin
}
try {
- versionInfo = new VersionInfo(this, 256);
+ versionInfo = new VersionInfo(this, numVersionBuckets);
} catch (SolrException e) {
log.error("Unable to use updateLog: " + e.getMessage(), e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
Modified: lucene/dev/branches/branch_5x/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml?rev=1680593&r1=1680592&r2=1680593&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml (original)
+++ lucene/dev/branches/branch_5x/solr/server/solr/configsets/basic_configs/conf/solrconfig.xml Wed May 20 15:04:47 2015
@@ -147,9 +147,17 @@
uncommitted changes to the index, so use of a hard autoCommit
is recommended (see below).
"dir" - the target directory for transaction logs, defaults to the
- solr data directory. -->
+ solr data directory.
+ "numVersionBuckets" - sets the number of buckets used to keep
+ track of max version values when checking for re-ordered
+ updates; increase this value to reduce the cost of
+ synchronizing access to version buckets during high-volume
+ indexing, this requires 8 bytes (long) * numVersionBuckets
+ of heap space per Solr core.
+ -->
<updateLog>
<str name="dir">${solr.ulog.dir:}</str>
+ <int name="">${solr.ulog.numVersionBuckets:256}</int>
</updateLog>
<!-- AutoCommit
Modified: lucene/dev/branches/branch_5x/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml?rev=1680593&r1=1680592&r2=1680593&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml (original)
+++ lucene/dev/branches/branch_5x/solr/server/solr/configsets/data_driven_schema_configs/conf/solrconfig.xml Wed May 20 15:04:47 2015
@@ -324,9 +324,17 @@
uncommitted changes to the index, so use of a hard autoCommit
is recommended (see below).
"dir" - the target directory for transaction logs, defaults to the
- solr data directory. -->
+ solr data directory.
+ "numVersionBuckets" - sets the number of buckets used to keep
+ track of max version values when checking for re-ordered
+ updates; increase this value to reduce the cost of
+ synchronizing access to version buckets during high-volume
+ indexing, this requires 8 bytes (long) * numVersionBuckets
+ of heap space per Solr core.
+ -->
<updateLog>
<str name="dir">${solr.ulog.dir:}</str>
+ <int name="">${solr.ulog.numVersionBuckets:256}</int>
</updateLog>
<!-- AutoCommit
Modified: lucene/dev/branches/branch_5x/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml?rev=1680593&r1=1680592&r2=1680593&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml (original)
+++ lucene/dev/branches/branch_5x/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml Wed May 20 15:04:47 2015
@@ -327,9 +327,17 @@
uncommitted changes to the index, so use of a hard autoCommit
is recommended (see below).
"dir" - the target directory for transaction logs, defaults to the
- solr data directory. -->
+ solr data directory.
+ "numVersionBuckets" - sets the number of buckets used to keep
+ track of max version values when checking for re-ordered
+ updates; increase this value to reduce the cost of
+ synchronizing access to version buckets during high-volume
+ indexing, this requires 8 bytes (long) * numVersionBuckets
+ of heap space per Solr core.
+ -->
<updateLog>
<str name="dir">${solr.ulog.dir:}</str>
+ <int name="">${solr.ulog.numVersionBuckets:256}</int>
</updateLog>
<!-- AutoCommit