You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/07/30 05:53:25 UTC
[lucene-solr] branch reference_impl_dev updated (30040a8 -> 30c372b)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a change to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 30040a8 @459 Resource keystone.
new 461364a @460 Bit of cleanup.
new 54f2ffc @461 Whoops, fix mistake in distrib update add.
new 30c372b @462 Make managed schema reloading a little less whacko.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../cloud/OverseerConfigSetMessageHandler.java | 2 +-
.../apache/solr/cloud/ZkSolrResourceLoader.java | 1 +
.../apache/solr/cloud/overseer/ZkStateWriter.java | 20 +++++---
.../org/apache/solr/core/ConfigSetProperties.java | 1 -
.../org/apache/solr/core/ConfigSetService.java | 7 ++-
.../java/org/apache/solr/core/CoreContainer.java | 2 +-
.../src/java/org/apache/solr/core/SolrCore.java | 11 ++---
.../org/apache/solr/handler/SolrConfigHandler.java | 55 +++++++++++++---------
.../org/apache/solr/update/SolrCmdDistributor.java | 2 +-
.../processor/DistributedUpdateProcessor.java | 20 +++++---
.../configsets/cloud-managed/conf/managed-schema | 4 +-
.../org/apache/solr/handler/TestReqParamsAPI.java | 6 +++
12 files changed, 84 insertions(+), 47 deletions(-)
[lucene-solr] 01/03: @460 Bit of cleanup.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 461364af99279245fda9810e6ea40f4541036bb5
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Wed Jul 29 23:48:18 2020 -0500
@460 Bit of cleanup.
---
.../apache/solr/cloud/overseer/ZkStateWriter.java | 20 +++++++++++++-------
.../org/apache/solr/core/ConfigSetProperties.java | 1 -
.../org/apache/solr/update/SolrCmdDistributor.java | 2 +-
3 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 07c1342..e167031 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -170,13 +170,12 @@ public class ZkStateWriter {
* @throws KeeperException if any ZooKeeper operation results in an error
* @throws InterruptedException if the current thread is interrupted
*/
- public ClusterState writePendingUpdates(ClusterState prevState) throws IllegalStateException, KeeperException, InterruptedException {
+ public ClusterState writePendingUpdates(ClusterState state) throws IllegalStateException, KeeperException, InterruptedException {
if (log.isDebugEnabled()) {
log.debug("writePendingUpdates() - start updates.size={}", updates.size());
}
-
- assert prevState != null;
+ ClusterState prevState = reader.getClusterState();
Timer.Context timerContext = stats.time("update_state");
boolean success = false;
ClusterState newClusterState = null;
@@ -303,7 +302,7 @@ public class ZkStateWriter {
log.debug("Write state.json bytes={} cs={}", data.length, newClusterState);
}
try {
- prevVersion = -1;
+ prevVersion = 0;
reader.getZkClient().create(path, data, CreateMode.PERSISTENT, true);
} catch (KeeperException.NodeExistsException e) {
stat = reader.getZkClient().setData(path, data, -1, true);
@@ -327,14 +326,21 @@ public class ZkStateWriter {
}
// }
- updates.clear();
// numUpdates = 0;
if (c != null) {
try {
+ System.out.println("waiting to see state " + prevVersion);
+ Integer finalPrevVersion = prevVersion;
reader.waitForState(c.getName(), 15, TimeUnit.SECONDS,
(l, col) -> {
- if (col != null && col.getZNodeVersion() > prevState.getZNodeVersion()) {
+
+ if (col != null) {
+ System.out.println("the version " + col.getZNodeVersion());
+ }
+
+ if (col != null && col.getZNodeVersion() > finalPrevVersion) {
if (log.isDebugEnabled()) log.debug("Waited for ver: {}", col.getZNodeVersion());
+ System.out.println("found the version");
return true;
}
return false;
@@ -358,7 +364,7 @@ public class ZkStateWriter {
if (exception != null) {
throw exception;
}
-
+ updates.clear();
success = true;
} finally {
timerContext.stop();
diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetProperties.java b/solr/core/src/java/org/apache/solr/core/ConfigSetProperties.java
index c0c75ea..a711238 100644
--- a/solr/core/src/java/org/apache/solr/core/ConfigSetProperties.java
+++ b/solr/core/src/java/org/apache/solr/core/ConfigSetProperties.java
@@ -60,7 +60,6 @@ public class ConfigSetProperties {
try (InputStream resource = loader.openResource(name)) {
try (InputStreamReader reader = new InputStreamReader(resource, StandardCharsets.UTF_8)) {
try {
- System.out.println("JSON:" + IOUtils.toString(reader));
return readFromInputStream(reader);
} finally {
ParWork.close(reader);
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 32bbe78..39dc1fe 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -130,7 +130,7 @@ public class SolrCmdDistributor implements Closeable {
log.info("check retry true");
return true;
} else {
- log.info("max retries exhausted retry false");
+ log.info("max retries exhausted or not a retryable error {} {}", err.req.retries, rspCode);
return false;
}
} else {
[lucene-solr] 02/03: @461 Whoops, fix mistake in distrib update add.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 54f2ffc40ec6036d9b2480b3d4947ac68f781a2f
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Jul 30 00:13:42 2020 -0500
@461 Whoops, fix mistake in distrib update add.
---
.../update/processor/DistributedUpdateProcessor.java | 20 ++++++++++++++------
1 file changed, 14 insertions(+), 6 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index c4c1a79..9fb7eea 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -236,6 +236,10 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
return;
}
+ SolrInputDocument clonedDoc = shouldCloneCmdDoc() ? cmd.solrDoc.deepCopy(): null;
+ if (clonedDoc != null) {
+ cmd.solrDoc = clonedDoc;
+ }
try (ParWork worker = new ParWork(this)) {
worker.collect(() -> {
if (vinfo != null) vinfo.lockForUpdate();
@@ -248,7 +252,8 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// force open a realTimeSearcher to trigger a ulog cache refresh.
// This refresh makes RTG handler aware of this update.q
if (ulog != null) {
- if (req.getSchema().isUsableForChildDocs() && shouldRefreshUlogCaches(cmd)) {
+ if (req.getSchema().isUsableForChildDocs()
+ && shouldRefreshUlogCaches(cmd)) {
ulog.openRealtimeSearcher();
}
}
@@ -259,13 +264,16 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
if (vinfo != null) vinfo.unlockForUpdate();
}
});
- SolrInputDocument clonedDoc = shouldCloneCmdDoc() ? cmd.solrDoc.deepCopy(): null;
- if (clonedDoc != null) {
- cmd.solrDoc = clonedDoc;
- }
if (req.getCore().getCoreContainer().isZooKeeperAware()) {
- doDistribAdd(worker, cmd);
+ worker.collect(() -> {
+ try {
+ doDistribAdd(worker, cmd);
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ });
}
+
worker.addCollect("distUpdate");
}
[lucene-solr] 03/03: @462 Make managed schema reloading a little
less whacko.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 30c372b1b5f6ee16748718a47939b3580c4c5efe
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Thu Jul 30 00:52:50 2020 -0500
@462 Make managed schema reloading a little less whacko.
---
.../cloud/OverseerConfigSetMessageHandler.java | 2 +-
.../apache/solr/cloud/ZkSolrResourceLoader.java | 1 +
.../org/apache/solr/core/ConfigSetService.java | 7 ++-
.../java/org/apache/solr/core/CoreContainer.java | 2 +-
.../src/java/org/apache/solr/core/SolrCore.java | 11 ++---
.../org/apache/solr/handler/SolrConfigHandler.java | 55 +++++++++++++---------
.../configsets/cloud-managed/conf/managed-schema | 4 +-
.../org/apache/solr/handler/TestReqParamsAPI.java | 6 +++
8 files changed, 56 insertions(+), 32 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
index 13b0d6d..9970d666 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
@@ -279,7 +279,7 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
}
return propertyDataStr.getBytes(StandardCharsets.UTF_8);
}
- return null;
+ return ZkStateReader.emptyJson;
}
private String getPropertyPath(String configName, String propertyPath) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java b/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
index 33b965c..fcb38a6 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkSolrResourceLoader.java
@@ -75,6 +75,7 @@ public class ZkSolrResourceLoader extends SolrResourceLoader implements Resource
InputStream is;
String file = (".".equals(resource)) ? configSetZkPath : configSetZkPath + "/" + resource;
+
try {
Stat stat = new Stat();
diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
index 9734811..6a4d07f 100644
--- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
@@ -72,7 +72,12 @@ public abstract class ConfigSetService {
// ConfigSet properties are loaded from ConfigSetProperties.DEFAULT_FILENAME file.
NamedList properties = loadConfigSetProperties(dcore, coreLoader);
// ConfigSet flags are loaded from the metadata of the ZK node of the configset.
- NamedList flags = loadConfigSetFlags(dcore, coreLoader);
+ NamedList flags = null;
+ try {
+ flags = loadConfigSetFlags(dcore, coreLoader);
+ } catch (Exception e) {
+ ParWork.propegateInterrupt(e);
+ }
boolean trusted =
(coreLoader instanceof ZkSolrResourceLoader
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 7b74150..3a858c4 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -1642,7 +1642,7 @@ public class CoreContainer implements Closeable {
Closeable oldCore = null;
boolean success = false;
try {
- solrCores.waitAddPendingCoreOps(cd.getName());
+ solrCores.waitForLoadingCoreToFinish(cd.getName(), 15000);
ConfigSet coreConfig = coreConfigService.loadConfigSet(cd);
log.info("Reloading SolrCore '{}' using configuration from {}", cd.getName(), coreConfig.getName());
newCore = core.reload(coreConfig);
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index ae5e279..a985675 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -3163,19 +3163,19 @@ public final class SolrCore implements SolrInfoBean, Closeable {
checkStale(zkClient, managedSchmaResourcePath, managedSchemaVersion)) {
log.info("core reload {}", coreName);
SolrConfigHandler configHandler = ((SolrConfigHandler) core.getRequestHandler("/config"));
- if (configHandler.getReloadLock().tryLock()) {
+ // if (configHandler.getReloadLock().tryLock()) {
try {
cc.reload(coreName);
} catch (SolrCoreState.CoreIsClosedException e) {
/*no problem this core is already closed*/
} finally {
- configHandler.getReloadLock().unlock();
+ // configHandler.getReloadLock().unlock();
}
- } else {
- log.info("Another reload is in progress. Not doing anything.");
- }
+// } else {
+// log.info("Another reload is in progress. Not doing anything.");
+// }
return;
}
//some files in conf directory may have other than managedschema, overlay, params
@@ -3187,7 +3187,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
try {
listener.run();
} catch (Exception e) {
- ParWork.propegateInterrupt(e);
ParWork.propegateInterrupt("Error in listener ", e);
}
});
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
index eee36ae..45e03ea 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
@@ -108,12 +108,11 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
public static final String CONFIGSET_EDITING_DISABLED_ARG = "disable.configEdit";
public static final boolean configEditing_disabled = Boolean.getBoolean(CONFIGSET_EDITING_DISABLED_ARG);
private static final Map<String, SolrConfig.SolrPluginInfo> namedPlugins;
- private Lock reloadLock = new ReentrantLock(true);
- private HttpClient httpClient;
+ // private Lock reloadLock = new ReentrantLock(true);
- public Lock getReloadLock() {
- return reloadLock;
- }
+// //public Lock getReloadLock() {
+// return reloadLock;
+// }
private boolean isImmutableConfigSet = false;
@@ -151,7 +150,6 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
@Override
public void inform(SolrCore core) {
isImmutableConfigSet = getImmutable(core);
- this.httpClient = core.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient();
}
public static boolean getImmutable(SolrCore core) {
@@ -229,21 +227,35 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
log.info("I already have the expected version {} of params", expectedVersion);
}
if (isStale && req.getCore().getResourceLoader() instanceof ZkSolrResourceLoader) {
- new Thread(() -> {
- if (!reloadLock.tryLock()) {
- log.info("Another reload is in progress . Not doing anything");
- return;
- }
- try {
- log.info("Trying to update my configs");
- SolrCore.getConfListener(req.getCore(), (ZkSolrResourceLoader) req.getCore().getResourceLoader()).run();
- } catch (Exception e) {
- ParWork.propegateInterrupt(e);
- log.error("Unable to refresh conf ", e);
- } finally {
- reloadLock.unlock();
+ // if (!reloadLock.tryLock()) {
+ // log.info("Another reload is in progress . Not doing anything");
+ // return;
+ // }
+ // reloadLock.unlock();
+ Runnable runner = new Runnable() {
+ @Override
+ public void run() {
+ // if (!reloadLock.tryLock()) {
+ // log.info("Another reload is in progress . Not doing anything");
+ // return;
+ // }
+ try {
+ log.info("Trying to update my configs");
+ SolrCore.getConfListener(req.getCore(),
+ (ZkSolrResourceLoader) req.getCore()
+ .getResourceLoader()).run();
+ } catch (Exception e) {
+ ParWork.propegateInterrupt(e);
+ if (e instanceof InterruptedException) {
+ return;
+ }
+ log.error("Unable to refresh conf ", e);
+ } finally {
+ // reloadLock.unlock();
+ }
}
- }, SolrConfigHandler.class.getSimpleName() + "-refreshconf").start();
+ };
+ runner.run();
} else {
if (log.isInfoEnabled()) {
log.info("isStale {} , resourceloader {}", isStale, req.getCore().getResourceLoader().getClass().getName());
@@ -370,7 +382,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
List<CommandOperation> ops = CommandOperation.readCommands(req.getContentStreams(), resp.getValues());
if (ops == null) return;
try {
- for (; ; ) {
+ for (int i = 0; i < 5; i++) {
ArrayList<CommandOperation> opsCopy = new ArrayList<>(ops.size());
for (CommandOperation op : ops) opsCopy.add(op.getCopy());
try {
@@ -387,6 +399,7 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
if (log.isInfoEnabled()) {
log.info("Race condition, the node is modified in ZK by someone else {}", e.getMessage());
}
+ Thread.sleep(250);
}
}
} catch (Exception e) {
diff --git a/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema b/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema
index 14e5b94..44ac101 100644
--- a/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema
+++ b/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema
@@ -17,8 +17,8 @@
-->
<schema name="minimal" version="1.1">
<fieldType name="string" class="solr.StrField"/>
- <fieldType name="int" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="int" class="${solr.tests.IntegerFieldType}" docValues="${solr.tests.numeric.dv}" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" omitNorms="true" positionIncrementGap="0"/>
<!-- for versioning -->
<field name="_version_" type="long" indexed="true" stored="true"/>
<field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java b/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java
index 5876ea9..da681a6 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java
@@ -58,6 +58,12 @@ public class TestReqParamsAPI extends SolrCloudTestCase {
@BeforeClass
public static void createCluster() throws Exception {
+ System.setProperty("solr.tests.IntegerFieldType", "org.apache.solr.schema.IntPointField");
+ System.setProperty("solr.tests.FloatFieldType", "org.apache.solr.schema.FloatPointField");
+ System.setProperty("solr.tests.LongFieldType", "org.apache.solr.schema.LongPointField");
+ System.setProperty("solr.tests.DoubleFieldType", "org.apache.solr.schema.DoublePointField");
+ System.setProperty("solr.tests.DateFieldType", "org.apache.solr.schema.DatePointField");
+ System.setProperty("solr.tests.EnumFieldType", "org.apache.solr.schema.EnumFieldType");
System.setProperty("managed.schema.mutable", "true");
configureCluster(2)
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-managed").resolve("conf"))