You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2019/06/14 19:45:33 UTC
[lucene-solr] branch branch_8x updated: SOLR-13420 Routed Aliases
now use collection properties instead of core properties
This is an automated email from the ASF dual-hosted git repository.
gus pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 2009c81 SOLR-13420 Routed Aliases now use collection properties instead of core properties
2009c81 is described below
commit 2009c813742811eb776e3223fe9bc529359880e4
Author: Gus Heck <gu...@apache.org>
AuthorDate: Fri Jun 14 15:44:36 2019 -0400
SOLR-13420 Routed Aliases now use collection properties instead of core properties
---
solr/CHANGES.txt | 8 +++++-
.../solr/cloud/api/collections/AliasCmd.java | 7 ++++-
.../processor/RoutedAliasUpdateProcessor.java | 30 +++++++++++++++++++---
.../TimeRoutedAliasUpdateProcessorTest.java | 21 ++++++++++++---
4 files changed, 58 insertions(+), 8 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f0e941b..55f1b6e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -43,6 +43,12 @@ Upgrade Notes
* SOLR-13541: Upgrade Jetty to 9.4.19.v20190610. (Erick Erickson, Cao Manh Dat)
+* SOLR-13420: Routed Aliases now use collection properties rather than core properties to identify collections that
+ belong to the alias by default. This should be invisible and fully backwards compatible from within solr, and
+ existing routed alias collections with core based properties will continue to work, but new collections created will
+ not add a property to core.properties anymoore so any external code that inspected core.properties will not find the
+ 'routedAliasName' key in new cores belonging to routed aliases.
+
New Features
----------------------
@@ -732,7 +738,7 @@ Upgrade Notes
* SOLR-12708: When requesting the status of an async request via REQUESTSTATUS collections API, the response will
include the list of internal async requests (if any) in the "success" or "failed" keys (in addition
- to them being included outside those keys for backwards compatibility). See SOLR-12708 for more
+ to them being included outside those keys for backwards compatibility). See SOLR-12708 for more
details
Bug fixes
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
index 05cca40..752bc5b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
@@ -25,6 +25,7 @@ import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.OverseerSolrResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.CollectionProperties;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
@@ -64,7 +65,6 @@ abstract class AliasCmd implements OverseerCollectionMessageHandler.Cmd {
"We require an explicit " + COLL_CONF);
}
createReqParams.set(NAME, createCollName);
- createReqParams.set("property." + ROUTED_ALIAS_NAME_CORE_PROP, aliasName);
// a CollectionOperation reads params and produces a message (Map) that is supposed to be sent to the Overseer.
// Although we could create the Map without it, there are a fair amount of rules we don't want to reproduce.
final Map<String, Object> createMsgMap = CollectionsHandler.CollectionOperation.CREATE_OP.execute(
@@ -88,6 +88,11 @@ abstract class AliasCmd implements OverseerCollectionMessageHandler.Cmd {
CollectionsHandler.waitForActiveCollection(createCollName, ocmh.overseer.getCoreContainer(),
new OverseerSolrResponse(results));
+ CollectionProperties collectionProperties = new CollectionProperties(ocmh.zkStateReader.getZkClient());
+ collectionProperties.setCollectionProperty(createCollName,ROUTED_ALIAS_NAME_CORE_PROP,aliasName);
+ while (!ocmh.zkStateReader.getCollectionProperties(createCollName,1000).containsKey(ROUTED_ALIAS_NAME_CORE_PROP)) {
+ Thread.sleep(50);
+ }
return results;
}
diff --git a/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
index 13dd731..384fcd8 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
@@ -24,6 +24,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.api.collections.RoutedAlias;
import org.apache.solr.common.SolrException;
@@ -33,10 +35,12 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
@@ -67,6 +71,11 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor {
private static final String ALIAS_DISTRIB_UPDATE_PARAM = "alias." + DISTRIB_UPDATE_PARAM; // param
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ // make sure we don't request collection properties any more frequently than once a minute during
+ // slow continuous indexing, and even less frequently during bulk indexing. (cache is updated by zk
+ // watch instead of re-requested until indexing has been stopped for the duration specified here)
+ public static final int CACHE_FOR_MILLIS = 60000;
+
// refs to std infrastructure
private final SolrQueryRequest req;
private final SolrCmdDistributor cmdDistrib;
@@ -79,9 +88,24 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor {
public static UpdateRequestProcessor wrap(SolrQueryRequest req, UpdateRequestProcessor next) {
- //TODO get from "Collection property"
- final String aliasName = req.getCore().getCoreDescriptor()
- .getCoreProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, null);
+ String aliasName = null;
+ // Demeter please don't arrest us... hide your eyes :(
+ // todo: a core should have a more direct way of finding a collection name, and the collection properties
+ SolrCore core = req.getCore();
+ CoreDescriptor coreDescriptor = core.getCoreDescriptor();
+ CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor();
+ if (cloudDescriptor != null) {
+ String collectionName = cloudDescriptor.getCollectionName();
+ CoreContainer coreContainer = core.getCoreContainer();
+ ZkController zkController = coreContainer.getZkController();
+ ZkStateReader zkStateReader = zkController.getZkStateReader();
+ Map<String, String> collectionProperties = zkStateReader.getCollectionProperties(collectionName, CACHE_FOR_MILLIS);
+ aliasName = collectionProperties.get(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP);
+ }
+ // fall back on core properties (legacy)
+ if (StringUtils.isBlank(aliasName)) {
+ aliasName = coreDescriptor.getCoreProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, null);
+ }
final DistribPhase shardDistribPhase =
DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
final DistribPhase aliasDistribPhase =
diff --git a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
index c1d25a6..6636bb6 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@@ -36,13 +37,13 @@ import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.cloud.api.collections.RoutedAlias;
import org.apache.solr.cloud.api.collections.TimeRoutedAlias;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.update.UpdateCommand;
@@ -54,6 +55,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.solr.cloud.api.collections.RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTIONS_ZKNODE;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROPS_ZKNODE;
@LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13059")
public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcessorTest {
@@ -98,7 +102,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
final String col23rd = alias + "_2017-10-23";
CollectionAdminRequest.createCollection(col23rd, configName, 2, 2)
.setMaxShardsPerNode(2)
- .withProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, alias)
+ .withProperty(ROUTED_ALIAS_NAME_CORE_PROP, alias)
.process(solrClient);
cluster.waitForActiveCollection(col23rd, 2, 4);
@@ -132,7 +136,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
// destined for this collection, Solr will see it already exists and add it to the alias.
final String col24th = alias + "_2017-10-24";
CollectionAdminRequest.createCollection(col24th, configName, 1, 1) // more shards and replicas now
- .withProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, alias)
+ .withProperty(ROUTED_ALIAS_NAME_CORE_PROP, alias)
.process(solrClient);
// index 3 documents in a random fashion
@@ -175,6 +179,17 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
);
assertInvariants(alias + "_2017-10-26", alias + "_2017-10-25", col24th);
+ // verify that collection properties are set when the collections are created. Note: first 2 collections in
+ // this test have a core property instead, of a collection property but that MUST continue to work as well
+ // for back compatibility's reasons.
+ Thread.sleep(1000);
+ byte[] data = cluster.getZkClient()
+ .getData(COLLECTIONS_ZKNODE + "/" + alias + "_2017-10-26" + "/" + COLLECTION_PROPS_ZKNODE,null, null, true);
+ assertNotNull(data);
+ assertTrue(data.length > 0);
+ Map<String,String> props = (Map<String, String>) Utils.fromJSON(data);
+ assertTrue(props.containsKey(ROUTED_ALIAS_NAME_CORE_PROP));
+ assertEquals(alias,props.get(ROUTED_ALIAS_NAME_CORE_PROP));
// update metadata to auto-delete oldest collections
CollectionAdminRequest.setAliasProperty(alias)