You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2019/06/14 17:51:42 UTC
[lucene-solr] branch master updated: SOLR-13420 Routed Aliases now
use collection properties instead of core properties
This is an automated email from the ASF dual-hosted git repository.
gus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 5d550a3 SOLR-13420 Routed Aliases now use collection properties instead of core properties
5d550a3 is described below
commit 5d550a34a9b54d317e51d777d5888b1520767c5a
Author: Gus Heck <gu...@apache.org>
AuthorDate: Fri Jun 14 13:51:16 2019 -0400
SOLR-13420 Routed Aliases now use collection properties instead of core properties
---
solr/CHANGES.txt | 6 +++++
.../solr/cloud/api/collections/AliasCmd.java | 7 ++++-
.../processor/RoutedAliasUpdateProcessor.java | 30 +++++++++++++++++++---
.../TimeRoutedAliasUpdateProcessorTest.java | 21 ++++++++++++---
4 files changed, 57 insertions(+), 7 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index de56036..83c949c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -78,6 +78,12 @@ Upgrade Notes
* SOLR-13541: Upgrade Jetty to 9.4.19.v20190610. (Erick Erickson, Cao Manh Dat)
+* SOLR-13420: Routed Aliases now use collection properties rather than core properties to identify collections that
+ belong to the alias by default. This should be invisible and fully backwards compatible from within solr, and
+ existing routed alias collections with core based properties will continue to work, but new collections created will
+ not add a property to core.properties anymoore so any external code that inspected core.properties will not find the
+ 'routedAliasName' key in new cores belonging to routed aliases.
+
New Features
----------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
index 05cca40..752bc5b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
@@ -25,6 +25,7 @@ import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.OverseerSolrResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.CollectionProperties;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
@@ -64,7 +65,6 @@ abstract class AliasCmd implements OverseerCollectionMessageHandler.Cmd {
"We require an explicit " + COLL_CONF);
}
createReqParams.set(NAME, createCollName);
- createReqParams.set("property." + ROUTED_ALIAS_NAME_CORE_PROP, aliasName);
// a CollectionOperation reads params and produces a message (Map) that is supposed to be sent to the Overseer.
// Although we could create the Map without it, there are a fair amount of rules we don't want to reproduce.
final Map<String, Object> createMsgMap = CollectionsHandler.CollectionOperation.CREATE_OP.execute(
@@ -88,6 +88,11 @@ abstract class AliasCmd implements OverseerCollectionMessageHandler.Cmd {
CollectionsHandler.waitForActiveCollection(createCollName, ocmh.overseer.getCoreContainer(),
new OverseerSolrResponse(results));
+ CollectionProperties collectionProperties = new CollectionProperties(ocmh.zkStateReader.getZkClient());
+ collectionProperties.setCollectionProperty(createCollName,ROUTED_ALIAS_NAME_CORE_PROP,aliasName);
+ while (!ocmh.zkStateReader.getCollectionProperties(createCollName,1000).containsKey(ROUTED_ALIAS_NAME_CORE_PROP)) {
+ Thread.sleep(50);
+ }
return results;
}
diff --git a/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
index 13dd731..384fcd8 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
@@ -24,6 +24,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.api.collections.RoutedAlias;
import org.apache.solr.common.SolrException;
@@ -33,10 +35,12 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
@@ -67,6 +71,11 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor {
private static final String ALIAS_DISTRIB_UPDATE_PARAM = "alias." + DISTRIB_UPDATE_PARAM; // param
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ // make sure we don't request collection properties any more frequently than once a minute during
+ // slow continuous indexing, and even less frequently during bulk indexing. (cache is updated by zk
+ // watch instead of re-requested until indexing has been stopped for the duration specified here)
+ public static final int CACHE_FOR_MILLIS = 60000;
+
// refs to std infrastructure
private final SolrQueryRequest req;
private final SolrCmdDistributor cmdDistrib;
@@ -79,9 +88,24 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor {
public static UpdateRequestProcessor wrap(SolrQueryRequest req, UpdateRequestProcessor next) {
- //TODO get from "Collection property"
- final String aliasName = req.getCore().getCoreDescriptor()
- .getCoreProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, null);
+ String aliasName = null;
+ // Demeter please don't arrest us... hide your eyes :(
+ // todo: a core should have a more direct way of finding a collection name, and the collection properties
+ SolrCore core = req.getCore();
+ CoreDescriptor coreDescriptor = core.getCoreDescriptor();
+ CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor();
+ if (cloudDescriptor != null) {
+ String collectionName = cloudDescriptor.getCollectionName();
+ CoreContainer coreContainer = core.getCoreContainer();
+ ZkController zkController = coreContainer.getZkController();
+ ZkStateReader zkStateReader = zkController.getZkStateReader();
+ Map<String, String> collectionProperties = zkStateReader.getCollectionProperties(collectionName, CACHE_FOR_MILLIS);
+ aliasName = collectionProperties.get(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP);
+ }
+ // fall back on core properties (legacy)
+ if (StringUtils.isBlank(aliasName)) {
+ aliasName = coreDescriptor.getCoreProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, null);
+ }
final DistribPhase shardDistribPhase =
DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
final DistribPhase aliasDistribPhase =
diff --git a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
index c1d25a6..6636bb6 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@@ -36,13 +37,13 @@ import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.cloud.api.collections.RoutedAlias;
import org.apache.solr.cloud.api.collections.TimeRoutedAlias;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.update.UpdateCommand;
@@ -54,6 +55,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.solr.cloud.api.collections.RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTIONS_ZKNODE;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROPS_ZKNODE;
@LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13059")
public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcessorTest {
@@ -98,7 +102,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
final String col23rd = alias + "_2017-10-23";
CollectionAdminRequest.createCollection(col23rd, configName, 2, 2)
.setMaxShardsPerNode(2)
- .withProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, alias)
+ .withProperty(ROUTED_ALIAS_NAME_CORE_PROP, alias)
.process(solrClient);
cluster.waitForActiveCollection(col23rd, 2, 4);
@@ -132,7 +136,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
// destined for this collection, Solr will see it already exists and add it to the alias.
final String col24th = alias + "_2017-10-24";
CollectionAdminRequest.createCollection(col24th, configName, 1, 1) // more shards and replicas now
- .withProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, alias)
+ .withProperty(ROUTED_ALIAS_NAME_CORE_PROP, alias)
.process(solrClient);
// index 3 documents in a random fashion
@@ -175,6 +179,17 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
);
assertInvariants(alias + "_2017-10-26", alias + "_2017-10-25", col24th);
+ // verify that collection properties are set when the collections are created. Note: first 2 collections in
+ // this test have a core property instead, of a collection property but that MUST continue to work as well
+ // for back compatibility's reasons.
+ Thread.sleep(1000);
+ byte[] data = cluster.getZkClient()
+ .getData(COLLECTIONS_ZKNODE + "/" + alias + "_2017-10-26" + "/" + COLLECTION_PROPS_ZKNODE,null, null, true);
+ assertNotNull(data);
+ assertTrue(data.length > 0);
+ Map<String,String> props = (Map<String, String>) Utils.fromJSON(data);
+ assertTrue(props.containsKey(ROUTED_ALIAS_NAME_CORE_PROP));
+ assertEquals(alias,props.get(ROUTED_ALIAS_NAME_CORE_PROP));
// update metadata to auto-delete oldest collections
CollectionAdminRequest.setAliasProperty(alias)