You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2019/01/29 02:53:12 UTC
[lucene-solr] branch solr-13131 updated (3f3098e -> 1b73b93)
This is an automated email from the ASF dual-hosted git repository.
gus pushed a change to branch solr-13131
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
discard 3f3098e SOLR-13149 First draft of Cagegory Routed Alias class. Not really useful or testable until I implement MaintanCategoryRoutedAlias however
discard c78e1b6 SOLR-13148 Factory for routed aliases, remove CollectionHandler deps on TimeRoutedAlias, minor cleanup
discard ce3dc6e SOLR-13148: remove hard coded pointers to TimeRoutedAlias
discard 5c90eed SOLR-12348: move start collection creation logic to TimeRoutedAlias
new f0b960f SOLR-13148: move start collection creation logic to TimeRoutedAlias
new 84fda4d SOLR-13148: remove hard coded pointers to TimeRoutedAlias
new f0bf992 SOLR-13148 Factory for routed aliases, remove CollectionHandler deps on TimeRoutedAlias, minor cleanup
new 1b73b93 SOLR-13149 First draft of Cagegory Routed Alias class. Not really useful or testable until I implement MaintanCategoryRoutedAlias however
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (3f3098e)
\
N -- N -- N refs/heads/solr-13131 (1b73b93)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[lucene-solr] 01/04: SOLR-13148: move start collection creation
logic to TimeRoutedAlias
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gus pushed a commit to branch solr-13131
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit f0b960fead2bddbe611414bfab72d445198ab435
Author: Moshe <mo...@mail.com>
AuthorDate: Mon Jan 28 02:06:47 2019 +0200
SOLR-13148: move start collection creation logic to TimeRoutedAlias
---
.../cloud/api/collections/CategoryRoutedAlias.java | 6 +++
.../solr/cloud/api/collections/CreateAliasCmd.java | 48 +++++++++-------------
.../solr/cloud/api/collections/RoutedAlias.java | 11 +++++
.../cloud/api/collections/TimeRoutedAlias.java | 20 +++++++++
4 files changed, 56 insertions(+), 29 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
index bc565f03..3ac09c6 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
@@ -20,6 +20,7 @@ package org.apache.solr.cloud.api.collections;
import java.time.Instant;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.cloud.Aliases;
@@ -59,4 +60,9 @@ public class CategoryRoutedAlias implements RoutedAlias {
public String createCollectionsIfRequired(AddUpdateCommand cmd) {
return null;
}
+
+ @Override
+ public String computeInitialCollectionName(String collection) {
+ return null;
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java
index 61931d9..dd8edc1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java
@@ -17,17 +17,13 @@
*/
package org.apache.solr.cloud.api.collections;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
import java.util.Collections;
-import java.util.Date;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
-import java.util.TimeZone;
import java.util.stream.Collectors;
import org.apache.solr.common.SolrException;
@@ -37,7 +33,6 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.util.DateMathParser;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
@@ -46,7 +41,7 @@ public class CreateAliasCmd extends AliasCmd {
private final OverseerCollectionMessageHandler ocmh;
private static boolean anyRoutingParams(ZkNodeProps message) {
- return message.keySet().stream().anyMatch(k -> k.startsWith(TimeRoutedAlias.ROUTER_PREFIX));
+ return message.keySet().stream().anyMatch(k -> k.startsWith(RoutedAlias.ROUTER_PREFIX));
}
public CreateAliasCmd(OverseerCollectionMessageHandler ocmh) {
@@ -113,35 +108,30 @@ public class CreateAliasCmd extends AliasCmd {
.filter(entry -> TimeRoutedAlias.PARAM_IS_PROP.test(entry.getKey()))
.forEach(entry -> aliasProperties.put(entry.getKey(), (String) entry.getValue())); // way easier than .collect
- TimeRoutedAlias timeRoutedAlias = new TimeRoutedAlias(aliasName, aliasProperties); // validates as well
-
- String start = message.getStr(TimeRoutedAlias.ROUTER_START);
- Instant startTime = parseStart(start, timeRoutedAlias.getTimeZone());
-
- String initialCollectionName = TimeRoutedAlias.formatCollectionNameFromInstant(aliasName, startTime);
-
- // Create the collection
- createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, ocmh);
- validateAllCollectionsExistAndNoDups(Collections.singletonList(initialCollectionName), zkStateReader);
+ RoutedAlias timeRoutedAlias = new TimeRoutedAlias(aliasName, aliasProperties); // validates as well
+
+ String start = message.getStr(RoutedAlias.ROUTER_START);
+ if (start != null) {
+ String initialCollectionName = timeRoutedAlias.computeInitialCollectionName(start);
+ if (initialCollectionName != null) {
+ ensureCollection(aliasName, zkStateReader, state, aliasProperties, initialCollectionName);
+ // Create/update the alias
+ zkStateReader.aliasesManager.applyModificationAndExportToZk(aliases -> aliases
+ .cloneWithCollectionAlias(aliasName, initialCollectionName)
+ .cloneWithCollectionAliasProperties(aliasName, aliasProperties));
+ return;
+ }
+ }
// Create/update the alias
zkStateReader.aliasesManager.applyModificationAndExportToZk(aliases -> aliases
- .cloneWithCollectionAlias(aliasName, initialCollectionName)
.cloneWithCollectionAliasProperties(aliasName, aliasProperties));
}
- private Instant parseStart(String str, TimeZone zone) {
- Instant start = DateMathParser.parseMath(new Date(), str, zone).toInstant();
- checkMilis(start);
- return start;
- }
-
- private void checkMilis(Instant date) {
- if (!date.truncatedTo(ChronoUnit.SECONDS).equals(date)) {
- throw new SolrException(BAD_REQUEST,
- "Date or date math for start time includes milliseconds, which is not supported. " +
- "(Hint: 'NOW' used without rounding always has this problem)");
- }
+ private void ensureCollection(String aliasName, ZkStateReader zkStateReader, ClusterState state, Map<String, String> aliasProperties, String initialCollectionName) throws Exception {
+ // Create the collection
+ createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, ocmh);
+ validateAllCollectionsExistAndNoDups(Collections.singletonList(initialCollectionName), zkStateReader);
}
private void validateAllCollectionsExistAndNoDups(List<String> collectionList, ZkStateReader zkStateReader) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
index dd7240f..b004d79 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
@@ -20,7 +20,9 @@ package org.apache.solr.cloud.api.collections;
import java.time.Instant;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import com.google.common.collect.Sets;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
@@ -32,7 +34,9 @@ public interface RoutedAlias {
String ROUTER_TYPE_NAME = ROUTER_PREFIX + "name";
String ROUTER_FIELD = ROUTER_PREFIX + "field";
String ROUTER_AUTO_DELETE_AGE = ROUTER_PREFIX + "autoDeleteAge";
+ String ROUTER_START = ROUTER_PREFIX + "start";
String CREATE_COLLECTION_PREFIX = "create-collection.";
+ Set<String> MINIMAL_REQUIRED_PARAMS = Sets.newHashSet(ROUTER_TYPE_NAME, ROUTER_FIELD);
String ROUTED_ALIAS_NAME_CORE_PROP = "routedAliasName"; // core prop
static SolrException newAliasMustExistException(String aliasName) {
@@ -47,6 +51,13 @@ public interface RoutedAlias {
*/
boolean updateParsedCollectionAliases(ZkController zkController);
+ /**
+ *
+ * @param startParam the start parameter passed to create alias cmd
+ * @return optional string of initial collection name
+ */
+ String computeInitialCollectionName(String startParam);
+
String getAliasName();
/**
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
index a4b29ea..007e2ce 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
@@ -24,6 +24,7 @@ import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.temporal.ChronoField;
+import java.time.temporal.ChronoUnit;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
@@ -119,6 +120,11 @@ public class TimeRoutedAlias implements RoutedAlias {
.parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
.toFormatter(Locale.ROOT).withZone(ZoneOffset.UTC); // deliberate -- collection names disregard TZ
+ @Override
+ public String computeInitialCollectionName(String dateStr) {
+ return formatCollectionNameFromInstant(aliasName, parseStringAsInstant(dateStr, timeZone));
+ }
+
public static Instant parseInstantFromCollectionName(String aliasName, String collection) {
final String dateTimePart = collection.substring(aliasName.length() + 1);
return DATE_TIME_FORMATTER.parse(dateTimePart, Instant::from);
@@ -135,6 +141,20 @@ public class TimeRoutedAlias implements RoutedAlias {
return aliasName + "_" + nextCollName;
}
+ Instant parseStringAsInstant(String str, TimeZone zone) {
+ Instant start = DateMathParser.parseMath(new Date(), str, zone).toInstant();
+ checkMilis(start);
+ return start;
+ }
+
+ private void checkMilis(Instant date) {
+ if (!date.truncatedTo(ChronoUnit.SECONDS).equals(date)) {
+ throw new SolrException(BAD_REQUEST,
+ "Date or date math for start time includes milliseconds, which is not supported. " +
+ "(Hint: 'NOW' used without rounding always has this problem)");
+ }
+ }
+
@Override
public boolean updateParsedCollectionAliases(ZkController zkController) {
[lucene-solr] 03/04: SOLR-13148 Factory for routed aliases,
remove CollectionHandler deps on TimeRoutedAlias, minor cleanup
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gus pushed a commit to branch solr-13131
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit f0bf99228a6f523fce917a3b598065ecfbe67919
Author: Gus Heck <gu...@apache.org>
AuthorDate: Mon Jan 28 16:02:08 2019 -0500
SOLR-13148 Factory for routed aliases, remove CollectionHandler deps
on TimeRoutedAlias, minor cleanup
---
.../cloud/api/collections/CategoryRoutedAlias.java | 20 ++-
.../solr/cloud/api/collections/CreateAliasCmd.java | 48 +++---
.../solr/cloud/api/collections/RoutedAlias.java | 81 ++++++++--
.../cloud/api/collections/TimeRoutedAlias.java | 166 ++++++++++-----------
.../solr/handler/admin/CollectionsHandler.java | 41 +++--
.../processor/RoutedAliasUpdateProcessor.java | 20 +--
.../apache/solr/cloud/CreateRoutedAliasTest.java | 2 +-
7 files changed, 216 insertions(+), 162 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
index 30b0747..5a352a6 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
@@ -18,8 +18,11 @@
package org.apache.solr.cloud.api.collections;
import java.time.Instant;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.cloud.Aliases;
@@ -29,10 +32,9 @@ public class CategoryRoutedAlias implements RoutedAlias {
private final String aliasName;
private final Map<String, String> aliasProperties;
- public CategoryRoutedAlias(String aliasName, Map<String, String> aliasProperties) {
-
+ CategoryRoutedAlias(String aliasName, Map<String, String> aliasMetadata) {
this.aliasName = aliasName;
- this.aliasProperties = aliasProperties;
+ this.aliasProperties = aliasMetadata;
}
@Override
@@ -61,7 +63,7 @@ public class CategoryRoutedAlias implements RoutedAlias {
}
@Override
- public String computeInitialCollectionName(String collection) {
+ public Optional<String> computeInitialCollectionName() {
return null;
}
@@ -69,4 +71,14 @@ public class CategoryRoutedAlias implements RoutedAlias {
public Map<String, String> getAliasMetadata() {
return aliasProperties;
}
+
+ @Override
+ public Set<String> getRequiredParams() {
+ return new HashSet<>();
+ }
+
+ @Override
+ public Set<String> getOptionalParams() {
+ return new HashSet<>();
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java
index 182eda3..320a4ae 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java
@@ -19,11 +19,12 @@ package org.apache.solr.cloud.api.collections;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
-import java.util.function.BiFunction;
import java.util.stream.Collectors;
import org.apache.solr.common.SolrException;
@@ -97,36 +98,29 @@ public class CreateAliasCmd extends AliasCmd {
}
private void callCreateRoutedAlias(ZkNodeProps message, String aliasName, ZkStateReader zkStateReader, ClusterState state) throws Exception {
- // Validate we got everything we need
+ // Validate we got a basic minimum
if (!message.getProperties().keySet().containsAll(RoutedAlias.MINIMAL_REQUIRED_PARAMS)) {
throw new SolrException(BAD_REQUEST, "A routed alias requires these params: " + RoutedAlias.MINIMAL_REQUIRED_PARAMS
+ " plus some create-collection prefixed ones.");
}
- String aliasType = message.getStr(RoutedAlias.ROUTER_TYPE_NAME);
-
- BiFunction<String, ZkNodeProps, RoutedAlias> aliasConstructor = RoutedAlias.constructorFactory.get(aliasType);
-
- if (aliasConstructor == null) {
- throw new SolrException(BAD_REQUEST, "Router name: " + aliasType + " is not in supported types, "
- + String.join(", ", RoutedAlias.constructorFactory.keySet()));
- }
-
-
-
- RoutedAlias routedAlias = aliasConstructor.apply(aliasName, message);
-
- String start = message.getStr(RoutedAlias.ROUTER_START);
- if (start != null) {
- String initialCollectionName = routedAlias.computeInitialCollectionName(start);
- if (initialCollectionName != null) {
- ensureCollection(aliasName, zkStateReader, state, routedAlias.getAliasMetadata(), initialCollectionName);
- // Create/update the alias
- zkStateReader.aliasesManager.applyModificationAndExportToZk(aliases -> aliases
- .cloneWithCollectionAlias(aliasName, initialCollectionName)
- .cloneWithCollectionAliasProperties(aliasName, routedAlias.getAliasMetadata()));
- return;
- }
+ // convert values to strings
+ Map<String, String> props = new LinkedHashMap<>();
+ message.getProperties().forEach((key, value) -> props.put(key, String.valueOf(value)));
+
+ // Further validation happens here
+ RoutedAlias routedAlias = RoutedAlias.fromProps(aliasName, props);
+
+ // If we can, create the first collection.
+ Optional<String> initialCollectionName = routedAlias.computeInitialCollectionName();
+ if (initialCollectionName.isPresent()) {
+ String initialColl = initialCollectionName.get();
+ ensureAliasCollection(aliasName, zkStateReader, state, routedAlias.getAliasMetadata(), initialColl);
+ // Create/update the alias
+ zkStateReader.aliasesManager.applyModificationAndExportToZk(aliases -> aliases
+ .cloneWithCollectionAlias(aliasName, initialColl)
+ .cloneWithCollectionAliasProperties(aliasName, routedAlias.getAliasMetadata()));
+ return;
}
// Create/update the alias
@@ -134,7 +128,7 @@ public class CreateAliasCmd extends AliasCmd {
.cloneWithCollectionAliasProperties(aliasName, routedAlias.getAliasMetadata()));
}
- private void ensureCollection(String aliasName, ZkStateReader zkStateReader, ClusterState state, Map<String, String> aliasProperties, String initialCollectionName) throws Exception {
+ private void ensureAliasCollection(String aliasName, ZkStateReader zkStateReader, ClusterState state, Map<String, String> aliasProperties, String initialCollectionName) throws Exception {
// Create the collection
createCollectionAndWait(state, aliasName, aliasProperties, initialCollectionName, ocmh);
validateAllCollectionsExistAndNoDups(Collections.singletonList(initialCollectionName), zkStateReader);
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
index 1a607ba..5e7df68 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
@@ -18,30 +18,35 @@
package org.apache.solr.cloud.api.collections;
import java.time.Instant;
+import java.util.Arrays;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
-import java.util.function.BiFunction;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
-import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.update.AddUpdateCommand;
+import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
+
public interface RoutedAlias {
- Map<String, BiFunction<String, ZkNodeProps, RoutedAlias>> constructorFactory = ImmutableMap.<String, BiFunction<String, ZkNodeProps, RoutedAlias>>builder()
- .put("time", TimeRoutedAlias::fromZkProps)
- .build();
+ /**
+ * Types supported. Every entry here must have a case in the switch statement in {@link #fromProps(String, Map)}
+ */
+ enum SupportedRouterTypes {
+ TIME,
+ CATEGORY
+ }
String ROUTER_PREFIX = "router.";
String ROUTER_TYPE_NAME = ROUTER_PREFIX + "name";
String ROUTER_FIELD = ROUTER_PREFIX + "field";
- String ROUTER_AUTO_DELETE_AGE = ROUTER_PREFIX + "autoDeleteAge";
- String ROUTER_START = ROUTER_PREFIX + "start";
String CREATE_COLLECTION_PREFIX = "create-collection.";
Set<String> MINIMAL_REQUIRED_PARAMS = Sets.newHashSet(ROUTER_TYPE_NAME, ROUTER_FIELD);
String ROUTED_ALIAS_NAME_CORE_PROP = "routedAliasName"; // core prop
@@ -52,6 +57,42 @@ public interface RoutedAlias {
}
/**
+ * Factory method for implementations of this interface. There should be no reason to construct instances
+ * elsewhere, and routed alias types are encouraged to have package private constructors.
+ *
+ * @param aliasName The alias name (will be returned by {@link #getAliasName()}
+ * @param props The properties from an overseer message.
+ * @return An implementation appropriate for the supplied properties, or null if no type is specified.
+ * @throws SolrException If the properties are invalid or the router type is unknown.
+ */
+ static RoutedAlias fromProps(String aliasName, Map<String, String> props) throws SolrException {
+
+ String typeStr = props.get(ROUTER_TYPE_NAME);
+ if (typeStr == null) {
+ return null;
+ }
+ SupportedRouterTypes routerType;
+ try {
+ routerType = SupportedRouterTypes.valueOf(typeStr.toUpperCase(Locale.ENGLISH));
+ } catch (IllegalArgumentException e) {
+ throw new SolrException(BAD_REQUEST, "Router name: " + typeStr + " is not in supported types, "
+ + Arrays.asList(SupportedRouterTypes.values()));
+ }
+ switch (routerType) {
+ case TIME:
+ return new TimeRoutedAlias(aliasName, props);
+ case CATEGORY:
+ return new CategoryRoutedAlias(aliasName, props);
+ default:
+ // if we got a type not handled by the switch there's been a bogus implementation.
+ throw new SolrException(SERVER_ERROR, "Router " + routerType + " is not fully implemented. If you see this" +
+ "error in an official release please file a bug report. Available types were:"
+ + Arrays.asList(SupportedRouterTypes.values()));
+
+ }
+ }
+
+ /**
* Ensure our parsed version of the alias collection list is up to date. If it was modified, return true.
* Note that this will return true if some other alias was modified or if properties were modified. These
* are spurious and the caller should be written to be tolerant of no material changes.
@@ -59,19 +100,28 @@ public interface RoutedAlias {
boolean updateParsedCollectionAliases(ZkController zkController);
/**
+ * Create the initial collection for this RoutedAlias if applicable.
+ *
+ * Routed Aliases do not aggregate existing collections, instead they create collections on the fly. If the initial
+ * collection can be determined from initialization parameters it should be calculated here.
*
- * @param startParam the start parameter passed to create alias cmd
* @return optional string of initial collection name
*/
- String computeInitialCollectionName(String startParam);
+ Optional<String> computeInitialCollectionName();
+
+ /**
+ * The name of the alias. This name is used in place of a collection name for both queries and updates.
+ *
+ * @return The name of the Alias.
+ */
String getAliasName();
/**
* Parses the elements of the collection list. Result is returned them in sorted order (desc) if there
* is a natural order for this type of routed alias
*/
- List<Map.Entry<Instant,String>> parseCollections(Aliases aliases);
+ List<Map.Entry<Instant, String>> parseCollections(Aliases aliases);
/**
* Check that the value we will be routing on is legal for this type of routed alias.
@@ -85,13 +135,16 @@ public interface RoutedAlias {
*
* @param cmd The command that might cause collection creation
* @return The name of the proper destination collection for the document which may or may not be a
- * newly created collection
+ * newly created collection
*/
- String createCollectionsIfRequired( AddUpdateCommand cmd);
+ String createCollectionsIfRequired(AddUpdateCommand cmd);
/**
- *
* @return get alias related metadata
*/
Map<String, String> getAliasMetadata();
+
+ Set<String> getRequiredParams();
+
+ Set<String> getOptionalParams();
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
index c8b0ebb..afc9ce4 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
@@ -30,19 +30,19 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
-import java.util.LinkedHashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
-import java.util.function.Predicate;
import com.google.common.base.Objects;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
-import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.RequiredSolrParams;
@@ -58,7 +58,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
-import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.CreationType.*;
+import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.CreationType.ASYNC_PREEMPTIVE;
+import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.CreationType.NONE;
+import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.CreationType.SYNCHRONOUS;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.params.CommonParams.TZ;
@@ -86,33 +88,30 @@ public class TimeRoutedAlias implements RoutedAlias {
public static final String ROUTER_START = ROUTER_PREFIX + "start";
public static final String ROUTER_INTERVAL = ROUTER_PREFIX + "interval";
public static final String ROUTER_MAX_FUTURE = ROUTER_PREFIX + "maxFutureMs";
+ public static final String ROUTER_AUTO_DELETE_AGE = ROUTER_PREFIX + "autoDeleteAge";
public static final String ROUTER_PREEMPTIVE_CREATE_MATH = ROUTER_PREFIX + "preemptiveCreateMath";
// plus TZ and NAME
/**
* Parameters required for creating a routed alias
*/
- public static final List<String> REQUIRED_ROUTER_PARAMS = Collections.unmodifiableList(Arrays.asList(
+ public static final Set<String> REQUIRED_ROUTER_PARAMS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
CommonParams.NAME,
ROUTER_TYPE_NAME,
ROUTER_FIELD,
ROUTER_START,
- ROUTER_INTERVAL));
+ ROUTER_INTERVAL)));
/**
* Optional parameters for creating a routed alias excluding parameters for collection creation.
*/
//TODO lets find a way to remove this as it's harder to maintain than required list
- public static final List<String> OPTIONAL_ROUTER_PARAMS = Collections.unmodifiableList(Arrays.asList(
+ public static final Set<String> OPTIONAL_ROUTER_PARAMS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
ROUTER_MAX_FUTURE,
ROUTER_AUTO_DELETE_AGE,
ROUTER_PREEMPTIVE_CREATE_MATH,
- TZ)); // kinda special
+ TZ))); // kinda special
- static Predicate<String> PARAM_IS_PROP =
- key -> key.equals(TZ) ||
- (key.startsWith(ROUTER_PREFIX) && !key.equals(ROUTER_START)) || //TODO reconsider START special case
- key.startsWith(CREATE_COLLECTION_PREFIX);
// This format must be compatible with collection name limitations
private static final DateTimeFormatter DATE_TIME_FORMATTER = new DateTimeFormatterBuilder()
@@ -122,58 +121,6 @@ public class TimeRoutedAlias implements RoutedAlias {
.parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
.toFormatter(Locale.ROOT).withZone(ZoneOffset.UTC); // deliberate -- collection names disregard TZ
- @Override
- public String computeInitialCollectionName(String dateStr) {
- return formatCollectionNameFromInstant(aliasName, parseStringAsInstant(dateStr, timeZone));
- }
-
- public static Instant parseInstantFromCollectionName(String aliasName, String collection) {
- final String dateTimePart = collection.substring(aliasName.length() + 1);
- return DATE_TIME_FORMATTER.parse(dateTimePart, Instant::from);
- }
-
- public static String formatCollectionNameFromInstant(String aliasName, Instant timestamp) {
- String nextCollName = DATE_TIME_FORMATTER.format(timestamp);
- for (int i = 0; i < 3; i++) { // chop off seconds, minutes, hours
- if (nextCollName.endsWith("_00")) {
- nextCollName = nextCollName.substring(0, nextCollName.length()-3);
- }
- }
- assert DATE_TIME_FORMATTER.parse(nextCollName, Instant::from).equals(timestamp);
- return aliasName + "_" + nextCollName;
- }
-
- Instant parseStringAsInstant(String str, TimeZone zone) {
- Instant start = DateMathParser.parseMath(new Date(), str, zone).toInstant();
- checkMilis(start);
- return start;
- }
-
- private void checkMilis(Instant date) {
- if (!date.truncatedTo(ChronoUnit.SECONDS).equals(date)) {
- throw new SolrException(BAD_REQUEST,
- "Date or date math for start time includes milliseconds, which is not supported. " +
- "(Hint: 'NOW' used without rounding always has this problem)");
- }
- }
-
-
- @Override
- public boolean updateParsedCollectionAliases(ZkController zkController) {
- final Aliases aliases = zkController.getZkStateReader().getAliases(); // note: might be different from last request
- if (this.parsedCollectionsAliases != aliases) {
- if (this.parsedCollectionsAliases != null) {
- log.debug("Observing possibly updated alias: {}", getAliasName());
- }
- this.parsedCollectionsDesc = parseCollections(aliases );
- this.parsedCollectionsAliases = aliases;
- return true;
- }
- return false;
- }
-
-
-
//
// Instance data and methods
//
@@ -186,11 +133,20 @@ public class TimeRoutedAlias implements RoutedAlias {
private final String preemptiveCreateMath;
private final String autoDeleteAgeMath; // ex: /DAY-30DAYS *optional*
private final TimeZone timeZone;
+ private String start;
+
+ TimeRoutedAlias(String aliasName, Map<String, String> aliasMetadata) throws SolrException {
+ // Validate we got everything we need
+ if (!aliasMetadata.keySet().containsAll(TimeRoutedAlias.REQUIRED_ROUTER_PARAMS)) {
+ throw new SolrException(BAD_REQUEST, "A time routed alias requires these params: " + TimeRoutedAlias.REQUIRED_ROUTER_PARAMS
+ + " plus some create-collection prefixed ones.");
+ }
- public TimeRoutedAlias(String aliasName, Map<String, String> aliasMetadata) {
- this.aliasName = aliasName;
this.aliasMetadata = aliasMetadata;
- final MapSolrParams params = new MapSolrParams(aliasMetadata); // for convenience
+
+ this.start = this.aliasMetadata.get(ROUTER_START);
+ this.aliasName = aliasName;
+ final MapSolrParams params = new MapSolrParams(this.aliasMetadata); // for convenience
final RequiredSolrParams required = params.required();
if (!"time".equals(required.get(ROUTER_TYPE_NAME))) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Only 'time' routed aliases is supported right now.");
@@ -204,7 +160,7 @@ public class TimeRoutedAlias implements RoutedAlias {
String pcmTmp = params.get(ROUTER_PREEMPTIVE_CREATE_MATH);
preemptiveCreateMath = pcmTmp != null ? (pcmTmp.startsWith("-") ? pcmTmp : "-" + pcmTmp) : null;
autoDeleteAgeMath = params.get(ROUTER_AUTO_DELETE_AGE); // no default
- timeZone = TimeZoneUtils.parseTimezone(aliasMetadata.get(CommonParams.TZ));
+ timeZone = TimeZoneUtils.parseTimezone(this.aliasMetadata.get(CommonParams.TZ));
// More validation:
@@ -243,6 +199,55 @@ public class TimeRoutedAlias implements RoutedAlias {
}
@Override
+ public Optional<String> computeInitialCollectionName() {
+ return Optional.of(formatCollectionNameFromInstant(aliasName, parseStringAsInstant(this.start, timeZone)));
+ }
+
+ public static Instant parseInstantFromCollectionName(String aliasName, String collection) {
+ final String dateTimePart = collection.substring(aliasName.length() + 1);
+ return DATE_TIME_FORMATTER.parse(dateTimePart, Instant::from);
+ }
+
+ public static String formatCollectionNameFromInstant(String aliasName, Instant timestamp) {
+ String nextCollName = DATE_TIME_FORMATTER.format(timestamp);
+ for (int i = 0; i < 3; i++) { // chop off seconds, minutes, hours
+ if (nextCollName.endsWith("_00")) {
+ nextCollName = nextCollName.substring(0, nextCollName.length()-3);
+ }
+ }
+ assert DATE_TIME_FORMATTER.parse(nextCollName, Instant::from).equals(timestamp);
+ return aliasName + "_" + nextCollName;
+ }
+
+ Instant parseStringAsInstant(String str, TimeZone zone) {
+ Instant start = DateMathParser.parseMath(new Date(), str, zone).toInstant();
+ checkMilis(start);
+ return start;
+ }
+
+ private void checkMilis(Instant date) {
+ if (!date.truncatedTo(ChronoUnit.SECONDS).equals(date)) {
+ throw new SolrException(BAD_REQUEST,
+ "Date or date math for start time includes milliseconds, which is not supported. " +
+ "(Hint: 'NOW' used without rounding always has this problem)");
+ }
+ }
+
+ @Override
+ public boolean updateParsedCollectionAliases(ZkController zkController) {
+ final Aliases aliases = zkController.getZkStateReader().getAliases(); // note: might be different from last request
+ if (this.parsedCollectionsAliases != aliases) {
+ if (this.parsedCollectionsAliases != null) {
+ log.debug("Observing possibly updated alias: {}", getAliasName());
+ }
+ this.parsedCollectionsDesc = parseCollections(aliases );
+ this.parsedCollectionsAliases = aliases;
+ return true;
+ }
+ return false;
+ }
+
+ @Override
public String getAliasName() {
return aliasName;
}
@@ -321,8 +326,6 @@ public class TimeRoutedAlias implements RoutedAlias {
}
}
-
-
@Override
public String createCollectionsIfRequired(AddUpdateCommand cmd) {
SolrQueryRequest req = cmd.getReq();
@@ -381,19 +384,14 @@ public class TimeRoutedAlias implements RoutedAlias {
return aliasMetadata;
}
- static RoutedAlias fromZkProps(String aliasName, ZkNodeProps aliasProps) throws SolrException {
- // Validate we got everything we need
- if (!aliasProps.getProperties().keySet().containsAll(TimeRoutedAlias.REQUIRED_ROUTER_PARAMS)) {
- throw new SolrException(BAD_REQUEST, "A time routed alias requires these params: " + TimeRoutedAlias.REQUIRED_ROUTER_PARAMS
- + " plus some create-collection prefixed ones.");
- }
-
- Map<String, String> aliasProperties = new LinkedHashMap<>();
- aliasProps.getProperties().entrySet().stream()
- .filter(entry -> TimeRoutedAlias.PARAM_IS_PROP.test(entry.getKey()))
- .forEach(entry -> aliasProperties.put(entry.getKey(), (String) entry.getValue())); // way easier than .collect
+ @Override
+ public Set<String> getRequiredParams() {
+ return REQUIRED_ROUTER_PARAMS;
+ }
- return new TimeRoutedAlias(aliasName, aliasProperties); // validates as well
+ @Override
+ public Set<String> getOptionalParams() {
+ return OPTIONAL_ROUTER_PARAMS;
}
/**
@@ -470,8 +468,6 @@ public class TimeRoutedAlias implements RoutedAlias {
}
}
-
-
private void preemptiveAsync(Runnable r, SolrCore core) {
preemptiveCreateOnceAlready = true;
core.runAsync(r);
@@ -558,6 +554,4 @@ public class TimeRoutedAlias implements RoutedAlias {
ASYNC_PREEMPTIVE,
SYNCHRONOUS
}
-
-
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 725d2bd..3f39808 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@@ -52,6 +53,7 @@ import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;
import org.apache.solr.cloud.ZkController.NotInClusterStateException;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkShardTerms;
+import org.apache.solr.cloud.api.collections.RoutedAlias;
import org.apache.solr.cloud.overseer.SliceMutator;
import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.Rule;
@@ -115,9 +117,7 @@ import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHan
import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.REQUESTID;
import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.SHARDS_PROP;
import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.SHARD_UNIQUE;
-import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.CREATE_COLLECTION_PREFIX;
-import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.OPTIONAL_ROUTER_PARAMS;
-import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.REQUIRED_ROUTER_PARAMS;
+import static org.apache.solr.cloud.api.collections.RoutedAlias.CREATE_COLLECTION_PREFIX;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
import static org.apache.solr.common.cloud.DocCollection.RULE;
@@ -551,20 +551,39 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
String alias = req.getParams().get(NAME);
SolrIdentifierValidator.validateAliasName(alias);
String collections = req.getParams().get("collections");
- Map<String, Object> result = copy(req.getParams(), null, REQUIRED_ROUTER_PARAMS);
- copy(req.getParams(), result, OPTIONAL_ROUTER_PARAMS);
+ RoutedAlias routedAlias = null;
+ Exception ex = null;
+ try {
+ // note that RA specific validation occurs here.
+ routedAlias = RoutedAlias.fromProps(alias, req.getParams().toMap(new HashMap<>()));
+ } catch (SolrException e) {
+ // we'll throw this later if we are in fact creating a routed alias.
+ ex = e;
+ }
if (collections != null) {
- if (result.size() > 1) { // (NAME should be there, and if it's not we will fail below)
+ if (routedAlias != null) {
throw new SolrException(BAD_REQUEST, "Collections cannot be specified when creating a time routed alias.");
+ } else {
+ //////////////////////////////////////
+ // Regular alias creation indicated //
+ //////////////////////////////////////
+ return copy(req.getParams().required(), null, NAME, "collections");
}
- // regular alias creation...
- return copy(req.getParams().required(), null, NAME, "collections");
}
- // Ok so we are creating a time routed alias from here
+ /////////////////////////////////////////////////
+ // We are creating a routed alias from here on //
+ /////////////////////////////////////////////////
+
+ // If our prior creation attempt had issues expose them now.
+ if (ex != null) {
+ throw ex;
+ }
+
+ // Now filter out just the parameters we care about from the request
+ Map<String, Object> result = copy(req.getParams(), null, routedAlias.getRequiredParams());
+ copy(req.getParams(), result, routedAlias.getOptionalParams());
- // for validation....
- copy(req.getParams().required(), null, REQUIRED_ROUTER_PARAMS);
ModifiableSolrParams createCollParams = new ModifiableSolrParams(); // without prefix
// add to result params that start with "create-collection.".
diff --git a/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
index 5929e94..ab83e22 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
@@ -25,9 +25,7 @@ import java.util.Map;
import java.util.stream.Collectors;
import org.apache.solr.cloud.ZkController;
-import org.apache.solr.cloud.api.collections.CategoryRoutedAlias;
import org.apache.solr.cloud.api.collections.RoutedAlias;
-import org.apache.solr.cloud.api.collections.TimeRoutedAlias;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.Aliases;
@@ -95,23 +93,7 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor {
return next;
} else {
try {
- final Map<String, String> aliasProperties = getAliasProps(req, aliasName);
- String routerType = aliasProperties.get("router.name");
- RoutedAlias alias;
- switch (routerType) {
- case "time": {
- log.debug("Time Routed Alias detected for {}", aliasName );
- alias = new TimeRoutedAlias(aliasName, aliasProperties);
- break;
- }
- case "category":{
- log.debug("Category Routed Alias detected for {}", aliasName );
- alias = new CategoryRoutedAlias(aliasName, aliasProperties);
- break;
- }
- default:
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown router type " + routerType);
- }
+ RoutedAlias alias = RoutedAlias.fromProps(aliasName, getAliasProps(req, aliasName));
return new RoutedAliasUpdateProcessor(req, next, aliasDistribPhase, alias);
} catch (Exception e) { // ensure we throw SERVER_ERROR not BAD_REQUEST at this stage
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Routed alias has invalid properties: " + e, e);
diff --git a/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java b/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java
index 2e18d18..85a755a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CreateRoutedAliasTest.java
@@ -282,7 +282,7 @@ public class CreateRoutedAliasTest extends SolrCloudTestCase {
"&router.interval=%2B30MINUTE" +
"&create-collection.collection.configName=_default" +
"&create-collection.numShards=1");
- assertFailure(get, "Only 'time' routed aliases is supported right now");
+ assertFailure(get, " is not in supported types, ");
}
@Test
[lucene-solr] 04/04: SOLR-13149 First draft of Cagegory Routed
Alias class. Not really useful or testable until I implement
MaintanCategoryRoutedAlias however
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gus pushed a commit to branch solr-13131
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 1b73b93b42bfbf20c12160a6e7b52e13b754d78a
Author: Gus Heck <gu...@apache.org>
AuthorDate: Mon Jan 28 21:29:26 2019 -0500
SOLR-13149 First draft of Cagegory Routed Alias class. Not really
useful or testable until I implement MaintanCategoryRoutedAlias however
---
.../cloud/api/collections/CategoryRoutedAlias.java | 158 +++++++++++++++++++--
.../MaintainCategoryRoutedAliasCmd.java | 30 ++++
.../solr/cloud/api/collections/RoutedAlias.java | 10 +-
.../cloud/api/collections/TimeRoutedAlias.java | 6 +-
4 files changed, 186 insertions(+), 18 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
index 5a352a6..a20cf63 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
@@ -17,28 +17,74 @@
package org.apache.solr.cloud.api.collections;
-import java.time.Instant;
+import java.lang.invoke.MethodHandles;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.solr.cloud.ZkController;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.update.AddUpdateCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class CategoryRoutedAlias implements RoutedAlias {
+public class CategoryRoutedAlias implements RoutedAlias<String> {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final String COLLECTION_INFIX = "__CRA__";
+
+ /**
+ * Parameters required for creating a category routed alias
+ */
+ public static final Set<String> REQUIRED_ROUTER_PARAMS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+ CommonParams.NAME,
+ ROUTER_TYPE_NAME,
+ ROUTER_FIELD)));
+
+ public static final String ROUTER_MAX_CARDINALITY = "router.maxCardinality";
+ public static final String ROUTER_MUST_MATCH = "router.mustMatch";
+
+ /**
+ * Optional parameters for creating a category routed alias excluding parameters for collection creation.
+ */
+ public static final Set<String> OPTIONAL_ROUTER_PARAMS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+ ROUTER_MAX_CARDINALITY,
+ ROUTER_MUST_MATCH)));
+
+ private List<String> collectionNames; // List of collections currently in the CRA
+ private Aliases parsedCollectionsAliases; // a cached reference to the source of what we parse into parsedCollectionsDesc
private final String aliasName;
- private final Map<String, String> aliasProperties;
+ private final Map<String, String> aliasMetadata;
CategoryRoutedAlias(String aliasName, Map<String, String> aliasMetadata) {
this.aliasName = aliasName;
- this.aliasProperties = aliasMetadata;
+ this.aliasMetadata = aliasMetadata;
}
@Override
public boolean updateParsedCollectionAliases(ZkController zkController) {
+ final Aliases aliases = zkController.getZkStateReader().getAliases(); // note: might be different from last request
+ if (this.parsedCollectionsAliases != aliases) {
+ if (this.parsedCollectionsAliases != null) {
+ log.debug("Observing possibly updated alias: {}", getAliasName());
+ }
+ // slightly inefficient, but not easy to make changes to the return value of parseCollections
+ this.collectionNames = parseCollections(aliases).stream().map(Map.Entry::getValue).collect(Collectors.toList());
+ this.parsedCollectionsAliases = aliases;
+ return true;
+ }
return false;
}
@@ -48,37 +94,125 @@ public class CategoryRoutedAlias implements RoutedAlias {
}
@Override
- public List<Map.Entry<Instant, String>> parseCollections(Aliases aliases) {
- return null;
+ public String getRouteField() {
+ return aliasMetadata.get(ROUTER_FIELD);
}
@Override
- public void validateRouteValue(AddUpdateCommand cmd) {
+ public List<Map.Entry<String, String>> parseCollections(Aliases aliases) {
+ final List<String> collections = aliases.getCollectionAliasListMap().get(aliasName);
+ if (collections == null) {
+ throw RoutedAlias.newAliasMustExistException(getAliasName());
+ }
+ List<Map.Entry<String,String>> result = new ArrayList<>(collections.size());
+ for (String collection : collections) {
+ String collCategory = parseCategoryFromCollectionName(aliasName, collection);
+ result.add(new AbstractMap.SimpleImmutableEntry<>(collCategory, collection));
+ }
+ // TODO Think about this... is order needed? if so perhaps better if insertion maintains order?
+ result.sort((e1, e2) -> e2.getKey().compareTo(e1.getKey())); // reverse sort by key
+ // note that this is also sorted by value since the value corresponds to the key plus a the alias name which
+ // is constant within a given alias.
+ return result;
+ }
+
+ /**
+ * Pattern for Category Routed Alias is aliasName__CRA__datadrivincategory. The __CRA__ infix is to
+ * reduce the chance of inadvertently duplicating (or worse yet, adopting) other collections
+ * that are not supposed to be included in the alias. With Time routed aliases the timestamp in
+ * the collection name was sufficiently unique, but given that aliasName could be anything and
+ * 2 part collection names of the form foo_bar are probably common in the wild, the infix seems
+ * necessary.
+ *
+ */
+ private String parseCategoryFromCollectionName(String aliasName, String collection) {
+ String prefix = aliasName + COLLECTION_INFIX;
+ if (!collection.startsWith(prefix)) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"Category Routed Alias collection names " +
+ "must start with the name of the alias plus " + COLLECTION_INFIX);
+ }
+ return collection.substring(prefix.length(),collection.length());
+ }
+
+ @Override
+ public void validateRouteValue(AddUpdateCommand cmd) throws SolrException {
+ //Mostly this will be filled out by SOLR-13150 and SOLR-13151
+ }
+
+ /**
+ * Calculate a safe collection name from a data value. Any non-word character is
+ * replace with an underscore
+ *
+ * @param dataValue a value from the route field for a particular document
+ * @return the suffix value for it's corresponding collection name.
+ */
+ private String safeKeyValue(String dataValue) {
+ return dataValue.trim().replaceAll("\\W", "_");
+ }
+
+ private String buildCollectionNameFromValue(String value) {
+ return aliasName + COLLECTION_INFIX + safeKeyValue(value);
}
@Override
public String createCollectionsIfRequired(AddUpdateCommand cmd) {
- return null;
+ SolrQueryRequest req = cmd.getReq();
+ SolrCore core = req.getCore();
+ CoreContainer coreContainer = core.getCoreContainer();
+ CollectionsHandler collectionsHandler = coreContainer.getCollectionsHandler();
+ String dataValue = String.valueOf(cmd.getSolrInputDocument().getFieldValue(getRouteField()));
+
+ String candidateCollectionName = buildCollectionNameFromValue(dataValue);
+
+ try {
+ // Note: CRA's have no way to predict values that determine collection so preemptive async creation
+ // is not possible. We have no choice but to block and wait (to do otherwise would imperil the overseer).
+ do {
+ if (this.collectionNames.contains(candidateCollectionName)) {
+ return candidateCollectionName;
+ } else {
+ // this could time out in which case we simply let it throw an error
+ MaintainCategoryRoutedAliasCmd.remoteInvoke(collectionsHandler, getAliasName(), candidateCollectionName);
+ // It's possible no collection was created because of a race and that's okay... we'll retry.
+
+ // Ensure our view of the aliases has updated. If we didn't do this, our zkStateReader might
+ // not yet know about the new alias (thus won't see the newly added collection to it), and we might think
+ // we failed.
+ collectionsHandler.getCoreContainer().getZkController().getZkStateReader().aliasesManager.update();
+
+ // we should see some sort of update to our aliases
+ if (!updateParsedCollectionAliases(coreContainer.getZkController())) { // thus we didn't make progress...
+ // this is not expected, even in known failure cases, but we check just in case
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "We need to create a new category routed collection but for unknown reasons were unable to do so.");
+ }
+ }
+ } while (true);
+ } catch (SolrException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
}
@Override
public Optional<String> computeInitialCollectionName() {
- return null;
+ return Optional.empty();
}
@Override
public Map<String, String> getAliasMetadata() {
- return aliasProperties;
+ return aliasMetadata;
}
@Override
public Set<String> getRequiredParams() {
- return new HashSet<>();
+ return REQUIRED_ROUTER_PARAMS;
}
@Override
public Set<String> getOptionalParams() {
- return new HashSet<>();
+ return OPTIONAL_ROUTER_PARAMS;
}
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainCategoryRoutedAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainCategoryRoutedAliasCmd.java
index 59274d6..93a832b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainCategoryRoutedAliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainCategoryRoutedAliasCmd.java
@@ -17,11 +17,41 @@
package org.apache.solr.cloud.api.collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.cloud.Overseer;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.admin.CollectionsHandler;
public class MaintainCategoryRoutedAliasCmd extends AliasCmd {
+
+ public static final String IF_CATEGORY_COLLECTION_NOT_FOUND = "ifCategoryCollectionNotFound";
+
+ /**
+ * Invokes this command from the client. If there's a problem it will throw an exception.
+ * Please note that is important to never add async to this invocation. This method must
+ * block (up to the standard OCP timeout) to prevent large batches of add's from sending a message
+ * to the overseer for every document added in RoutedAliasUpdateProcessor.
+ */
+ public static NamedList remoteInvoke(CollectionsHandler collHandler, String aliasName, String categoryCollection)
+ throws Exception {
+ final String operation = CollectionParams.CollectionAction.MAINTAINCATEGORYROUTEDALIAS.toLower();
+ Map<String, Object> msg = new HashMap<>();
+ msg.put(Overseer.QUEUE_OPERATION, operation);
+ msg.put(CollectionParams.NAME, aliasName);
+ msg.put(IF_CATEGORY_COLLECTION_NOT_FOUND, categoryCollection);
+ final SolrResponse rsp = collHandler.sendToOCPQueue(new ZkNodeProps(msg));
+ if (rsp.getException() != null) {
+ throw rsp.getException();
+ }
+ return rsp.getResponse();
+ }
+
@Override
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
//todo
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
index 5e7df68..c083ba2 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
@@ -17,7 +17,6 @@
package org.apache.solr.cloud.api.collections;
-import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
@@ -34,7 +33,7 @@ import org.apache.solr.update.AddUpdateCommand;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
-public interface RoutedAlias {
+public interface RoutedAlias<K> {
/**
* Types supported. Every entry here must have a case in the switch statement in {@link #fromProps(String, Map)}
@@ -117,18 +116,20 @@ public interface RoutedAlias {
*/
String getAliasName();
+ String getRouteField();
+
/**
* Parses the elements of the collection list. Result is returned them in sorted order (desc) if there
* is a natural order for this type of routed alias
*/
- List<Map.Entry<Instant, String>> parseCollections(Aliases aliases);
+ List<Map.Entry<K, String>> parseCollections(Aliases aliases);
/**
* Check that the value we will be routing on is legal for this type of routed alias.
*
* @param cmd the command containing the document
*/
- void validateRouteValue(AddUpdateCommand cmd);
+ void validateRouteValue(AddUpdateCommand cmd) throws SolrException;
/**
* Create any required collections and return the name of the collection to which the current document should be sent.
@@ -147,4 +148,5 @@ public interface RoutedAlias {
Set<String> getRequiredParams();
Set<String> getOptionalParams();
+
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
index afc9ce4..e48d406 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
@@ -71,7 +71,7 @@ import static org.apache.solr.common.params.CommonParams.TZ;
* @see MaintainTimeRoutedAliasCmd
* @see RoutedAliasUpdateProcessor
*/
-public class TimeRoutedAlias implements RoutedAlias {
+public class TimeRoutedAlias implements RoutedAlias<Instant> {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
// This class is created once per request and the overseer methods prevent duplicate create requests
@@ -252,6 +252,7 @@ public class TimeRoutedAlias implements RoutedAlias {
return aliasName;
}
+ @Override
public String getRouteField() {
return routeField;
}
@@ -314,7 +315,7 @@ public class TimeRoutedAlias implements RoutedAlias {
}
@Override
- public void validateRouteValue(AddUpdateCommand cmd) {
+ public void validateRouteValue(AddUpdateCommand cmd) throws SolrException {
final Instant docTimestamp =
parseRouteKey(cmd.getSolrInputDocument().getFieldValue(getRouteField()));
@@ -554,4 +555,5 @@ public class TimeRoutedAlias implements RoutedAlias {
ASYNC_PREEMPTIVE,
SYNCHRONOUS
}
+
}
[lucene-solr] 02/04: SOLR-13148: remove hard coded pointers to
TimeRoutedAlias
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gus pushed a commit to branch solr-13131
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 84fda4d88bd3a434a2a0c0d4dd7c8507684d34f6
Author: Moshe <mo...@mail.com>
AuthorDate: Mon Jan 28 02:41:40 2019 +0200
SOLR-13148: remove hard coded pointers to TimeRoutedAlias
---
.../solr/cloud/api/collections/AliasCmd.java | 2 +-
.../cloud/api/collections/CategoryRoutedAlias.java | 6 ++++-
.../solr/cloud/api/collections/CreateAliasCmd.java | 30 +++++++++++++---------
.../solr/cloud/api/collections/RoutedAlias.java | 13 ++++++++++
.../cloud/api/collections/TimeRoutedAlias.java | 25 ++++++++++++++++++
5 files changed, 62 insertions(+), 14 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
index 3bc1d0b..c653569 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
@@ -31,7 +31,7 @@ import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.request.LocalSolrQueryRequest;
import static org.apache.solr.cloud.api.collections.RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP;
-import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.CREATE_COLLECTION_PREFIX;
+import static org.apache.solr.cloud.api.collections.RoutedAlias.CREATE_COLLECTION_PREFIX;
import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;
import static org.apache.solr.common.params.CommonParams.NAME;
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
index 3ac09c6..30b0747 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CategoryRoutedAlias.java
@@ -20,7 +20,6 @@ package org.apache.solr.cloud.api.collections;
import java.time.Instant;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.cloud.Aliases;
@@ -65,4 +64,9 @@ public class CategoryRoutedAlias implements RoutedAlias {
public String computeInitialCollectionName(String collection) {
return null;
}
+
+ @Override
+ public Map<String, String> getAliasMetadata() {
+ return aliasProperties;
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java
index dd8edc1..182eda3 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateAliasCmd.java
@@ -19,11 +19,11 @@ package org.apache.solr.cloud.api.collections;
import java.util.Collections;
import java.util.HashSet;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.function.BiFunction;
import java.util.stream.Collectors;
import org.apache.solr.common.SolrException;
@@ -98,34 +98,40 @@ public class CreateAliasCmd extends AliasCmd {
private void callCreateRoutedAlias(ZkNodeProps message, String aliasName, ZkStateReader zkStateReader, ClusterState state) throws Exception {
// Validate we got everything we need
- if (!message.getProperties().keySet().containsAll(TimeRoutedAlias.REQUIRED_ROUTER_PARAMS)) {
- throw new SolrException(BAD_REQUEST, "A routed alias requires these params: " + TimeRoutedAlias.REQUIRED_ROUTER_PARAMS
+ if (!message.getProperties().keySet().containsAll(RoutedAlias.MINIMAL_REQUIRED_PARAMS)) {
+ throw new SolrException(BAD_REQUEST, "A routed alias requires these params: " + RoutedAlias.MINIMAL_REQUIRED_PARAMS
+ " plus some create-collection prefixed ones.");
}
- Map<String, String> aliasProperties = new LinkedHashMap<>();
- message.getProperties().entrySet().stream()
- .filter(entry -> TimeRoutedAlias.PARAM_IS_PROP.test(entry.getKey()))
- .forEach(entry -> aliasProperties.put(entry.getKey(), (String) entry.getValue())); // way easier than .collect
+ String aliasType = message.getStr(RoutedAlias.ROUTER_TYPE_NAME);
- RoutedAlias timeRoutedAlias = new TimeRoutedAlias(aliasName, aliasProperties); // validates as well
+ BiFunction<String, ZkNodeProps, RoutedAlias> aliasConstructor = RoutedAlias.constructorFactory.get(aliasType);
+
+ if (aliasConstructor == null) {
+ throw new SolrException(BAD_REQUEST, "Router name: " + aliasType + " is not in supported types, "
+ + String.join(", ", RoutedAlias.constructorFactory.keySet()));
+ }
+
+
+
+ RoutedAlias routedAlias = aliasConstructor.apply(aliasName, message);
String start = message.getStr(RoutedAlias.ROUTER_START);
if (start != null) {
- String initialCollectionName = timeRoutedAlias.computeInitialCollectionName(start);
+ String initialCollectionName = routedAlias.computeInitialCollectionName(start);
if (initialCollectionName != null) {
- ensureCollection(aliasName, zkStateReader, state, aliasProperties, initialCollectionName);
+ ensureCollection(aliasName, zkStateReader, state, routedAlias.getAliasMetadata(), initialCollectionName);
// Create/update the alias
zkStateReader.aliasesManager.applyModificationAndExportToZk(aliases -> aliases
.cloneWithCollectionAlias(aliasName, initialCollectionName)
- .cloneWithCollectionAliasProperties(aliasName, aliasProperties));
+ .cloneWithCollectionAliasProperties(aliasName, routedAlias.getAliasMetadata()));
return;
}
}
// Create/update the alias
zkStateReader.aliasesManager.applyModificationAndExportToZk(aliases -> aliases
- .cloneWithCollectionAliasProperties(aliasName, aliasProperties));
+ .cloneWithCollectionAliasProperties(aliasName, routedAlias.getAliasMetadata()));
}
private void ensureCollection(String aliasName, ZkStateReader zkStateReader, ClusterState state, Map<String, String> aliasProperties, String initialCollectionName) throws Exception {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
index b004d79..1a607ba 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java
@@ -21,15 +21,22 @@ import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.BiFunction;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
+import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.update.AddUpdateCommand;
public interface RoutedAlias {
+ Map<String, BiFunction<String, ZkNodeProps, RoutedAlias>> constructorFactory = ImmutableMap.<String, BiFunction<String, ZkNodeProps, RoutedAlias>>builder()
+ .put("time", TimeRoutedAlias::fromZkProps)
+ .build();
+
String ROUTER_PREFIX = "router.";
String ROUTER_TYPE_NAME = ROUTER_PREFIX + "name";
String ROUTER_FIELD = ROUTER_PREFIX + "field";
@@ -81,4 +88,10 @@ public interface RoutedAlias {
* newly created collection
*/
String createCollectionsIfRequired( AddUpdateCommand cmd);
+
+ /**
+ *
+ * @return get alias related metadata
+ */
+ Map<String, String> getAliasMetadata();
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
index 007e2ce..c8b0ebb 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -41,6 +42,7 @@ import com.google.common.base.Objects;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
+import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.RequiredSolrParams;
@@ -177,6 +179,7 @@ public class TimeRoutedAlias implements RoutedAlias {
//
private final String aliasName;
+ private final Map<String, String> aliasMetadata;
private final String routeField;
private final String intervalMath; // ex: +1DAY
private final long maxFutureMs;
@@ -186,6 +189,7 @@ public class TimeRoutedAlias implements RoutedAlias {
public TimeRoutedAlias(String aliasName, Map<String, String> aliasMetadata) {
this.aliasName = aliasName;
+ this.aliasMetadata = aliasMetadata;
final MapSolrParams params = new MapSolrParams(aliasMetadata); // for convenience
final RequiredSolrParams required = params.required();
if (!"time".equals(required.get(ROUTER_TYPE_NAME))) {
@@ -371,6 +375,27 @@ public class TimeRoutedAlias implements RoutedAlias {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
+
+ @Override
+ public Map<String, String> getAliasMetadata() {
+ return aliasMetadata;
+ }
+
+ static RoutedAlias fromZkProps(String aliasName, ZkNodeProps aliasProps) throws SolrException {
+ // Validate we got everything we need
+ if (!aliasProps.getProperties().keySet().containsAll(TimeRoutedAlias.REQUIRED_ROUTER_PARAMS)) {
+ throw new SolrException(BAD_REQUEST, "A time routed alias requires these params: " + TimeRoutedAlias.REQUIRED_ROUTER_PARAMS
+ + " plus some create-collection prefixed ones.");
+ }
+
+ Map<String, String> aliasProperties = new LinkedHashMap<>();
+ aliasProps.getProperties().entrySet().stream()
+ .filter(entry -> TimeRoutedAlias.PARAM_IS_PROP.test(entry.getKey()))
+ .forEach(entry -> aliasProperties.put(entry.getKey(), (String) entry.getValue())); // way easier than .collect
+
+ return new TimeRoutedAlias(aliasName, aliasProperties); // validates as well
+ }
+
/**
* Create as many collections as required. This method loops to allow for the possibility that the docTimestamp
* requires more than one collection to be created. Since multiple threads may be invoking maintain on separate