You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ge...@apache.org on 2021/12/12 11:35:37 UTC
[solr] branch main updated: SOLR-15745: Convert create-core v2 API to annotations (#450)
This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 66c76dd SOLR-15745: Convert create-core v2 API to annotations (#450)
66c76dd is described below
commit 66c76dd50ffd3ebaf7473fc62e279803678c227d
Author: Jason Gerlowski <ge...@apache.org>
AuthorDate: Sun Dec 12 06:35:30 2021 -0500
SOLR-15745: Convert create-core v2 API to annotations (#450)
---
.../solr/handler/admin/CoreAdminHandler.java | 2 +
.../solr/handler/admin/api/CreateCoreAPI.java | 78 +++++++++++
.../org/apache/solr/handler/api/V2ApiUtils.java | 7 +
.../solr/handler/admin/V2CoresAPIMappingTest.java | 150 +++++++++++++++++++++
.../solr/client/solrj/request/CoreApiMapping.java | 2 -
.../solrj/request/beans/CreateCorePayload.java | 83 ++++++++++++
.../src/resources/apispec/cores.Commands.json | 85 ------------
.../apache/solr/common/util/JsonValidatorTest.java | 1 -
8 files changed, 320 insertions(+), 88 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
index 5950b11..d045a9e 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
@@ -36,6 +36,7 @@ import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.handler.admin.api.CreateCoreAPI;
import org.apache.solr.handler.admin.api.InvokeClassAPI;
import org.apache.solr.handler.admin.api.OverseerOperationAPI;
import org.apache.solr.handler.admin.api.RejoinLeaderElectionAPI;
@@ -410,6 +411,7 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
public Collection<Api> getApis() {
final List<Api> apis = Lists.newArrayList(coreAdminHandlerApi.getApis());
// Only some core-admin APIs use the v2 AnnotatedApi framework
+ apis.addAll(AnnotatedApi.getApis(new CreateCoreAPI(this)));
apis.addAll(AnnotatedApi.getApis(new InvokeClassAPI(this)));
apis.addAll(AnnotatedApi.getApis(new RejoinLeaderElectionAPI(this)));
apis.addAll(AnnotatedApi.getApis(new OverseerOperationAPI(this)));
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCoreAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCoreAPI.java
new file mode 100644
index 0000000..948c9e3
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCoreAPI.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.admin.api;
+
+import org.apache.solr.api.Command;
+import org.apache.solr.api.EndPoint;
+import org.apache.solr.api.PayloadObj;
+import org.apache.solr.client.solrj.request.beans.CreateCorePayload;
+import org.apache.solr.handler.admin.CoreAdminHandler;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
+import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX;
+import static org.apache.solr.common.params.CoreAdminParams.ACTION;
+import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.CREATE;
+import static org.apache.solr.handler.ClusterAPI.wrapParams;
+import static org.apache.solr.handler.api.V2ApiUtils.flattenMapWithPrefix;
+import static org.apache.solr.handler.api.V2ApiUtils.flattenToCommaDelimitedString;
+import static org.apache.solr.security.PermissionNameProvider.Name.CORE_EDIT_PERM;
+
+/**
+ * V2 API for creating a new core on the receiving node.
+ *
+ * This API (POST /v2/cores {'create': {...}}) is analogous to the v1 /admin/cores?action=CREATE command.
+ *
+ * @see CreateCorePayload
+ */
+@EndPoint(path = {"/cores"},
+ method = POST,
+ permission = CORE_EDIT_PERM)
+public class CreateCoreAPI {
+ public static final String V2_CREATE_CORE_CMD = "create";
+
+ private final CoreAdminHandler coreAdminHandler;
+
+ public CreateCoreAPI(CoreAdminHandler coreAdminHandler) {
+ this.coreAdminHandler = coreAdminHandler;
+ }
+
+ @Command(name = V2_CREATE_CORE_CMD)
+ public void createCore(PayloadObj<CreateCorePayload> obj) throws Exception {
+ final CreateCorePayload v2Body = obj.get();
+ final Map<String, Object> v1Params = v2Body.toMap(new HashMap<>());
+ v1Params.put(ACTION, CREATE.name().toLowerCase(Locale.ROOT));
+
+ if (v2Body.isTransient != null) {
+ v1Params.put("transient", v1Params.remove("isTransient"));
+ }
+ if (v2Body.properties != null) {
+ v1Params.remove("properties");
+ flattenMapWithPrefix(v2Body.properties, v1Params, PROPERTY_PREFIX);
+ }
+ if (v2Body.roles != null && !v2Body.roles.isEmpty()) {
+ v1Params.remove("roles"); // Not strictly needed, since the method below would overwrite it.
+ flattenToCommaDelimitedString(v1Params, v2Body.roles, "roles");
+ }
+
+ coreAdminHandler.handleRequestBody(wrapParams(obj.getRequest(), v1Params), obj.getResponse());
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java
index 9827c59..7de1ba8 100644
--- a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java
+++ b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java
@@ -17,6 +17,7 @@
package org.apache.solr.handler.api;
+import java.util.List;
import java.util.Map;
/**
@@ -33,4 +34,10 @@ public class V2ApiUtils {
toFlatten.forEach((k, v) -> destination.put(additionalPrefix + k, v));
}
+
+ public static void flattenToCommaDelimitedString(Map<String, Object> destination, List<String> toFlatten, String newKey) {
+ final String flattenedStr = String.join(",", toFlatten);
+ destination.put(newKey, flattenedStr);
+ }
+
}
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/V2CoresAPIMappingTest.java b/solr/core/src/test/org/apache/solr/handler/admin/V2CoresAPIMappingTest.java
new file mode 100644
index 0000000..6d20869
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/admin/V2CoresAPIMappingTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.admin;
+
+import com.google.common.collect.Maps;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.api.Api;
+import org.apache.solr.api.ApiBag;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.CommandOperation;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.handler.admin.api.CreateCoreAPI;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.apache.solr.common.params.CollectionAdminParams.NUM_SHARDS;
+import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
+import static org.apache.solr.common.params.CommonParams.ACTION;
+import static org.apache.solr.common.params.CoreAdminParams.*;
+import static org.apache.solr.common.params.CoreAdminParams.CoreAdminAction.CREATE;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+/**
+ * Unit tests for the /cores APIs.
+ *
+ * Note that the V2 requests made by these tests are not necessarily semantically valid. They shouldn't be taken as
+ * examples. In several instances, mutually exclusive JSON parameters are provided. This is done to exercise conversion
+ * of all parameters, even if particular combinations are never expected in the same request.
+ */
+public class V2CoresAPIMappingTest extends SolrTestCaseJ4 {
+ private ApiBag apiBag;
+ private ArgumentCaptor<SolrQueryRequest> queryRequestCaptor;
+ private CoreAdminHandler mockCoreAdminHandler;
+
+ @BeforeClass
+ public static void ensureWorkingMockito() {
+ assumeWorkingMockito();
+ }
+
+ @Before
+ public void setUpMocks() {
+ mockCoreAdminHandler = mock(CoreAdminHandler.class);
+ queryRequestCaptor = ArgumentCaptor.forClass(SolrQueryRequest.class);
+
+ apiBag = new ApiBag(false);
+ apiBag.registerObject(new CreateCoreAPI(mockCoreAdminHandler));
+ }
+
+ @Test
+ public void testCreateCoreAllProperties() throws Exception {
+ final SolrParams v1Params = captureConvertedV1Params("/cores", "POST",
+ "{'create': {" +
+ "'name': 'someCoreName', " +
+ "'instanceDir': 'someInstanceDir', " +
+ "'dataDir': 'someDataDir', " +
+ "'ulogDir': 'someUpdateLogDirectory', " +
+ "'schema': 'some-schema-file-name', " +
+ "'config': 'some-config-file-name', " +
+ "'configSet': 'someConfigSetName', " +
+ "'loadOnStartup': true, " +
+ "'isTransient': true, " +
+ "'shard': 'someShardName', " +
+ "'collection': 'someCollectionName', " +
+ "'replicaType': 'TLOG', " +
+ "'coreNodeName': 'someNodeName', " +
+ "'numShards': 123, " +
+ "'roles': ['role1', 'role2'], " +
+ "'properties': {'prop1': 'val1', 'prop2': 'val2'}, " +
+ "'newCollection': true, " +
+ "'async': 'requestTrackingId' " +
+ "}}");
+
+ assertEquals(CREATE.name().toLowerCase(Locale.ROOT), v1Params.get(ACTION));
+ assertEquals("someCoreName", v1Params.get(NAME));
+ assertEquals("someInstanceDir", v1Params.get(INSTANCE_DIR));
+ assertEquals("someDataDir", v1Params.get(DATA_DIR));
+ assertEquals("someUpdateLogDirectory", v1Params.get(ULOG_DIR));
+ assertEquals("some-schema-file-name", v1Params.get(SCHEMA));
+ assertEquals("some-config-file-name", v1Params.get(CONFIG));
+ assertEquals("someConfigSetName", v1Params.get(CONFIGSET));
+ assertEquals(true, v1Params.getPrimitiveBool(LOAD_ON_STARTUP));
+ assertEquals(true, v1Params.getPrimitiveBool(TRANSIENT));
+ assertEquals("someShardName", v1Params.get(SHARD));
+ assertEquals("someCollectionName", v1Params.get(COLLECTION));
+ assertEquals("TLOG", v1Params.get(REPLICA_TYPE));
+ assertEquals("someNodeName", v1Params.get(CORE_NODE_NAME));
+ assertEquals(123, v1Params.getPrimitiveInt(NUM_SHARDS));
+ assertEquals("role1,role2", v1Params.get(ROLES));
+ assertEquals("val1", v1Params.get("property.prop1"));
+ assertEquals("val2", v1Params.get("property.prop2"));
+ assertEquals(true, v1Params.getPrimitiveBool(NEW_COLLECTION));
+ assertEquals("requestTrackingId", v1Params.get(ASYNC));
+ }
+
+ private SolrParams captureConvertedV1Params(String path, String method, String v2RequestBody) throws Exception {
+ final HashMap<String, String> parts = new HashMap<>();
+ final Api api = apiBag.lookup(path, method, parts);
+ final SolrQueryResponse rsp = new SolrQueryResponse();
+ final LocalSolrQueryRequest req = new LocalSolrQueryRequest(null, Maps.newHashMap()) {
+ @Override
+ public List<CommandOperation> getCommands(boolean validateInput) {
+ if (v2RequestBody == null) return Collections.emptyList();
+ return ApiBag.getCommandOperations(new ContentStreamBase.StringStream(v2RequestBody), api.getCommandSchema(), true);
+ }
+
+ @Override
+ public Map<String, String> getPathTemplateValues() {
+ return parts;
+ }
+
+ @Override
+ public String getHttpMethod() {
+ return method;
+ }
+ };
+
+
+ api.call(req, rsp);
+ verify(mockCoreAdminHandler).handleRequestBody(queryRequestCaptor.capture(), any());
+ return queryRequestCaptor.getValue().getParams();
+ }
+}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreApiMapping.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreApiMapping.java
index a20a427..84d526f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreApiMapping.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreApiMapping.java
@@ -35,7 +35,6 @@ import static org.apache.solr.client.solrj.request.CoreApiMapping.EndPoint.*;
*/
public class CoreApiMapping {
public enum Meta implements CommandMeta {
- CREATE(CORES_COMMANDS, POST, CoreAdminAction.CREATE, "create", Collections.singletonMap("config", "configSet")),
UNLOAD(PER_CORE_COMMANDS, POST, CoreAdminAction.UNLOAD, "unload", null),
RELOAD(PER_CORE_COMMANDS, POST, CoreAdminAction.RELOAD, "reload", null),
STATUS(CORES_STATUS, GET, CoreAdminAction.STATUS, "status", null),
@@ -88,7 +87,6 @@ public class CoreApiMapping {
public enum EndPoint implements ApiMapping.V2EndPoint {
CORES_STATUS("cores.Status"),
- CORES_COMMANDS("cores.Commands"),
PER_CORE_COMMANDS("cores.core.Commands");
final String specName;
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/CreateCorePayload.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/CreateCorePayload.java
new file mode 100644
index 0000000..4a7fbc7
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/CreateCorePayload.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request.beans;
+
+import org.apache.solr.common.annotation.JsonProperty;
+import org.apache.solr.common.util.ReflectMapWriter;
+
+import java.util.List;
+import java.util.Map;
+
+public class CreateCorePayload implements ReflectMapWriter {
+ @JsonProperty(required = true)
+ public String name;
+
+ @JsonProperty
+ public String instanceDir;
+
+ @JsonProperty
+ public String dataDir;
+
+ @JsonProperty
+ public String ulogDir;
+
+ @JsonProperty
+ public String schema;
+
+ @JsonProperty
+ public String config;
+
+ @JsonProperty
+ public String configSet;
+
+ @JsonProperty
+ public Boolean loadOnStartup;
+
+ // If our JsonProperty clone was more feature-rich here we could specify the property be called 'transient', but
+ // without that support it needs to be named something else to avoid conflicting with the 'transient' keyword in Java
+ @JsonProperty
+ public Boolean isTransient;
+
+ @JsonProperty
+ public String shard;
+
+ @JsonProperty
+ public String collection;
+
+ // TODO - what type is 'roles' expected to be?
+ @JsonProperty
+ public List<String> roles;
+
+ @JsonProperty
+ public String replicaType;
+
+ @JsonProperty
+ public Map<String, Object> properties;
+
+ @JsonProperty
+ public String coreNodeName;
+
+ @JsonProperty
+ public Integer numShards;
+
+ @JsonProperty
+ public Boolean newCollection;
+
+ @JsonProperty
+ public String async;
+}
diff --git a/solr/solrj/src/resources/apispec/cores.Commands.json b/solr/solrj/src/resources/apispec/cores.Commands.json
deleted file mode 100644
index 29777de..0000000
--- a/solr/solrj/src/resources/apispec/cores.Commands.json
+++ /dev/null
@@ -1,85 +0,0 @@
-{
- "documentation": "https://lucene.apache.org/solr/guide/coreadmin-api.html",
- "description": "Actions on non-specific cores. See the /cores/{core} endpoint for actions on specific, named, cores.",
- "methods": [
- "POST"
- ],
- "url": {
- "paths": [
- "/cores"
- ]
- },
- "commands": {
- "create": {
- "type" : "object",
- "documentation": "https://lucene.apache.org/solr/guide/coreadmin-api.html#coreadmin-create",
- "description": "Creates a new core. If you are using SolrCloud, you should use the Collections API instead. While a core for a SolrCloud cluster can be created with this API, it is considered an expert-level action. The Collections API has commands for creating new shards and replicas that ensure the safety of those actions within your cluster.",
- "properties": {
- "name": {
- "type": "string",
- "description": "The core name to create. If a core with this name already exists, an error will be returned."
- },
- "instanceDir": {
- "type": "string",
- "description": "The core instance directory, where files for this core should be stored. While this parameter is not required, if it is not defined it will default to a path relative to Solr Home that includes the name you've given the new core. This location MUST EXIST prior to creating the core, and it must include a conf directory that includes solrconfig.xml and your schema, either as a schema.xml file or using the managed schema feature."
- },
- "schema": {
- "type": "string",
- "description": "Name of the schema file to use for the core. Please note that if you are using a 'managed schema' (Solr's default behavior) then any value for this property that does not match the effective managedSchemaResourceName will be read once, backed up, and converted for managed schema use. If you are using the default name (schema.xml or the managed schema name), you do not need to define the schema file name."
- },
- "dataDir": {
- "type": "string",
- "description": "Name of the data directory relative to instanceDir. This is where the index files will be stored."
- },
- "config": {
- "type": "string",
- "description": "Name of the config file (i.e., solrconfig.xml) relative to instanceDir. If you are using the default name (solrconfig.xml), you do not need to define the config file name."
- },
- "configSet": {
- "type": "string",
- "documentation": "https://lucene.apache.org/solr/guide/config-sets.html",
- "description": "The name of a config set to use. The config set must already exist. The solr.xml file defines the location of the configset base directory, and configuration files can be shared between cores by defining sub-directories. The files in the named configSet will be used for the schema and config properties instead of defining them explicitly."
- },
- "loadOnStartup": {
- "type": "boolean",
- "description": "If true, the core will be loaded on startup. Set to false to enable lazy loading, where the core will only be loaded if it is referenced or called.",
- "default": "true"
- },
- "transient": {
- "type": "boolean",
- "description": "Allows Solr to unload the core if resources are required.",
- "default": "false"
- },
- "shard": {
- "type": "string",
- "description": "In SolrCloud mode, the shard this core should belong to."
- },
- "collection": {
- "type": "string",
- "description": "The name of the collection this core belongs to."
- },
- "props": {
- "type": "object",
- "documentation": "https://solr.apache.org/guide/core-discovery.html",
- "description": "Allows adding core.properties for the collection.",
- "additionalProperties": true
- },
- "coreNodeName": {
- "type": "string",
- "description": "The replica name."
- },
- "numShards": {
- "type":"integer",
- "description":"The number of shards to create for this core."
- },
- "async": {
- "type": "string",
- "description": "Defines a request ID that can be used to track this action after it's submitted. The action will be processed asynchronously when this is defined."
- }
- },
- "required": [
- "name"
- ]
- }
- }
-}
diff --git a/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java b/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java
index 2ef8699..10cc866 100644
--- a/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java
+++ b/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java
@@ -28,7 +28,6 @@ import static org.apache.solr.common.util.ValidatingJsonMap.NOT_NULL;
public class JsonValidatorTest extends SolrTestCaseJ4 {
public void testSchema() {
- checkSchema("cores.Commands");
checkSchema("cores.core.Commands");
checkSchema("cluster.security.BasicAuth.Commands");
checkSchema("cluster.security.RuleBasedAuthorization");