You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/09/07 01:05:03 UTC
[lucene-solr] branch jira/SOLR-13722 updated: repossitory CRUD
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/SOLR-13722
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/SOLR-13722 by this push:
new f8ac22f repossitory CRUD
f8ac22f is described below
commit f8ac22f764af62e344b7c82ba09bc4274ba3a8f2
Author: noble <no...@apache.org>
AuthorDate: Sat Sep 7 11:04:44 2019 +1000
repossitory CRUD
---
.../solr/handler/admin/BaseHandlerApiSupport.java | 7 ++
.../org/apache/solr/handler/admin/ClusterAPI.java | 79 +++++++++++++++++++---
.../solr/handler/admin/CollectionHandlerApi.java | 9 ---
.../solr/handler/TestContainerReqHandler.java | 42 ++++++++++++
.../client/solrj/request/CollectionApiMapping.java | 1 +
.../apache/solr/common/cloud/ZkStateReader.java | 4 +-
.../src/resources/apispec/cluster.Commands.json | 15 ----
.../cluster.Commands.repository.properties.json | 3 +-
.../src/resources/apispec/cluster.repository.json | 27 ++++++++
.../apache/solr/common/util/JsonValidatorTest.java | 1 +
10 files changed, 151 insertions(+), 37 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java b/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java
index 1993a89..d340b91 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java
@@ -17,6 +17,7 @@
package org.apache.solr.handler.admin;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -38,6 +39,8 @@ import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.Utils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
@@ -49,6 +52,8 @@ import static org.apache.solr.common.util.StrUtils.splitSmart;
* to actions and old parameter names to new parameter names
*/
public abstract class BaseHandlerApiSupport implements ApiSupport {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
protected final Map<SolrRequest.METHOD, Map<V2EndPoint, List<ApiCommand>>> commandsMapping;
protected BaseHandlerApiSupport() {
@@ -121,8 +126,10 @@ public abstract class BaseHandlerApiSupport implements ApiSupport {
}
} catch (SolrException e) {
+ log.error("error running command", e);
throw e;
} catch (Exception e) {
+ log.error("error running command", e);
throw new SolrException(BAD_REQUEST, e); //TODO BAD_REQUEST is a wild guess; should we flip the default? fail here to investigate how this happens in tests
} finally {
req.setParams(params);
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ClusterAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/ClusterAPI.java
index 498326f..826d738 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterAPI.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterAPI.java
@@ -34,6 +34,7 @@ import org.apache.solr.client.solrj.request.CollectionApiMapping;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterProperties;
+import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.CommandOperation;
@@ -51,6 +52,7 @@ import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.SimplePostTool;
+import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,6 +63,7 @@ import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
import static org.apache.solr.client.solrj.request.CollectionApiMapping.EndPoint.CLUSTER_CMD;
import static org.apache.solr.client.solrj.request.CollectionApiMapping.EndPoint.CLUSTER_NODES;
+import static org.apache.solr.client.solrj.request.CollectionApiMapping.EndPoint.CLUSTER_REPO;
import static org.apache.solr.common.util.CommandOperation.captureErrors;
import static org.apache.solr.common.util.StrUtils.formatString;
import static org.apache.solr.core.ConfigOverlay.ZNODEVER;
@@ -130,23 +133,25 @@ class ClusterAPI {
enum Commands implements ApiCommand {
-/*
- ADD_REPO(CLUSTER_CMD, POST, "add-repository") {
+ ADD_REPO(CLUSTER_REPO, POST, "add") {
@Override
void call(ApiInfo info) throws Exception {
+ repositoryCRUD(info);
}
},
- UPDATE_REPO(CLUSTER_CMD, POST, "update-repository") {
+ UPDATE_REPO(CLUSTER_REPO, POST, "update") {
@Override
void call(ApiInfo info) throws Exception {
+ repositoryCRUD(info);
}
},
- DELETE_REPO(CLUSTER_CMD, POST, "delete-repository") {
+ DELETE_REPO(CLUSTER_REPO, POST, "delete") {
@Override
void call(ApiInfo info) throws Exception {
+ repositoryCRUD(info);
+
}
},
-*/
GET_NODES(CLUSTER_NODES, GET, null) {
@Override
@@ -318,10 +323,10 @@ class ClusterAPI {
;
- private CollectionApiMapping.CommandMeta _meta;
+ private CollectionApiMapping.CommandMeta meta;
Commands(CollectionApiMapping.V2EndPoint endPoint, SolrRequest.METHOD method, String cmdName) {
- _meta = new CollectionApiMapping.CommandMeta() {
+ meta = new CollectionApiMapping.CommandMeta() {
@Override
public String getName() {
return cmdName;
@@ -342,7 +347,7 @@ class ClusterAPI {
@Override
public CollectionApiMapping.CommandMeta meta() {
- return _meta;
+ return meta;
}
@@ -350,7 +355,7 @@ class ClusterAPI {
public void invoke(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) throws Exception {
CommandOperation op = null;
if (meta().getHttpMethod() == SolrRequest.METHOD.POST) {
- if (_meta.getName() != null) {
+ if (meta.getName() != null) {
List<CommandOperation> commands = req.getCommands(true);
if (commands == null || commands.size() != 1)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "should have exactly one command");
@@ -370,6 +375,50 @@ class ClusterAPI {
}
+ private static void repositoryCRUD(ApiInfo info) {
+ try {
+ SolrZkClient zkClient = info.coreContainer.getZkController().getZkClient();
+ Map data = Utils.getDeepCopy(Utils.getJson(zkClient, ZkStateReader.PACKAGE_REPO, true),3);
+ Map<String, Object> dataMap = null;
+ String name = null;
+ name = info.op.getCommandData() instanceof String ?
+ (String) info.op.getCommandData() :
+ info.op.getStr("name");
+
+ List<String> path = asList("repository", name);
+ boolean contains = Utils.getObjectByPath(data, false, path) != null;
+
+ if(info.op.name.equals(Commands.ADD_REPO.meta.getName())){
+ if(contains){
+ info.op.addError("repository "+ name + " already exists");
+ return;
+ }
+ dataMap = info.op.getDataMap();
+
+ } else if(info.op.name.equals(Commands.UPDATE_REPO.meta.getName())) {
+ if(!contains){
+ info.op.addError("repository "+ name + " does not exist");
+ return;
+ }
+ dataMap = info.op.getDataMap();
+
+ } else if(info.op.name.equals(Commands.DELETE_REPO.meta.getName())){
+ if(!contains){
+ info.op.addError("repository "+ name + " does not exist");
+ return;
+ }
+ }
+
+ Map delta = new LinkedHashMap();
+ Utils.setObjectByPath(delta, path, dataMap, true);
+ updateRepository(zkClient, delta);
+ } catch (SolrException se){
+ throw se;
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
static boolean addUpdatePackage(ApiInfo params) throws Exception {
if (!RuntimeLib.isEnabled()) {
params.op.addError("node not started with enable.runtime.lib=true");
@@ -380,7 +429,7 @@ class ClusterAPI {
RuntimeLib lib = new RuntimeLib(handler.handler.coreContainer);
CommandOperation op = params.op;
String name = op.getStr("name");
- ClusterProperties clusterProperties = new ClusterProperties(((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getZkClient());
+ ClusterProperties clusterProperties = new ClusterProperties(params.coreContainer.getZkController().getZkClient());
Map<String, Object> props = clusterProperties.getClusterProperties();
List<String> pathToLib = asList(CommonParams.PACKAGE, name);
Map existing = (Map) Utils.getObjectByPath(props, false, pathToLib);
@@ -490,4 +539,14 @@ class ClusterAPI {
}
}
+
+ static void updateRepository(SolrZkClient client, Map delta) throws KeeperException, InterruptedException {
+ client.atomicUpdate(ZkStateReader.PACKAGE_REPO, zkData -> {
+ if (zkData == null) return Utils.toJSON(delta);
+ Map<String, Object> zkJson = (Map<String, Object>) Utils.fromJSON(zkData);
+ boolean modified = Utils.mergeJson(zkJson, delta);
+ return modified ? Utils.toJSON(zkJson) : null;
+ });
+
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java
index ccf55a6..97f7d96 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java
@@ -73,15 +73,6 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport {
apiMapping.put(api.meta(), api );
}
-// addApi(apiMapping, Meta.GET_NODES, CollectionHandlerApi::getNodes);
-// addApi(apiMapping, Meta.POST_BLOB, CollectionHandlerApi::postBlob);
-// addApi(apiMapping, Meta.SET_CLUSTER_PROPERTY_OBJ, CollectionHandlerApi::setClusterObj);
-// addApi(apiMapping, Meta.ADD_PACKAGE, wrap(CollectionHandlerApi::addUpdatePackage));
-// addApi(apiMapping, Meta.UPDATE_PACKAGE, wrap(CollectionHandlerApi::addUpdatePackage));
-// addApi(apiMapping, Meta.DELETE_RUNTIME_LIB, wrap(CollectionHandlerApi::deletePackage));
-// addApi(apiMapping, Meta.ADD_REQ_HANDLER, wrap(CollectionHandlerApi::addRequestHandler));
-// addApi(apiMapping, Meta.DELETE_REQ_HANDLER, wrap(CollectionHandlerApi::deleteReqHandler));
-
for (Meta meta : Meta.values()) {
if (apiMapping.get(meta) == null) {
log.error("ERROR_INIT. No corresponding API implementation for : " + meta.commandName);
diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java b/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java
index b422475..ddede79 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java
@@ -52,6 +52,7 @@ import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.NavigableObject;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
@@ -815,6 +816,47 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
}
+ public void testRepoCRUD() throws Exception{
+ MiniSolrCloudCluster cluster = configureCluster(4)
+ .withJettyConfig(jetty -> jetty.enableV2(true))
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ try {
+ String payload = "{add : {name : myrepo, url: 'http://localhost/abc' , version : '1.1'}}";
+ new V2Request.Builder("/cluster/repository")
+ .withPayload(payload)
+ .withMethod(SolrRequest.METHOD.POST)
+ .build().process(cluster.getSolrClient());
+ Map repojson = Utils.getJson(cluster.getZkClient(), ZkStateReader.PACKAGE_REPO, true);
+
+ assertEquals("http://localhost/abc", Utils.getObjectByPath(repojson, true, "/repository/myrepo/url"));
+ assertEquals("1.1", Utils.getObjectByPath(repojson, true, "/repository/myrepo/version"));
+ payload = "{update : {name : myrepo, url: 'http://localhost/abc' , version : '1.2'}}";
+ new V2Request.Builder("/cluster/repository")
+ .withPayload(payload)
+ .withMethod(SolrRequest.METHOD.POST)
+ .build().process(cluster.getSolrClient());
+ repojson = Utils.getJson(cluster.getZkClient(), ZkStateReader.PACKAGE_REPO, true);
+
+ assertEquals("http://localhost/abc", Utils.getObjectByPath(repojson, true, "/repository/myrepo/url"));
+ assertEquals("1.2", Utils.getObjectByPath(repojson, true, "/repository/myrepo/version"));
+
+ payload = "{delete : myrepo}";
+ new V2Request.Builder("/cluster/repository")
+ .withPayload(payload)
+ .withMethod(SolrRequest.METHOD.POST)
+ .build().process(cluster.getSolrClient());
+ repojson = Utils.getJson(cluster.getZkClient(), ZkStateReader.PACKAGE_REPO, true);
+
+ assertNull( Utils.getObjectByPath(repojson, true, "/repository/myrepo"));
+
+
+ }finally {
+ cluster.shutdown();
+ }
+
+ }
+
private void postBlob(SolrClient client, ByteBuffer blob, String sh256) throws SolrServerException, IOException {
V2Response rsp = new V2Request.Builder("/cluster/blob")
.withMethod(SolrRequest.METHOD.POST)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java
index 97b446f..b9a6cc3 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java
@@ -342,6 +342,7 @@ public class CollectionApiMapping {
CLUSTER_BLOB("cluster.blob"),
CLUSTER_ALIASES("cluster.aliases"),
CLUSTER_CMD("cluster.Commands"),
+ CLUSTER_REPO("cluster.repository"),
CLUSTER_NODES("cluster.nodes"),
CLUSTER_CMD_STATUS("cluster.commandstatus"),
CLUSTER_CMD_STATUS_DELETE("cluster.commandstatus.delete"),
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index b9c66cf..31ff3f5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -110,6 +110,7 @@ public class ZkStateReader implements SolrCloseable {
public static final String ALIASES = "/aliases.json";
public static final String CLUSTER_STATE = "/clusterstate.json";
public static final String CLUSTER_PROPS = "/clusterprops.json";
+ public static final String PACKAGE_REPO = "/package_repo.json";
public static final String COLLECTION_PROPS_ZKNODE = "collectionprops.json";
public static final String REJOIN_AT_HEAD_PROP = "rejoinAtHead";
public static final String SOLR_SECURITY_CONF_PATH = "/security.json";
@@ -156,7 +157,6 @@ public class ZkStateReader implements SolrCloseable {
private static final int GET_LEADER_RETRY_INTERVAL_MS = 50;
private static final int GET_LEADER_RETRY_DEFAULT_TIMEOUT = Integer.parseInt(System.getProperty("zkReaderGetLeaderRetryTimeoutMs", "4000"));
- ;
public static final String LEADER_ELECT_ZKNODE = "leader_elect";
@@ -1572,7 +1572,7 @@ public class ZkStateReader implements SolrCloseable {
Stat stat = new Stat();
byte[] data = zkClient.getData(collectionPath, watcher, stat, true);
ClusterState state = ClusterState.load(stat.getVersion(), data,
- Collections.<String>emptySet(), collectionPath);
+ Collections.emptySet(), collectionPath);
ClusterState.CollectionRef collectionRef = state.getCollectionStates().get(coll);
return collectionRef == null ? null : collectionRef.get();
} catch (KeeperException.NoNodeException e) {
diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.json b/solr/solrj/src/resources/apispec/cluster.Commands.json
index 5d5ae7f..e0da898 100644
--- a/solr/solrj/src/resources/apispec/cluster.Commands.json
+++ b/solr/solrj/src/resources/apispec/cluster.Commands.json
@@ -209,21 +209,6 @@
"delete-requesthandler" : {
"description" : "delete a requesthandler",
"type": "string"
- },
- "add-repository": {
- "description": "",
- "documentation": "",
- "#include": "cluster.Commands.repository.properties"
-
- },
- "update-repository": {
- "description": "",
- "documentation": "",
- "#include": "cluster.Commands.repository.properties"
- },
- "delete-repository": {
- "description": "",
- "type": "string"
}
}
diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.repository.properties.json b/solr/solrj/src/resources/apispec/cluster.Commands.repository.properties.json
index 05b8359..9f46f59 100644
--- a/solr/solrj/src/resources/apispec/cluster.Commands.repository.properties.json
+++ b/solr/solrj/src/resources/apispec/cluster.Commands.repository.properties.json
@@ -12,5 +12,6 @@
"version" : {
"type" : "string"
}
- }
+ },
+ "required" : ["name", "url"]
}
\ No newline at end of file
diff --git a/solr/solrj/src/resources/apispec/cluster.repository.json b/solr/solrj/src/resources/apispec/cluster.repository.json
new file mode 100644
index 0000000..28096d3
--- /dev/null
+++ b/solr/solrj/src/resources/apispec/cluster.repository.json
@@ -0,0 +1,27 @@
+{
+ "methods": [
+ "GET", "POST"
+ ],
+ "url": {
+ "paths": [
+ "/cluster/repository"
+ ]
+ },
+ "commands": {
+ "add": {
+ "description": "",
+ "documentation": "",
+ "#include": "cluster.Commands.repository.properties"
+
+ },
+ "update": {
+ "description": "",
+ "documentation": "",
+ "#include": "cluster.Commands.repository.properties"
+ },
+ "delete": {
+ "description": "",
+ "type": "string"
+ }
+ }
+}
diff --git a/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java b/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java
index 8c06266..6626c9b 100644
--- a/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java
+++ b/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java
@@ -41,6 +41,7 @@ public class JsonValidatorTest extends SolrTestCaseJ4 {
checkSchema("core.SchemaEdit");
checkSchema("cluster.configs.Commands");
checkSchema("cluster.Commands");
+ checkSchema("cluster.repository");
}