You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/09/07 01:05:03 UTC

[lucene-solr] branch jira/SOLR-13722 updated: repossitory CRUD

This is an automated email from the ASF dual-hosted git repository.

noble pushed a commit to branch jira/SOLR-13722
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/jira/SOLR-13722 by this push:
     new f8ac22f  repossitory CRUD
f8ac22f is described below

commit f8ac22f764af62e344b7c82ba09bc4274ba3a8f2
Author: noble <no...@apache.org>
AuthorDate: Sat Sep 7 11:04:44 2019 +1000

    repossitory CRUD
---
 .../solr/handler/admin/BaseHandlerApiSupport.java  |  7 ++
 .../org/apache/solr/handler/admin/ClusterAPI.java  | 79 +++++++++++++++++++---
 .../solr/handler/admin/CollectionHandlerApi.java   |  9 ---
 .../solr/handler/TestContainerReqHandler.java      | 42 ++++++++++++
 .../client/solrj/request/CollectionApiMapping.java |  1 +
 .../apache/solr/common/cloud/ZkStateReader.java    |  4 +-
 .../src/resources/apispec/cluster.Commands.json    | 15 ----
 .../cluster.Commands.repository.properties.json    |  3 +-
 .../src/resources/apispec/cluster.repository.json  | 27 ++++++++
 .../apache/solr/common/util/JsonValidatorTest.java |  1 +
 10 files changed, 151 insertions(+), 37 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java b/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java
index 1993a89..d340b91 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/BaseHandlerApiSupport.java
@@ -17,6 +17,7 @@
 
 package org.apache.solr.handler.admin;
 
+import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -38,6 +39,8 @@ import org.apache.solr.common.util.CommandOperation;
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.response.SolrQueryResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
 import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
@@ -49,6 +52,8 @@ import static org.apache.solr.common.util.StrUtils.splitSmart;
  * to actions and old parameter names to new parameter names
  */
 public abstract class BaseHandlerApiSupport implements ApiSupport {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
   protected final Map<SolrRequest.METHOD, Map<V2EndPoint, List<ApiCommand>>> commandsMapping;
 
   protected BaseHandlerApiSupport() {
@@ -121,8 +126,10 @@ public abstract class BaseHandlerApiSupport implements ApiSupport {
           }
 
         } catch (SolrException e) {
+          log.error("error running command", e);
           throw e;
         } catch (Exception e) {
+          log.error("error running command", e);
           throw new SolrException(BAD_REQUEST, e); //TODO BAD_REQUEST is a wild guess; should we flip the default?  fail here to investigate how this happens in tests
         } finally {
           req.setParams(params);
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ClusterAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/ClusterAPI.java
index 498326f..826d738 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterAPI.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterAPI.java
@@ -34,6 +34,7 @@ import org.apache.solr.client.solrj.request.CollectionApiMapping;
 import org.apache.solr.common.MapWriter;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ClusterProperties;
+import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.util.CommandOperation;
@@ -51,6 +52,7 @@ import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.util.RTimer;
 import org.apache.solr.util.SimplePostTool;
+import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -61,6 +63,7 @@ import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET;
 import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
 import static org.apache.solr.client.solrj.request.CollectionApiMapping.EndPoint.CLUSTER_CMD;
 import static org.apache.solr.client.solrj.request.CollectionApiMapping.EndPoint.CLUSTER_NODES;
+import static org.apache.solr.client.solrj.request.CollectionApiMapping.EndPoint.CLUSTER_REPO;
 import static org.apache.solr.common.util.CommandOperation.captureErrors;
 import static org.apache.solr.common.util.StrUtils.formatString;
 import static org.apache.solr.core.ConfigOverlay.ZNODEVER;
@@ -130,23 +133,25 @@ class ClusterAPI {
 
   enum Commands implements ApiCommand {
 
-/*
-    ADD_REPO(CLUSTER_CMD, POST, "add-repository") {
+    ADD_REPO(CLUSTER_REPO, POST, "add") {
       @Override
       void call(ApiInfo info) throws Exception {
+        repositoryCRUD(info);
       }
     },
-    UPDATE_REPO(CLUSTER_CMD, POST, "update-repository") {
+    UPDATE_REPO(CLUSTER_REPO, POST, "update") {
       @Override
       void call(ApiInfo info) throws Exception {
+        repositoryCRUD(info);
       }
     },
-    DELETE_REPO(CLUSTER_CMD, POST, "delete-repository") {
+    DELETE_REPO(CLUSTER_REPO, POST, "delete") {
       @Override
       void call(ApiInfo info) throws Exception {
+        repositoryCRUD(info);
+
       }
     },
-*/
 
     GET_NODES(CLUSTER_NODES, GET, null) {
       @Override
@@ -318,10 +323,10 @@ class ClusterAPI {
 
     ;
 
-    private CollectionApiMapping.CommandMeta _meta;
+    private CollectionApiMapping.CommandMeta meta;
 
     Commands(CollectionApiMapping.V2EndPoint endPoint, SolrRequest.METHOD method, String cmdName) {
-      _meta = new CollectionApiMapping.CommandMeta() {
+      meta = new CollectionApiMapping.CommandMeta() {
         @Override
         public String getName() {
           return cmdName;
@@ -342,7 +347,7 @@ class ClusterAPI {
 
     @Override
     public CollectionApiMapping.CommandMeta meta() {
-      return _meta;
+      return meta;
 
     }
 
@@ -350,7 +355,7 @@ class ClusterAPI {
     public void invoke(SolrQueryRequest req, SolrQueryResponse rsp, BaseHandlerApiSupport apiHandler) throws Exception {
       CommandOperation op = null;
       if (meta().getHttpMethod() == SolrRequest.METHOD.POST) {
-        if (_meta.getName() != null) {
+        if (meta.getName() != null) {
           List<CommandOperation> commands = req.getCommands(true);
           if (commands == null || commands.size() != 1)
             throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "should have exactly one command");
@@ -370,6 +375,50 @@ class ClusterAPI {
 
   }
 
+  private static void repositoryCRUD(ApiInfo info) {
+    try {
+      SolrZkClient zkClient = info.coreContainer.getZkController().getZkClient();
+      Map data = Utils.getDeepCopy(Utils.getJson(zkClient, ZkStateReader.PACKAGE_REPO, true),3);
+      Map<String, Object> dataMap = null;
+      String name = null;
+      name = info.op.getCommandData() instanceof String ?
+          (String) info.op.getCommandData() :
+          info.op.getStr("name");
+
+      List<String> path = asList("repository", name);
+      boolean contains = Utils.getObjectByPath(data, false, path) != null;
+
+      if(info.op.name.equals(Commands.ADD_REPO.meta.getName())){
+        if(contains){
+          info.op.addError("repository "+ name + " already exists");
+          return;
+        }
+       dataMap = info.op.getDataMap();
+
+      } else if(info.op.name.equals(Commands.UPDATE_REPO.meta.getName())) {
+        if(!contains){
+          info.op.addError("repository "+ name + " does not exist");
+          return;
+        }
+        dataMap = info.op.getDataMap();
+
+      } else if(info.op.name.equals(Commands.DELETE_REPO.meta.getName())){
+        if(!contains){
+          info.op.addError("repository "+ name + " does not exist");
+          return;
+        }
+      }
+
+      Map delta = new LinkedHashMap();
+      Utils.setObjectByPath(delta, path, dataMap, true);
+      updateRepository(zkClient, delta);
+    } catch (SolrException se){
+      throw se;
+    } catch (Exception e) {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+    }
+  }
+
   static boolean addUpdatePackage(ApiInfo params) throws Exception {
     if (!RuntimeLib.isEnabled()) {
       params.op.addError("node not started with enable.runtime.lib=true");
@@ -380,7 +429,7 @@ class ClusterAPI {
     RuntimeLib lib = new RuntimeLib(handler.handler.coreContainer);
     CommandOperation op = params.op;
     String name = op.getStr("name");
-    ClusterProperties clusterProperties = new ClusterProperties(((CollectionHandlerApi) params.apiHandler).handler.coreContainer.getZkController().getZkClient());
+    ClusterProperties clusterProperties = new ClusterProperties(params.coreContainer.getZkController().getZkClient());
     Map<String, Object> props = clusterProperties.getClusterProperties();
     List<String> pathToLib = asList(CommonParams.PACKAGE, name);
     Map existing = (Map) Utils.getObjectByPath(props, false, pathToLib);
@@ -490,4 +539,14 @@ class ClusterAPI {
     }
 
   }
+
+  static void updateRepository(SolrZkClient client, Map delta) throws KeeperException, InterruptedException {
+    client.atomicUpdate(ZkStateReader.PACKAGE_REPO, zkData -> {
+      if (zkData == null) return Utils.toJSON(delta);
+      Map<String, Object> zkJson = (Map<String, Object>) Utils.fromJSON(zkData);
+      boolean modified = Utils.mergeJson(zkJson, delta);
+      return modified ? Utils.toJSON(zkJson) : null;
+    });
+
+  }
 }
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java
index ccf55a6..97f7d96 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java
@@ -73,15 +73,6 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport {
       apiMapping.put(api.meta(), api );
     }
 
-//    addApi(apiMapping, Meta.GET_NODES, CollectionHandlerApi::getNodes);
-//    addApi(apiMapping, Meta.POST_BLOB, CollectionHandlerApi::postBlob);
-//    addApi(apiMapping, Meta.SET_CLUSTER_PROPERTY_OBJ, CollectionHandlerApi::setClusterObj);
-//    addApi(apiMapping, Meta.ADD_PACKAGE, wrap(CollectionHandlerApi::addUpdatePackage));
-//    addApi(apiMapping, Meta.UPDATE_PACKAGE, wrap(CollectionHandlerApi::addUpdatePackage));
-//    addApi(apiMapping, Meta.DELETE_RUNTIME_LIB, wrap(CollectionHandlerApi::deletePackage));
-//    addApi(apiMapping, Meta.ADD_REQ_HANDLER, wrap(CollectionHandlerApi::addRequestHandler));
-//    addApi(apiMapping, Meta.DELETE_REQ_HANDLER, wrap(CollectionHandlerApi::deleteReqHandler));
-
     for (Meta meta : Meta.values()) {
       if (apiMapping.get(meta) == null) {
         log.error("ERROR_INIT. No corresponding API implementation for : " + meta.commandName);
diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java b/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java
index b422475..ddede79 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java
@@ -52,6 +52,7 @@ import org.apache.solr.cloud.SolrCloudTestCase;
 import org.apache.solr.common.NavigableObject;
 import org.apache.solr.common.cloud.ClusterProperties;
 import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.MapSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
@@ -815,6 +816,47 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
 
   }
 
+  public void testRepoCRUD() throws Exception{
+    MiniSolrCloudCluster cluster = configureCluster(4)
+        .withJettyConfig(jetty -> jetty.enableV2(true))
+        .addConfig("conf", configset("cloud-minimal"))
+        .configure();
+    try {
+      String payload = "{add : {name : myrepo, url: 'http://localhost/abc' , version : '1.1'}}";
+      new V2Request.Builder("/cluster/repository")
+          .withPayload(payload)
+          .withMethod(SolrRequest.METHOD.POST)
+          .build().process(cluster.getSolrClient());
+      Map repojson = Utils.getJson(cluster.getZkClient(), ZkStateReader.PACKAGE_REPO, true);
+
+      assertEquals("http://localhost/abc", Utils.getObjectByPath(repojson, true, "/repository/myrepo/url"));
+      assertEquals("1.1", Utils.getObjectByPath(repojson, true, "/repository/myrepo/version"));
+      payload = "{update : {name : myrepo, url: 'http://localhost/abc' , version : '1.2'}}";
+    new V2Request.Builder("/cluster/repository")
+          .withPayload(payload)
+          .withMethod(SolrRequest.METHOD.POST)
+          .build().process(cluster.getSolrClient());
+      repojson = Utils.getJson(cluster.getZkClient(), ZkStateReader.PACKAGE_REPO, true);
+
+      assertEquals("http://localhost/abc", Utils.getObjectByPath(repojson, true, "/repository/myrepo/url"));
+      assertEquals("1.2", Utils.getObjectByPath(repojson, true, "/repository/myrepo/version"));
+
+      payload = "{delete :  myrepo}";
+    new V2Request.Builder("/cluster/repository")
+          .withPayload(payload)
+          .withMethod(SolrRequest.METHOD.POST)
+          .build().process(cluster.getSolrClient());
+      repojson = Utils.getJson(cluster.getZkClient(), ZkStateReader.PACKAGE_REPO, true);
+
+      assertNull( Utils.getObjectByPath(repojson, true, "/repository/myrepo"));
+
+
+    }finally {
+      cluster.shutdown();
+    }
+
+  }
+
   private void postBlob(SolrClient client, ByteBuffer blob, String sh256) throws SolrServerException, IOException {
     V2Response rsp = new V2Request.Builder("/cluster/blob")
         .withMethod(SolrRequest.METHOD.POST)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java
index 97b446f..b9a6cc3 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java
@@ -342,6 +342,7 @@ public class CollectionApiMapping {
     CLUSTER_BLOB("cluster.blob"),
     CLUSTER_ALIASES("cluster.aliases"),
     CLUSTER_CMD("cluster.Commands"),
+    CLUSTER_REPO("cluster.repository"),
     CLUSTER_NODES("cluster.nodes"),
     CLUSTER_CMD_STATUS("cluster.commandstatus"),
     CLUSTER_CMD_STATUS_DELETE("cluster.commandstatus.delete"),
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index b9c66cf..31ff3f5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -110,6 +110,7 @@ public class ZkStateReader implements SolrCloseable {
   public static final String ALIASES = "/aliases.json";
   public static final String CLUSTER_STATE = "/clusterstate.json";
   public static final String CLUSTER_PROPS = "/clusterprops.json";
+  public static final String PACKAGE_REPO = "/package_repo.json";
   public static final String COLLECTION_PROPS_ZKNODE = "collectionprops.json";
   public static final String REJOIN_AT_HEAD_PROP = "rejoinAtHead";
   public static final String SOLR_SECURITY_CONF_PATH = "/security.json";
@@ -156,7 +157,6 @@ public class ZkStateReader implements SolrCloseable {
 
   private static final int GET_LEADER_RETRY_INTERVAL_MS = 50;
   private static final int GET_LEADER_RETRY_DEFAULT_TIMEOUT = Integer.parseInt(System.getProperty("zkReaderGetLeaderRetryTimeoutMs", "4000"));
-  ;
 
   public static final String LEADER_ELECT_ZKNODE = "leader_elect";
 
@@ -1572,7 +1572,7 @@ public class ZkStateReader implements SolrCloseable {
         Stat stat = new Stat();
         byte[] data = zkClient.getData(collectionPath, watcher, stat, true);
         ClusterState state = ClusterState.load(stat.getVersion(), data,
-            Collections.<String>emptySet(), collectionPath);
+            Collections.emptySet(), collectionPath);
         ClusterState.CollectionRef collectionRef = state.getCollectionStates().get(coll);
         return collectionRef == null ? null : collectionRef.get();
       } catch (KeeperException.NoNodeException e) {
diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.json b/solr/solrj/src/resources/apispec/cluster.Commands.json
index 5d5ae7f..e0da898 100644
--- a/solr/solrj/src/resources/apispec/cluster.Commands.json
+++ b/solr/solrj/src/resources/apispec/cluster.Commands.json
@@ -209,21 +209,6 @@
     "delete-requesthandler" : {
       "description" : "delete a requesthandler",
       "type": "string"
-    },
-    "add-repository": {
-      "description": "",
-      "documentation": "",
-      "#include": "cluster.Commands.repository.properties"
-
-    },
-    "update-repository": {
-      "description": "",
-      "documentation": "",
-      "#include": "cluster.Commands.repository.properties"
-    },
-    "delete-repository": {
-      "description": "",
-      "type": "string"
     }
 
   }
diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.repository.properties.json b/solr/solrj/src/resources/apispec/cluster.Commands.repository.properties.json
index 05b8359..9f46f59 100644
--- a/solr/solrj/src/resources/apispec/cluster.Commands.repository.properties.json
+++ b/solr/solrj/src/resources/apispec/cluster.Commands.repository.properties.json
@@ -12,5 +12,6 @@
     "version" : {
       "type" : "string"
     }
-  }
+  },
+  "required" : ["name", "url"]
 }
\ No newline at end of file
diff --git a/solr/solrj/src/resources/apispec/cluster.repository.json b/solr/solrj/src/resources/apispec/cluster.repository.json
new file mode 100644
index 0000000..28096d3
--- /dev/null
+++ b/solr/solrj/src/resources/apispec/cluster.repository.json
@@ -0,0 +1,27 @@
+{
+  "methods": [
+    "GET", "POST"
+  ],
+  "url": {
+    "paths": [
+      "/cluster/repository"
+    ]
+  },
+  "commands": {
+    "add": {
+      "description": "",
+      "documentation": "",
+      "#include": "cluster.Commands.repository.properties"
+
+    },
+    "update": {
+      "description": "",
+      "documentation": "",
+      "#include": "cluster.Commands.repository.properties"
+    },
+    "delete": {
+      "description": "",
+      "type": "string"
+    }
+  }
+}
diff --git a/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java b/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java
index 8c06266..6626c9b 100644
--- a/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java
+++ b/solr/solrj/src/test/org/apache/solr/common/util/JsonValidatorTest.java
@@ -41,6 +41,7 @@ public class JsonValidatorTest extends SolrTestCaseJ4  {
     checkSchema("core.SchemaEdit");
     checkSchema("cluster.configs.Commands");
     checkSchema("cluster.Commands");
+    checkSchema("cluster.repository");
   }