You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2017/02/02 08:29:37 UTC
[1/2] lucene-solr:branch_6x: SOLR-7955: Auto create .system
collection on first request if it does not exist
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 616225549 -> afae0d971
SOLR-7955: Auto create .system collection on first request if it does not exist
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ec778c5c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ec778c5c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ec778c5c
Branch: refs/heads/branch_6x
Commit: ec778c5c8f6b3bcb551a3ae9999c73f664e2de90
Parents: 6162255
Author: Noble Paul <no...@apache.org>
Authored: Thu Feb 2 15:26:26 2017 +1030
Committer: Noble Paul <no...@apache.org>
Committed: Thu Feb 2 18:59:15 2017 +1030
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../org/apache/solr/servlet/HttpSolrCall.java | 39 +++++++++++++++++
.../apache/solr/handler/TestBlobHandler.java | 44 +++++++++++---------
.../solr/handler/TestSystemCollAutoCreate.java | 29 +++++++++++++
4 files changed, 96 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ec778c5c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d003633..18e65d1 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -64,6 +64,9 @@ Optimizations
* SOLR-9933: SolrCoreParser now supports configuration of custom SpanQueryBuilder classes.
(Daniel Collins, Christine Poerschke)
+* SOLR-7955: Auto create .system collection on first request if it does not exist (noble)
+
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ec778c5c/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index 1f98da9..bbe9074 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -37,6 +37,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CloseShieldInputStream;
@@ -80,6 +81,7 @@ import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.ContentStreamHandlerBase;
import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.request.SolrRequestHandler;
@@ -98,6 +100,7 @@ import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
import org.apache.solr.servlet.cache.Method;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.util.RTimerTree;
+import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,9 +109,13 @@ import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.RELOAD;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.common.params.CoreAdminParams.ACTION;
+import static org.apache.solr.handler.admin.CollectionsHandler.SYSTEM_COLL;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.FORWARD;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.PASSTHROUGH;
@@ -287,6 +294,9 @@ public class HttpSolrCall {
// if we couldn't find it locally, look on other nodes
extractRemotePath(corename, origCorename, idx);
if (action != null) return;
+ //core is not available locally or remotely
+ autoCreateSystemColl();
+ if(action != null) return;
}
// With a valid core...
@@ -321,6 +331,35 @@ public class HttpSolrCall {
action = PASSTHROUGH;
}
+
+ protected void autoCreateSystemColl() throws Exception {
+ if (SYSTEM_COLL.equals(corename) && "POST".equals(req.getMethod()) && !cores.getZkController().getClusterState().hasCollection(SYSTEM_COLL)) {
+ log.info("Going to auto-create .system collection");
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ String repFactor = String.valueOf(Math.min(3, cores.getZkController().getClusterState().getLiveNodes().size()));
+ cores.getCollectionsHandler().handleRequestBody(new LocalSolrQueryRequest(null,
+ new ModifiableSolrParams()
+ .add(ACTION, CREATE.toString())
+ .add( NAME, SYSTEM_COLL)
+ .add(REPLICATION_FACTOR, repFactor)), rsp);
+ if (rsp.getValues().get("success") == null) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Could not auto-create .system collection: "+ Utils.toJSONString(rsp.getValues()));
+ }
+ TimeOut timeOut = new TimeOut(3, TimeUnit.SECONDS);
+ for (; ; ) {
+ if (cores.getZkController().getClusterState().getCollectionOrNull(SYSTEM_COLL) != null) {
+ break;
+ } else {
+ if (timeOut.hasTimedOut()) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find .system collection even after 3 seconds");
+ }
+ Thread.sleep(50);
+ }
+ }
+
+ action = RETRY;
+ }
+ }
/**
* Extract handler from the URL path if not set.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ec778c5c/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java b/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java
index 4fda926..c395d20 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java
@@ -90,27 +90,33 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
"field",
"type")));
- byte[] bytarr = new byte[1024];
- for (int i = 0; i < bytarr.length; i++) bytarr[i] = (byte) (i % 127);
- byte[] bytarr2 = new byte[2048];
- for (int i = 0; i < bytarr2.length; i++) bytarr2[i] = (byte) (i % 127);
- String blobName = "test";
- postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr), 1);
- postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr2), 2);
-
- url = baseUrl + "/.system/blob/test/1";
- map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
- List l = (List) Utils.getObjectByPath(map, false, Arrays.asList("response", "docs"));
- assertNotNull("" + map, l);
- assertTrue("" + map, l.size() > 0);
- map = (Map) l.get(0);
- assertEquals("" + bytarr.length, String.valueOf(map.get("size")));
-
- compareInputAndOutput(baseUrl + "/.system/blob/test?wt=filestream", bytarr2);
- compareInputAndOutput(baseUrl + "/.system/blob/test/1?wt=filestream", bytarr);
+ checkBlobPost(baseUrl, cloudClient);
}
}
+ static void checkBlobPost(String baseUrl, CloudSolrClient cloudClient) throws Exception {
+ String url;
+ Map map;
+ byte[] bytarr = new byte[1024];
+ for (int i = 0; i < bytarr.length; i++) bytarr[i] = (byte) (i % 127);
+ byte[] bytarr2 = new byte[2048];
+ for (int i = 0; i < bytarr2.length; i++) bytarr2[i] = (byte) (i % 127);
+ String blobName = "test";
+ postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr), 1);
+ postAndCheck(cloudClient, baseUrl, blobName, ByteBuffer.wrap(bytarr2), 2);
+
+ url = baseUrl + "/.system/blob/test/1";
+ map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
+ List l = (List) Utils.getObjectByPath(map, false, Arrays.asList("response", "docs"));
+ assertNotNull("" + map, l);
+ assertTrue("" + map, l.size() > 0);
+ map = (Map) l.get(0);
+ assertEquals("" + bytarr.length, String.valueOf(map.get("size")));
+
+ compareInputAndOutput(baseUrl + "/.system/blob/test?wt=filestream", bytarr2, cloudClient);
+ compareInputAndOutput(baseUrl + "/.system/blob/test/1?wt=filestream", bytarr, cloudClient);
+ }
+
public static void createSystemCollection(SolrClient client) throws SolrServerException, IOException {
CollectionAdminResponse response1;
CollectionAdminRequest.Create createCollectionRequest = new CollectionAdminRequest.Create()
@@ -152,7 +158,7 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
return new String(Utils.toJSON(map), StandardCharsets.UTF_8);
}
- private void compareInputAndOutput(String url, byte[] bytarr) throws IOException {
+ static void compareInputAndOutput(String url, byte[] bytarr, CloudSolrClient cloudClient) throws IOException {
HttpClient httpClient = cloudClient.getLbClient().getHttpClient();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ec778c5c/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java b/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java
new file mode 100644
index 0000000..cadda58
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/TestSystemCollAutoCreate.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler;
+
+
+import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.common.cloud.DocCollection;
+
+public class TestSystemCollAutoCreate extends AbstractFullDistribZkTestBase {
+ public void testAutoCreate() throws Exception {
+ TestBlobHandler.checkBlobPost(cloudJettys.get(0).jetty.getBaseUrl().toExternalForm(), cloudClient);
+ DocCollection sysColl = cloudClient.getZkStateReader().getClusterState().getCollection(".system");
+ }
+}
[2/2] lucene-solr:branch_6x: SOLR-7955: Auto create .system
collection on first request if it does not exist
Posted by no...@apache.org.
SOLR-7955: Auto create .system collection on first request if it does not exist
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/afae0d97
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/afae0d97
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/afae0d97
Branch: refs/heads/branch_6x
Commit: afae0d971867216d1f02a96395ba939a37c8d86e
Parents: ec778c5
Author: Noble Paul <no...@apache.org>
Authored: Thu Feb 2 18:59:23 2017 +1030
Committer: Noble Paul <no...@apache.org>
Committed: Thu Feb 2 18:59:23 2017 +1030
----------------------------------------------------------------------
.../core/src/java/org/apache/solr/servlet/HttpSolrCall.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/afae0d97/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index bbe9074..75b4e68 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -295,7 +295,7 @@ public class HttpSolrCall {
extractRemotePath(corename, origCorename, idx);
if (action != null) return;
//core is not available locally or remotely
- autoCreateSystemColl();
+ autoCreateSystemColl(corename);
if(action != null) return;
}
@@ -332,8 +332,11 @@ public class HttpSolrCall {
action = PASSTHROUGH;
}
- protected void autoCreateSystemColl() throws Exception {
- if (SYSTEM_COLL.equals(corename) && "POST".equals(req.getMethod()) && !cores.getZkController().getClusterState().hasCollection(SYSTEM_COLL)) {
+ protected void autoCreateSystemColl(String corename) throws Exception {
+ if (core == null &&
+ SYSTEM_COLL.equals(corename) &&
+ "POST".equals(req.getMethod()) &&
+ !cores.getZkController().getClusterState().hasCollection(SYSTEM_COLL)) {
log.info("Going to auto-create .system collection");
SolrQueryResponse rsp = new SolrQueryResponse();
String repFactor = String.valueOf(Math.min(3, cores.getZkController().getClusterState().getLiveNodes().size()));