You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by no...@apache.org on 2023/06/05 06:03:41 UTC
[solr] branch branch_9x updated: SOLR-16812: Support CBOR format for update/query (#1655)
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 95f2ea967ca SOLR-16812: Support CBOR format for update/query (#1655)
95f2ea967ca is described below
commit 95f2ea967cab6f98994aed53e3398721e498386f
Author: Noble Paul <no...@users.noreply.github.com>
AuthorDate: Mon Jun 5 15:32:49 2023 +1000
SOLR-16812: Support CBOR format for update/query (#1655)
---
solr/core/build.gradle | 1 +
.../src/java/org/apache/solr/core/SolrCore.java | 2 +
.../apache/solr/handler/UpdateRequestHandler.java | 4 +
.../org/apache/solr/handler/loader/CborLoader.java | 154 +++++++++
.../apache/solr/response/CborResponseWriter.java | 193 +++++++++++
solr/core/src/resources/ImplicitPlugins.json | 7 +
.../test/org/apache/solr/core/SolrCoreTest.java | 2 +
.../org/apache/solr/util/TestCborDataFormat.java | 221 +++++++++++++
solr/solrj/build.gradle | 2 +
.../client/solrj/request/GenericSolrRequest.java | 18 ++
.../solr/client/solrj/SolrExampleCborTest.java | 353 +++++++++++++++++++++
versions.lock | 2 +-
12 files changed, 958 insertions(+), 1 deletion(-)
diff --git a/solr/core/build.gradle b/solr/core/build.gradle
index 9d8d164a00e..d06f79132ef 100644
--- a/solr/core/build.gradle
+++ b/solr/core/build.gradle
@@ -120,6 +120,7 @@ dependencies {
implementation 'com.fasterxml.jackson.core:jackson-core'
implementation 'com.fasterxml.jackson.core:jackson-databind'
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile'
+ implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor'
implementation 'org.apache.httpcomponents:httpclient'
implementation 'org.apache.httpcomponents:httpcore'
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index c77b885beb4..dd8032a4e88 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -124,6 +124,7 @@ import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.BinaryResponseWriter;
import org.apache.solr.response.CSVResponseWriter;
+import org.apache.solr.response.CborResponseWriter;
import org.apache.solr.response.GeoJSONResponseWriter;
import org.apache.solr.response.GraphMLResponseWriter;
import org.apache.solr.response.JacksonJsonWriter;
@@ -3034,6 +3035,7 @@ public class SolrCore implements SolrInfoBean, Closeable {
m.put("ruby", new RubyResponseWriter());
m.put("raw", new RawResponseWriter());
m.put(CommonParams.JAVABIN, new BinaryResponseWriter());
+ m.put("cbor", new CborResponseWriter());
m.put("csv", new CSVResponseWriter());
m.put("schema.xml", new SchemaXmlResponseWriter());
m.put("smile", new SmileResponseWriter());
diff --git a/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java
index 368b40cdb55..805a712e0a5 100644
--- a/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java
@@ -32,6 +32,7 @@ import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.loader.CSVLoader;
+import org.apache.solr.handler.loader.CborLoader;
import org.apache.solr.handler.loader.ContentStreamLoader;
import org.apache.solr.handler.loader.JavabinLoader;
import org.apache.solr.handler.loader.JsonLoader;
@@ -146,6 +147,7 @@ public class UpdateRequestHandler extends ContentStreamHandlerBase
registry.put("application/json", new JsonLoader().init(p));
registry.put("application/csv", new CSVLoader().init(p));
registry.put("application/javabin", new JavabinLoader(instance).init(p));
+ registry.put("application/cbor", CborLoader.createLoader(p));
registry.put("text/csv", registry.get("application/csv"));
registry.put("text/xml", registry.get("application/xml"));
registry.put("text/json", registry.get("application/json"));
@@ -154,6 +156,7 @@ public class UpdateRequestHandler extends ContentStreamHandlerBase
pathVsLoaders.put(DOC_PATH, registry.get("application/json"));
pathVsLoaders.put(CSV_PATH, registry.get("application/csv"));
pathVsLoaders.put(BIN_PATH, registry.get("application/javabin"));
+ pathVsLoaders.put(CBOR_PATH, registry.get("application/cbor"));
return registry;
}
@@ -184,4 +187,5 @@ public class UpdateRequestHandler extends ContentStreamHandlerBase
public static final String JSON_PATH = "/update/json";
public static final String CSV_PATH = "/update/csv";
public static final String BIN_PATH = "/update/bin";
+ public static final String CBOR_PATH = "/update/cbor";
}
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CborLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/CborLoader.java
new file mode 100644
index 00000000000..fd8c269d7cc
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/loader/CborLoader.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.loader;
+
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import com.fasterxml.jackson.dataformat.cbor.CBORParser;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+
+/**
+ * This class can load a single document or a stream of documents in CBOR format this is equivalent
+ * of loading a single json documet or an array of json documents
+ */
+public class CborLoader {
+ final CBORFactory cborFactory;
+ private final Consumer<SolrInputDocument> sink;
+
+ public CborLoader(CBORFactory cborFactory, Consumer<SolrInputDocument> sink) {
+ this.cborFactory = cborFactory == null ? new CBORFactory() : cborFactory;
+ this.sink = sink;
+ }
+
+ public void stream(InputStream is) throws IOException {
+ ObjectMapper mapper = new ObjectMapper(new CBORFactory());
+ try (CBORParser parser = (CBORParser) mapper.getFactory().createParser(is)) {
+ JsonToken t;
+ while ((t = parser.nextToken()) != null) {
+
+ if (t == JsonToken.START_ARRAY) {
+ // this is an array of docs
+ t = parser.nextToken();
+ if (t == JsonToken.START_OBJECT) {
+ handleDoc(parser);
+ }
+
+ } else if (t == JsonToken.START_OBJECT) {
+ // this is just a single doc
+ handleDoc(parser);
+ }
+ }
+ }
+ }
+
+ private void handleDoc(CBORParser p) throws IOException {
+ SolrInputDocument doc = new SolrInputDocument();
+ for (; ; ) {
+ JsonToken t = p.nextToken();
+ if (t == JsonToken.END_OBJECT) {
+ if (!doc.isEmpty()) {
+ sink.accept(doc);
+ }
+ return;
+ }
+ String name;
+ if (t == JsonToken.FIELD_NAME) {
+ name = p.getCurrentName();
+ t = p.nextToken();
+ if (t == JsonToken.START_ARRAY) {
+
+ List<Object> l = new ArrayList<>();
+ for (; ; ) {
+ t = p.nextToken();
+ if (t == JsonToken.END_ARRAY) break;
+ else {
+ l.add(readVal(t, p));
+ }
+ }
+ if (!l.isEmpty()) {
+ doc.addField(name, l);
+ }
+
+ } else {
+ doc.addField(name, readVal(t, p));
+ }
+ }
+ }
+ }
+
+ private Object readVal(JsonToken t, CBORParser p) throws IOException {
+ if (t == JsonToken.VALUE_NULL) {
+ return null;
+ }
+ if (t == JsonToken.VALUE_STRING) {
+ return p.getValueAsString();
+ }
+ if (t == JsonToken.VALUE_TRUE) {
+ return Boolean.TRUE;
+ }
+ if (t == JsonToken.VALUE_FALSE) {
+ return Boolean.FALSE;
+ }
+ if (t == JsonToken.VALUE_NUMBER_INT || t == JsonToken.VALUE_NUMBER_FLOAT) {
+ return p.getNumberValue();
+ }
+ throw new RuntimeException("Unknown type :" + t);
+ }
+
+ public static ContentStreamLoader createLoader(SolrParams p) {
+ CBORFactory factory = new CBORFactory();
+ return new ContentStreamLoader() {
+ @Override
+ public void load(
+ SolrQueryRequest req,
+ SolrQueryResponse rsp,
+ ContentStream stream,
+ UpdateRequestProcessor processor)
+ throws IOException {
+ int commitWithin = req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1);
+ boolean overwrite = req.getParams().getBool(UpdateParams.OVERWRITE, true);
+ new CborLoader(
+ factory,
+ doc -> {
+ AddUpdateCommand add = new AddUpdateCommand(req);
+ add.commitWithin = commitWithin;
+ add.solrDoc = doc;
+ add.overwrite = overwrite;
+ try {
+ processor.processAdd(add);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .stream(stream.getStream());
+ }
+ }.init(p);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/response/CborResponseWriter.java b/solr/core/src/java/org/apache/solr/response/CborResponseWriter.java
new file mode 100644
index 00000000000..8f52093cac0
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/response/CborResponseWriter.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.response;
+
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import com.fasterxml.jackson.dataformat.cbor.CBORGenerator;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import org.apache.solr.client.solrj.impl.BinaryResponseParser;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+
+/**
+ * A response writer impl that can write results in CBOR (cbor.io) format when wt=cbor. It uses the
+ * jackson library to write the stream out
+ */
+public class CborResponseWriter extends BinaryResponseWriter {
+ final CBORFactory cborFactory;
+
+ public CborResponseWriter() {
+ cborFactory = new CBORFactory();
+ }
+
+ @Override
+ public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse response)
+ throws IOException {
+ WriterImpl writer = new WriterImpl(cborFactory, out, req, response);
+ writer.writeResponse();
+ writer.gen.flush();
+ }
+
+ @Override
+ public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
+ return BinaryResponseParser.BINARY_CONTENT_TYPE;
+ }
+
+ static class WriterImpl extends JSONWriter {
+
+ final CBORGenerator gen;
+
+ public WriterImpl(
+ CBORFactory factory, OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp)
+ throws IOException {
+ super(null, req, rsp);
+ gen = factory.createGenerator(out);
+ }
+
+ @Override
+ public void writeResponse() throws IOException {
+ super.writeNamedList(null, rsp.getValues());
+ }
+
+ @SuppressWarnings("rawtypes")
+ public void write(NamedList nl) throws IOException {
+ super.writeNamedList(null, nl);
+ }
+
+ @Override
+ public void writeNumber(String name, Number val) throws IOException {
+ if (val instanceof Integer) {
+ gen.writeNumber(val.intValue());
+ } else if (val instanceof Long) {
+ gen.writeNumber(val.longValue());
+ } else if (val instanceof Float) {
+ gen.writeNumber(val.floatValue());
+ } else if (val instanceof Double) {
+ gen.writeNumber(val.doubleValue());
+ } else if (val instanceof Short) {
+ gen.writeNumber(val.shortValue());
+ } else if (val instanceof Byte) {
+ gen.writeNumber(val.byteValue());
+ } else if (val instanceof BigInteger) {
+ gen.writeNumber((BigInteger) val);
+ } else if (val instanceof BigDecimal) {
+ gen.writeNumber((BigDecimal) val);
+ } else {
+ gen.writeString(val.getClass().getName() + ':' + val.toString());
+ // default... for debugging only
+ }
+ }
+
+ @Override
+ public void writeBool(String name, Boolean val) throws IOException {
+ gen.writeBoolean(val);
+ }
+
+ @Override
+ public void writeNull(String name) throws IOException {
+ gen.writeNull();
+ }
+
+ @Override
+ public void writeStr(String name, String val, boolean needsEscaping) throws IOException {
+ gen.writeString(val);
+ }
+
+ @Override
+ public void writeLong(String name, long val) throws IOException {
+ gen.writeNumber(val);
+ }
+
+ @Override
+ public void writeInt(String name, int val) throws IOException {
+ gen.writeNumber(val);
+ }
+
+ @Override
+ public void writeBool(String name, boolean val) throws IOException {
+ gen.writeBoolean(val);
+ }
+
+ @Override
+ public void writeFloat(String name, float val) throws IOException {
+ gen.writeNumber(val);
+ }
+
+ @Override
+ public void writeArrayCloser() throws IOException {
+ gen.writeEndArray();
+ }
+
+ @Override
+ public void writeArraySeparator() {
+ // do nothing
+ }
+
+ @Override
+ public void writeArrayOpener(int size) throws IOException, IllegalArgumentException {
+ gen.writeStartArray();
+ }
+
+ @Override
+ public void writeMapCloser() throws IOException {
+ gen.writeEndObject();
+ }
+
+ @Override
+ public void writeKey(String fname, boolean needsEscaping) throws IOException {
+ gen.writeFieldName(fname);
+ }
+
+ @Override
+ public void writeMapOpener(int size) throws IOException, IllegalArgumentException {
+ gen.writeStartObject();
+ }
+
+ @Override
+ public void writeByteArr(String name, byte[] buf, int offset, int len) throws IOException {
+ gen.writeBinary(buf, offset, len);
+ }
+
+ @Override
+ public void writeMapSeparator() throws IOException {
+ // do nothing
+ }
+
+ @Override
+ public void indent() throws IOException {
+ // do nothing
+ }
+
+ @Override
+ public void indent(int lev) throws IOException {
+ // do nothing
+ }
+
+ @Override
+ public int incLevel() {
+ return 0;
+ }
+
+ @Override
+ public int decLevel() {
+ return 0;
+ }
+ }
+}
diff --git a/solr/core/src/resources/ImplicitPlugins.json b/solr/core/src/resources/ImplicitPlugins.json
index b0ee1fec3ca..c4da631c17a 100644
--- a/solr/core/src/resources/ImplicitPlugins.json
+++ b/solr/core/src/resources/ImplicitPlugins.json
@@ -11,6 +11,13 @@
"update.contentType": "application/json"
}
},
+ "/update/cbor": {
+ "useParams":"_UPDATE_CBOR",
+ "class": "solr.UpdateRequestHandler",
+ "invariants": {
+ "update.contentType": "application/cbor"
+ }
+ },
"/update/csv": {
"useParams":"_UPDATE_CSV",
"class": "solr.UpdateRequestHandler",
diff --git a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java
index b7cd61e99e7..e401bcc695a 100644
--- a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java
+++ b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java
@@ -129,6 +129,8 @@ public class SolrCoreTest extends SolrTestCaseJ4 {
++ihCount;
assertEquals(pathToClassMap.get("/update/json/docs"), "solr.UpdateRequestHandler");
++ihCount;
+ assertEquals(pathToClassMap.get("/update/cbor"), "solr.UpdateRequestHandler");
+ ++ihCount;
assertEquals(pathToClassMap.get("/analysis/document"), "solr.DocumentAnalysisRequestHandler");
++ihCount;
assertEquals(pathToClassMap.get("/analysis/field"), "solr.FieldAnalysisRequestHandler");
diff --git a/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java b/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java
new file mode 100644
index 00000000000..536a21757aa
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.LongAdder;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.InputStreamResponseParser;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.handler.loader.CborLoader;
+import org.apache.solr.response.XMLResponseWriter;
+
+public class TestCborDataFormat extends SolrCloudTestCase {
+
+ @SuppressWarnings("unchecked")
+ public void testRoundTrip() throws Exception {
+ String testCollection = "testRoundTrip";
+
+ MiniSolrCloudCluster cluster =
+ configureCluster(1)
+ .addConfig(
+ "conf", TEST_PATH().resolve("configsets").resolve("cloud-managed").resolve("conf"))
+ .configure();
+ try {
+ System.setProperty("managed.schema.mutable", "true");
+ CloudSolrClient client = cluster.getSolrClient();
+ CollectionAdminRequest.createCollection(testCollection, "conf", 1, 1).process(client);
+ modifySchema(testCollection, client);
+
+ byte[] b =
+ Files.readAllBytes(
+ new File(ExternalPaths.SOURCE_HOME, "example/films/films.json").toPath());
+ // every operation is performed twice. We should only take the second number
+ // so that we give JVM a chance to optimize that code
+ index(testCollection, client, createJsonReq(b), true);
+ index(testCollection, client, createJsonReq(b), true);
+
+ index(testCollection, client, createJavabinReq(b), true);
+ index(testCollection, client, createJavabinReq(b), true);
+
+ index(testCollection, client, createCborReq(b), true);
+ index(testCollection, client, createCborReq(b), false);
+
+ runQuery(testCollection, client, "javabin");
+ runQuery(testCollection, client, "javabin");
+ runQuery(testCollection, client, "json");
+ runQuery(testCollection, client, "json");
+ b = runQuery(testCollection, client, "cbor");
+ b = runQuery(testCollection, client, "cbor");
+ ObjectMapper objectMapper = new ObjectMapper(new CBORFactory());
+ Object o = objectMapper.readValue(b, Object.class);
+ List<Object> l = (List<Object>) Utils.getObjectByPath(o, false, "response/docs");
+ assertEquals(1100, l.size());
+ } finally {
+ System.clearProperty("managed.schema.mutable");
+ cluster.shutdown();
+ }
+ }
+
+ private void index(
+ String testCollection, CloudSolrClient client, GenericSolrRequest r, boolean del)
+ throws Exception {
+ RTimer timer = new RTimer();
+ client.request(r, testCollection);
+ System.out.println("INDEX_TIME: " + r.contentWriter.getContentType() + " : " + timer.getTime());
+ if (del) {
+ UpdateRequest req = new UpdateRequest().deleteByQuery("*:*");
+ req.setParam("commit", "true");
+ client.request(req, testCollection);
+ }
+ }
+
+ private byte[] runQuery(String testCollection, CloudSolrClient client, String wt)
+ throws SolrServerException, IOException {
+ NamedList<Object> result;
+ QueryRequest request;
+ RTimer timer = new RTimer();
+ request = new QueryRequest(new SolrQuery("*:*").setRows(1111));
+ request.setResponseParser(new InputStreamResponseParser(wt));
+ result = client.request(request, testCollection);
+ byte[] b = copyStream((InputStream) result.get("stream"));
+ System.out.println(wt + "_time : " + timer.getTime());
+ return b;
+ }
+
+ private static byte[] copyStream(InputStream inputStream) throws IOException {
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ byte[] buffer = new byte[4096];
+ int bytesRead;
+ while ((bytesRead = inputStream.read(buffer)) != -1) {
+ outputStream.write(buffer, 0, bytesRead);
+ }
+ return outputStream.toByteArray();
+ }
+
+ private void modifySchema(String testCollection, CloudSolrClient client)
+ throws SolrServerException, IOException {
+ GenericSolrRequest req =
+ new GenericSolrRequest(SolrRequest.METHOD.POST, "/schema")
+ .setContentWriter(
+ new RequestWriter.StringPayloadContentWriter(
+ "{\n"
+ + "\"add-field-type\" : {"
+ + "\"name\":\"knn_vector_10\",\"class\":\"solr.DenseVectorField\",\"vectorDimension\":10,\"similarityFunction\":\"cosine\",\"knnAlgorithm\":\"hnsw\"},\n"
+ + "\"add-field\" : ["
+ + "{\"name\":\"name\",\"type\":\"sortabletext\",\"multiValued\":false,\"stored\":true},\n"
+ + "{\"name\":\"initial_release_date\",\"type\":\"string\",\"stored\":true},\n"
+ + "{\"name\":\"directed_by\",\"type\":\"string\",\"multiValued\":true,\"stored\":true},\n"
+ + "{\"name\":\"genre\",\"type\":\"string\",\"multiValued\":true,\"stored\":true},\n"
+ + "{\"name\":\"film_vector\",\"type\":\"knn_vector_10\",\"indexed\":true,\"stored\":true}]}",
+ XMLResponseWriter.CONTENT_TYPE_XML_UTF8));
+
+ client.request(req, testCollection);
+ }
+
+ private GenericSolrRequest createJsonReq(byte[] b) {
+ return new GenericSolrRequest(
+ SolrRequest.METHOD.POST,
+ "/update/json/docs",
+ new MapSolrParams(Map.of("commit", "true")))
+ .withContent(b, "application/json");
+ }
+
+ @SuppressWarnings("rawtypes")
+ private GenericSolrRequest createJavabinReq(byte[] b) throws IOException {
+ List l = (List) Utils.fromJSON(b);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ new JavaBinCodec().marshal(l.iterator(), baos);
+
+ return new GenericSolrRequest(
+ SolrRequest.METHOD.POST, "/update", new MapSolrParams(Map.of("commit", "true")))
+ .withContent(baos.toByteArray(), "application/javabin");
+ }
+
+ private GenericSolrRequest createCborReq(byte[] b) throws IOException {
+ return new GenericSolrRequest(
+ SolrRequest.METHOD.POST, "/update/cbor", new MapSolrParams(Map.of("commit", "true")))
+ .withContent(serializeToCbor(b), "application/cbor");
+ }
+
+ @SuppressWarnings("unchecked")
+ public void test() throws Exception {
+ Path filmsJson = new File(ExternalPaths.SOURCE_HOME, "example/films/films.json").toPath();
+
+ long sz = Files.size(filmsJson);
+ assertEquals(633600, sz);
+
+ List<Object> films = null;
+ try (InputStream is = Files.newInputStream(filmsJson)) {
+ films = (List<Object>) Utils.fromJSON(is);
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ new JavaBinCodec().marshal(Map.of("films", films), baos);
+ assertEquals(234520, baos.toByteArray().length);
+
+ byte[] b = Files.readAllBytes(filmsJson);
+ byte[] bytes = serializeToCbor(b);
+ assertEquals(290672, bytes.length);
+ LongAdder docsSz = new LongAdder();
+ new CborLoader(null, (document) -> docsSz.increment()).stream(new ByteArrayInputStream(bytes));
+ assertEquals(films.size(), docsSz.intValue());
+ }
+
+ private byte[] serializeToCbor(byte[] is) throws IOException {
+ ByteArrayOutputStream baos;
+ ObjectMapper jsonMapper = new ObjectMapper(new JsonFactory());
+
+ // Read JSON file as a JsonNode
+ JsonNode jsonNode = jsonMapper.readTree(is);
+ // Create a CBOR ObjectMapper
+ CBORFactory jf = new CBORFactory();
+ ObjectMapper cborMapper = new ObjectMapper(jf);
+ baos = new ByteArrayOutputStream();
+ JsonGenerator jsonGenerator = cborMapper.createGenerator(baos);
+
+ jsonGenerator.writeTree(jsonNode);
+ jsonGenerator.close();
+ return baos.toByteArray();
+ }
+}
diff --git a/solr/solrj/build.gradle b/solr/solrj/build.gradle
index 6c8d5405cbf..b1e892ecec3 100644
--- a/solr/solrj/build.gradle
+++ b/solr/solrj/build.gradle
@@ -66,6 +66,8 @@ dependencies {
testImplementation 'org.hamcrest:hamcrest'
testImplementation 'commons-io:commons-io'
+ testImplementation 'com.fasterxml.jackson.core:jackson-databind'
+ testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor'
testImplementation 'org.eclipse.jetty.toolchain:jetty-servlet-api'
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/GenericSolrRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/GenericSolrRequest.java
index 7f08f454a8f..e9babd55371 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/GenericSolrRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/GenericSolrRequest.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.client.solrj.request;
+import java.io.IOException;
+import java.io.OutputStream;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.RequestWriter.ContentWriter;
@@ -42,6 +44,22 @@ public class GenericSolrRequest extends SolrRequest<SimpleSolrResponse> {
return this;
}
+ public GenericSolrRequest withContent(byte[] buf, String type) {
+ contentWriter =
+ new ContentWriter() {
+ @Override
+ public void write(OutputStream os) throws IOException {
+ os.write(buf);
+ }
+
+ @Override
+ public String getContentType() {
+ return type;
+ }
+ };
+ return this;
+ }
+
@Override
public ContentWriter getContentWriter(String expectedType) {
return contentWriter;
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleCborTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleCborTest.java
new file mode 100644
index 00000000000..f021e028386
--- /dev/null
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleCborTest.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+/**
+ * This test only tests a subset of tests specified in the parent class. It's because we do not
+ * provide a full impl of all types of responses in SolrJ. It just handles very common types and
+ * converts them from Map to NamedList
+ */
+public class SolrExampleCborTest extends SolrExampleTests {
+ @BeforeClass
+ public static void beforeTest() throws Exception {
+ createAndStartJetty(legacyExampleCollection1SolrHome());
+ }
+
+ @Override
+ public SolrClient createNewSolrClient() {
+ return new HttpSolrClient.Builder(getServerUrl())
+ .allowMultiPartPost(random().nextBoolean())
+ .withRequestWriter(cborRequestWriter())
+ .withResponseParser(cborResponseparser())
+ .build();
+ }
+
+ @Override
+ @Ignore
+ public void testQueryPerf() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testExampleConfig() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testAugmentFields() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testRawFields() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testUpdateRequestWithParameters() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testContentStreamRequest() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testStreamingRequest() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testMultiContentWriterRequest() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testMultiContentStreamRequest() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testLukeHandler() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testStatistics() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testFaceting() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testPivotFacets() {
+ /*Ignore*/
+ }
+
+ @Override
+ public void testPivotFacetsStats() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testPivotFacetsStatsNotSupported() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testPivotFacetsQueries() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testPivotFacetsRanges() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testPivotFacetsMissing() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testUpdateField() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testUpdateMultiValuedField() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testSetNullUpdates() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testSetNullUpdateOrder() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testQueryWithParams() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testChildDoctransformer() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testExpandComponent() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testMoreLikeThis() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testAddChildToChildFreeDoc() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testDeleteParentDoc() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testCommitWithinOnAdd() {
+ /*Ignore*/
+ }
+
+ @Override
+ @Ignore
+ public void testAddDelete() {
+ /*Ignore*/
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static RequestWriter cborRequestWriter() {
+ return new BinaryRequestWriter() {
+
+ @Override
+ public ContentWriter getContentWriter(SolrRequest<?> request) {
+ if (request instanceof UpdateRequest) {
+ UpdateRequest updateRequest = (UpdateRequest) request;
+ List<SolrInputDocument> docs = updateRequest.getDocuments();
+ if (docs == null || docs.isEmpty()) return super.getContentWriter(request);
+ return new ContentWriter() {
+ @Override
+ public void write(OutputStream os) throws IOException {
+
+ List<Map> mapDocs = new ArrayList<>();
+ for (SolrInputDocument doc : docs) {
+ mapDocs.add(doc.toMap(new LinkedHashMap<>()));
+ }
+
+ CBORFactory jf = new CBORFactory();
+ ObjectMapper cborMapper = new ObjectMapper(jf);
+ cborMapper.writeValue(os, mapDocs);
+ }
+
+ @Override
+ public String getContentType() {
+ return "application/cbor";
+ }
+ };
+ } else {
+ return super.getContentWriter(request);
+ }
+ }
+
+ @Override
+ public String getUpdateContentType() {
+ return "application/cbor";
+ }
+
+ @Override
+ public String getPath(SolrRequest<?> req) {
+ if (req instanceof UpdateRequest) {
+ UpdateRequest updateRequest = (UpdateRequest) req;
+ List<SolrInputDocument> docs = updateRequest.getDocuments();
+ if (docs == null || docs.isEmpty()) return super.getPath(req);
+ return "/update/cbor";
+ } else {
+ return super.getPath(req);
+ }
+ }
+ };
+ }
+
+ private static ResponseParser cborResponseparser() {
+ return new ResponseParser() {
+
+ @Override
+ public String getWriterType() {
+ return "cbor";
+ }
+
+ @Override
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public NamedList<Object> processResponse(InputStream b, String encoding) {
+ ObjectMapper objectMapper = new ObjectMapper(new CBORFactory());
+ try {
+ Map m = (Map) objectMapper.readValue(b, Object.class);
+ NamedList nl = new NamedList();
+ m.forEach(
+ (k, v) -> {
+ if (v instanceof Map) {
+ Map map = (Map) v;
+ if ("response".equals(k)) {
+ v = convertResponse((Map) v);
+ } else {
+ v = new NamedList(map);
+ }
+ }
+ nl.add(k.toString(), v);
+ });
+ return nl;
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);
+ }
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private SolrDocumentList convertResponse(Map m) {
+ List<Map> docs = (List<Map>) m.get("docs");
+ SolrDocumentList sdl = new SolrDocumentList();
+ Object o = m.get("numFound");
+ if (o != null) sdl.setNumFound(((Number) o).longValue());
+ o = m.get("start");
+ if (o != null) sdl.setStart(((Number) o).longValue());
+ o = m.get("numFoundExact");
+ if (o != null) sdl.setNumFoundExact((Boolean) o);
+
+ if (docs != null) {
+ for (Map doc : docs) {
+ SolrDocument sd = new SolrDocument();
+ doc.forEach((k, v) -> sd.addField(k.toString(), v));
+ sdl.add(sd);
+ }
+ }
+ return sdl;
+ }
+
+ @Override
+ public NamedList<Object> processResponse(Reader reader) {
+ return null;
+ }
+ };
+ }
+}
diff --git a/versions.lock b/versions.lock
index 3da09ea3622..15b5810d425 100644
--- a/versions.lock
+++ b/versions.lock
@@ -10,6 +10,7 @@ com.fasterxml.jackson:jackson-bom:2.14.2 (12 constraints: 42f8fc44)
com.fasterxml.jackson.core:jackson-annotations:2.14.2 (10 constraints: 95be250f)
com.fasterxml.jackson.core:jackson-core:2.14.2 (13 constraints: 3902ce18)
com.fasterxml.jackson.core:jackson-databind:2.14.2 (18 constraints: 78638fbe)
+com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.14.2 (2 constraints: 5f1c13f1)
com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.14.2 (1 constraints: b90ea866)
com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.14.2 (2 constraints: a52484e0)
com.fasterxml.woodstox:woodstox-core:6.5.0 (3 constraints: ab2868b8)
@@ -389,7 +390,6 @@ com.amazonaws:aws-java-sdk-core:1.12.15 (2 constraints: 501a5183)
com.amazonaws:aws-java-sdk-kms:1.12.15 (1 constraints: d60cb42a)
com.amazonaws:aws-java-sdk-s3:1.12.15 (1 constraints: e0125c30)
com.amazonaws:jmespath-java:1.12.15 (2 constraints: 501a5183)
-com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.14.2 (2 constraints: 5f1c13f1)
com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.14.2 (2 constraints: a619c912)
com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.14.2 (2 constraints: 0b24ff81)
com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2 (3 constraints: 7c3ddb0f)