You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2024/03/20 14:07:29 UTC
(camel) 01/02: CAMEL-20587 - Camel-Qdrant: Add a datatype for transforming langchain embeddings in qdrant objects
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch CAMEL-20587
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 36855245022c07e5d018999e5a7a30bab1986072
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Wed Mar 20 14:35:33 2024 +0100
CAMEL-20587 - Camel-Qdrant: Add a datatype for transforming langchain embeddings in qdrant objects
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
.../apache/camel/catalog/components/qdrant.json | 2 +-
.../apache/camel/catalog/transformers.properties | 1 +
.../catalog/transformers/qdrant-embeddings.json | 14 +++++
...LangchainEmbeddingsComponentQdrantTargetIT.java | 7 +--
.../embeddings/LangchainEmbeddingsTestSupport.java | 31 -----------
components/camel-qdrant/pom.xml | 6 +++
.../org/apache/camel/component/qdrant/qdrant.json | 2 +-
.../org/apache/camel/transformer.properties | 7 +++
.../org/apache/camel/transformer/qdrant-embeddings | 2 +
.../camel/transformer/qdrant-embeddings.json | 14 +++++
.../org/apache/camel/component/qdrant/Qdrant.java | 3 ++
.../QdrantEmbeddingsDataTypeTransformer.java | 62 ++++++++++++++++++++++
.../endpoint/dsl/QdrantEndpointBuilderFactory.java | 2 +-
13 files changed, 116 insertions(+), 37 deletions(-)
diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/qdrant.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/qdrant.json
index 40d6552be81..d1195a204ad 100644
--- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/qdrant.json
+++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/qdrant.json
@@ -41,7 +41,7 @@
"CamelQdrantReadConsistency": { "index": 5, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "io.qdrant.client.grpc.Points$ReadConsistency", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Read Consistency.", "constantName": "org.apache.camel.component.qdrant.Qdrant$Headers#READ_CONSISTENCY" },
"CamelQdrantWithPayload": { "index": 6, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "defaultValue": "true", "description": "Include Payload.", "constantName": "org.apache.camel.component.qdrant.Qdrant$Headers#INCLUDE_PAYLOAD" },
"CamelQdrantWithVectors": { "index": 7, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "defaultValue": "false", "description": "Include Vectors.", "constantName": "org.apache.camel.component.qdrant.Qdrant$Headers#INCLUDE_VECTORS" },
- "CamelQdrantSize": { "index": 8, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The number of elements.", "constantName": "org.apache.camel.component.qdrant.Qdrant$Headers#SIZE" }
+ "CamelQdrantSize": { "index": 9, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The point id to use for operation.", "constantName": "org.apache.camel.component.qdrant.Qdrant$Headers#POINT_ID" }
},
"properties": {
"collection": { "index": 0, "kind": "path", "displayName": "Collection", "group": "producer", "label": "", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The collection Name" },
diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers.properties b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers.properties
index c29b8c75b35..2b30b4fbb10 100644
--- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers.properties
+++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers.properties
@@ -26,5 +26,6 @@ http-application-cloudevents
protobuf-binary
protobuf-x-java-object
protobuf-x-struct
+qdrant-embeddings
slack-application-cloudevents
text-plain
diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/qdrant-embeddings.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/qdrant-embeddings.json
new file mode 100644
index 00000000000..9f82859c76c
--- /dev/null
+++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/qdrant-embeddings.json
@@ -0,0 +1,14 @@
+{
+ "transformer": {
+ "kind": "transformer",
+ "name": "qdrant:embeddings",
+ "title": "Qdrant (Embeddings)",
+ "description": "Prepares the message to become an object writable by Qdrant component",
+ "deprecated": false,
+ "javaType": "org.apache.camel.component.qdrant.transform.QdrantEmbeddingsDataTypeTransformer",
+ "groupId": "org.apache.camel",
+ "artifactId": "camel-qdrant",
+ "version": "4.5.0-SNAPSHOT"
+ }
+}
+
diff --git a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentQdrantTargetIT.java b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentQdrantTargetIT.java
index 03bc3640836..445453b2cf2 100644
--- a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentQdrantTargetIT.java
+++ b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentQdrantTargetIT.java
@@ -28,6 +28,7 @@ import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.component.qdrant.Qdrant;
import org.apache.camel.component.qdrant.QdrantAction;
import org.apache.camel.component.qdrant.QdrantComponent;
+import org.apache.camel.spi.DataType;
import org.apache.camel.test.infra.qdrant.services.QdrantService;
import org.apache.camel.test.infra.qdrant.services.QdrantServiceFactory;
import org.apache.camel.test.junit5.CamelTestSupport;
@@ -38,7 +39,6 @@ import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.api.extension.RegisterExtension;
-import static org.apache.camel.component.langchain.embeddings.LangchainEmbeddingsTestSupport.POINT_ID_HEADER;
import static org.assertj.core.api.Assertions.assertThat;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
@@ -111,8 +111,9 @@ public class LangchainEmbeddingsComponentQdrantTargetIT extends CamelTestSupport
from("direct:in")
.to("langchain-embeddings:test")
.setHeader(Qdrant.Headers.ACTION).constant(QdrantAction.UPSERT)
- .setHeader(POINT_ID_HEADER).constant(POINT_ID)
- .bean(LangchainEmbeddingsTestSupport.AsPointStruct.class)
+ .setHeader(Qdrant.Headers.POINT_ID).constant(POINT_ID)
+ .transform(
+ new DataType("qdrant:embeddings"))
.to(QDRANT_URI);
}
};
diff --git a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java
index 326c456ef80..f7781284dae 100644
--- a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java
+++ b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java
@@ -21,46 +21,15 @@ import java.util.List;
import com.google.protobuf.InvalidProtocolBufferException;
import dev.langchain4j.data.embedding.Embedding;
-import dev.langchain4j.data.segment.TextSegment;
import io.milvus.param.dml.InsertParam;
-import io.qdrant.client.ValueFactory;
-import io.qdrant.client.VectorsFactory;
-import io.qdrant.client.grpc.Points;
import org.apache.camel.Exchange;
import org.apache.camel.Handler;
public class LangchainEmbeddingsTestSupport {
- public static final String POINT_ID_HEADER = "point.id";
private LangchainEmbeddingsTestSupport() {
}
- // TODO: this should be turned into a proper converter or data type
- public static class AsPointStruct {
- @Handler
- public Points.PointStruct asPointStruct(Exchange e) {
- Embedding embedding = e.getMessage().getHeader(LangchainEmbeddings.Headers.VECTOR, Embedding.class);
- TextSegment text = e.getMessage().getBody(TextSegment.class);
- Points.PointId id = e.getMessage().getHeader(POINT_ID_HEADER, Points.PointId.class);
-
- var builder = Points.PointStruct.newBuilder();
- builder.setId(id);
- builder.setVectors(VectorsFactory.vectors(embedding.vector()));
-
- if (text != null) {
- // this is the default for langchain4j
- // https://github.com/langchain4j/langchain4j/blob/3e432486ffc5cb80861e118cbc974f478c3accfc/langchain4j-qdrant/src/main/java/dev/langchain4j/store/embedding/qdrant/QdrantEmbeddingStore.java#L261
- builder.putPayload("text_segment", ValueFactory.value(text.text()));
-
- text.metadata()
- .asMap()
- .forEach((key, value) -> builder.putPayload(key, ValueFactory.value(value)));
- }
-
- return builder.build();
- }
- }
-
public static class AsInsertParam {
@Handler
public InsertParam AsInsertParam(Exchange e) throws InvalidProtocolBufferException {
diff --git a/components/camel-qdrant/pom.xml b/components/camel-qdrant/pom.xml
index e7c6530c5ca..324eb666326 100644
--- a/components/camel-qdrant/pom.xml
+++ b/components/camel-qdrant/pom.xml
@@ -48,6 +48,12 @@
<artifactId>camel-support</artifactId>
</dependency>
+ <dependency>
+ <groupId>dev.langchain4j</groupId>
+ <artifactId>langchain4j-core</artifactId>
+ <version>${langchain4j.version}</version>
+ </dependency>
+
<dependency>
<groupId>io.qdrant</groupId>
<artifactId>client</artifactId>
diff --git a/components/camel-qdrant/src/generated/resources/META-INF/org/apache/camel/component/qdrant/qdrant.json b/components/camel-qdrant/src/generated/resources/META-INF/org/apache/camel/component/qdrant/qdrant.json
index 40d6552be81..d1195a204ad 100644
--- a/components/camel-qdrant/src/generated/resources/META-INF/org/apache/camel/component/qdrant/qdrant.json
+++ b/components/camel-qdrant/src/generated/resources/META-INF/org/apache/camel/component/qdrant/qdrant.json
@@ -41,7 +41,7 @@
"CamelQdrantReadConsistency": { "index": 5, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "io.qdrant.client.grpc.Points$ReadConsistency", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Read Consistency.", "constantName": "org.apache.camel.component.qdrant.Qdrant$Headers#READ_CONSISTENCY" },
"CamelQdrantWithPayload": { "index": 6, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "defaultValue": "true", "description": "Include Payload.", "constantName": "org.apache.camel.component.qdrant.Qdrant$Headers#INCLUDE_PAYLOAD" },
"CamelQdrantWithVectors": { "index": 7, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "defaultValue": "false", "description": "Include Vectors.", "constantName": "org.apache.camel.component.qdrant.Qdrant$Headers#INCLUDE_VECTORS" },
- "CamelQdrantSize": { "index": 8, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The number of elements.", "constantName": "org.apache.camel.component.qdrant.Qdrant$Headers#SIZE" }
+ "CamelQdrantSize": { "index": 9, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The point id to use for operation.", "constantName": "org.apache.camel.component.qdrant.Qdrant$Headers#POINT_ID" }
},
"properties": {
"collection": { "index": 0, "kind": "path", "displayName": "Collection", "group": "producer", "label": "", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The collection Name" },
diff --git a/components/camel-qdrant/src/generated/resources/META-INF/services/org/apache/camel/transformer.properties b/components/camel-qdrant/src/generated/resources/META-INF/services/org/apache/camel/transformer.properties
new file mode 100644
index 00000000000..f5bf8155027
--- /dev/null
+++ b/components/camel-qdrant/src/generated/resources/META-INF/services/org/apache/camel/transformer.properties
@@ -0,0 +1,7 @@
+# Generated by camel build tools - do NOT edit this file!
+transformers=qdrant:embeddings
+groupId=org.apache.camel
+artifactId=camel-qdrant
+version=4.5.0-SNAPSHOT
+projectName=Camel :: Qdrant
+projectDescription=Camel Qdrant support
diff --git a/components/camel-qdrant/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings b/components/camel-qdrant/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings
new file mode 100644
index 00000000000..1d2c114407c
--- /dev/null
+++ b/components/camel-qdrant/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings
@@ -0,0 +1,2 @@
+# Generated by camel build tools - do NOT edit this file!
+class=org.apache.camel.component.qdrant.transform.QdrantEmbeddingsDataTypeTransformer
diff --git a/components/camel-qdrant/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings.json b/components/camel-qdrant/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings.json
new file mode 100644
index 00000000000..9f82859c76c
--- /dev/null
+++ b/components/camel-qdrant/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings.json
@@ -0,0 +1,14 @@
+{
+ "transformer": {
+ "kind": "transformer",
+ "name": "qdrant:embeddings",
+ "title": "Qdrant (Embeddings)",
+ "description": "Prepares the message to become an object writable by Qdrant component",
+ "deprecated": false,
+ "javaType": "org.apache.camel.component.qdrant.transform.QdrantEmbeddingsDataTypeTransformer",
+ "groupId": "org.apache.camel",
+ "artifactId": "camel-qdrant",
+ "version": "4.5.0-SNAPSHOT"
+ }
+}
+
diff --git a/components/camel-qdrant/src/main/java/org/apache/camel/component/qdrant/Qdrant.java b/components/camel-qdrant/src/main/java/org/apache/camel/component/qdrant/Qdrant.java
index 9650f6d02ff..6ce2b3b47de 100644
--- a/components/camel-qdrant/src/main/java/org/apache/camel/component/qdrant/Qdrant.java
+++ b/components/camel-qdrant/src/main/java/org/apache/camel/component/qdrant/Qdrant.java
@@ -53,5 +53,8 @@ public class Qdrant {
@Metadata(description = "The number of elements.", javaType = "int")
public static final String SIZE = "CamelQdrantSize";
+
+ @Metadata(description = "The point id to use for operation.", javaType = "int")
+ public static final String POINT_ID = "CamelQdrantSize";
}
}
diff --git a/components/camel-qdrant/src/main/java/org/apache/camel/component/qdrant/transform/QdrantEmbeddingsDataTypeTransformer.java b/components/camel-qdrant/src/main/java/org/apache/camel/component/qdrant/transform/QdrantEmbeddingsDataTypeTransformer.java
new file mode 100644
index 00000000000..5e5cb5af2d0
--- /dev/null
+++ b/components/camel-qdrant/src/main/java/org/apache/camel/component/qdrant/transform/QdrantEmbeddingsDataTypeTransformer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.camel.component.qdrant.transform;
+
+import java.util.UUID;
+
+import dev.langchain4j.data.embedding.Embedding;
+import dev.langchain4j.data.segment.TextSegment;
+import io.qdrant.client.PointIdFactory;
+import io.qdrant.client.ValueFactory;
+import io.qdrant.client.VectorsFactory;
+import io.qdrant.client.grpc.Points;
+import org.apache.camel.Message;
+import org.apache.camel.component.qdrant.Qdrant;
+import org.apache.camel.spi.DataType;
+import org.apache.camel.spi.DataTypeTransformer;
+import org.apache.camel.spi.Transformer;
+
+/**
+ * Maps a Langchain Embeddings to a Qdrant PointStruct to write an embeddings vector on a Qdrant Database.
+ */
+@DataTypeTransformer(name = "qdrant:embeddings",
+ description = "Prepares the message to become an object writable by Qdrant component")
+public class QdrantEmbeddingsDataTypeTransformer extends Transformer {
+
+ @Override
+ public void transform(Message message, DataType fromType, DataType toType) {
+ Embedding embedding = message.getHeader("CamelLangchainEmbeddingsVector", Embedding.class);
+ TextSegment text = message.getBody(TextSegment.class);
+ Points.PointId id
+ = message.getHeader(Qdrant.Headers.POINT_ID, () -> PointIdFactory.id(UUID.randomUUID()), Points.PointId.class);
+
+ var builder = Points.PointStruct.newBuilder();
+ builder.setId(id);
+ builder.setVectors(VectorsFactory.vectors(embedding.vector()));
+
+ if (text != null) {
+ builder.putPayload("text_segment", ValueFactory.value(text.text()));
+
+ text.metadata()
+ .asMap()
+ .forEach((key, value) -> builder.putPayload(key, ValueFactory.value(value)));
+ }
+
+ message.setBody(builder.build());
+ }
+}
diff --git a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/QdrantEndpointBuilderFactory.java b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/QdrantEndpointBuilderFactory.java
index 085a0a2daf1..63eb66b1023 100644
--- a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/QdrantEndpointBuilderFactory.java
+++ b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/QdrantEndpointBuilderFactory.java
@@ -399,7 +399,7 @@ public interface QdrantEndpointBuilderFactory {
}
/**
- * The number of elements.
+ * The point id to use for operation.
*
* The option is a: {@code int} type.
*