You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2024/03/21 11:05:54 UTC
(camel) 01/02: CAMEL-20594 - Camel-Milvus: Add a datatype for transforming langchain embeddings in Milvus objects
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch CAMEL-20586
in repository https://gitbox.apache.org/repos/asf/camel.git
commit ad1c284c0a2da09a453f892eb1b8ae7f1c051e68
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Thu Mar 21 11:49:58 2024 +0100
CAMEL-20594 - Camel-Milvus: Add a datatype for transforming langchain embeddings in Milvus objects
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
...LangchainEmbeddingsComponentMilvusTargetIT.java | 10 +++-
.../embeddings/LangchainEmbeddingsTestSupport.java | 50 ------------------
components/camel-milvus/pom.xml | 6 +++
.../org/apache/camel/component/milvus/milvus.json | 5 +-
.../org/apache/camel/transformer.properties | 7 +++
.../org/apache/camel/transformer/milvus-embeddings | 2 +
.../camel/transformer/milvus-embeddings.json | 14 +++++
.../org/apache/camel/transformer/qdrant-embeddings | 2 +
.../camel/transformer/qdrant-embeddings.json | 14 +++++
.../org/apache/camel/component/milvus/Milvus.java | 9 ++++
.../MilvusEmbeddingsDataTypeTransformer.java | 60 ++++++++++++++++++++++
11 files changed, 126 insertions(+), 53 deletions(-)
diff --git a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java
index 3ad16173bbe..516003c0aa6 100644
--- a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java
+++ b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java
@@ -90,6 +90,12 @@ public class LangchainEmbeddingsComponentMilvusTargetIT extends CamelTestSupport
.withDimension(384)
.build();
+ FieldType fieldType3 = FieldType.newBuilder()
+ .withName("text")
+ .withDataType(DataType.VarChar)
+ .withMaxLength(65535)
+ .build();
+
CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder()
.withCollectionName("embeddings")
.withDescription("customer info")
@@ -97,6 +103,7 @@ public class LangchainEmbeddingsComponentMilvusTargetIT extends CamelTestSupport
.withEnableDynamicField(false)
.addFieldType(fieldType1)
.addFieldType(fieldType2)
+ .addFieldType(fieldType3)
.build();
Exchange result = fluentTemplate.to(MILVUS_URI)
@@ -170,9 +177,8 @@ public class LangchainEmbeddingsComponentMilvusTargetIT extends CamelTestSupport
public void configure() {
from("direct:in")
.to("langchain-embeddings:test")
- .bean(LangchainEmbeddingsTestSupport.AsInsertParam.class)
.setHeader(Milvus.Headers.ACTION).constant(MilvusAction.INSERT)
- .bean(LangchainEmbeddingsTestSupport.AsInsertParam.class)
+ .transform(new org.apache.camel.spi.DataType("milvus:embeddings"))
.to(MILVUS_URI);
}
};
diff --git a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java
deleted file mode 100644
index f7781284dae..00000000000
--- a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.camel.component.langchain.embeddings;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import dev.langchain4j.data.embedding.Embedding;
-import io.milvus.param.dml.InsertParam;
-import org.apache.camel.Exchange;
-import org.apache.camel.Handler;
-
-public class LangchainEmbeddingsTestSupport {
-
- private LangchainEmbeddingsTestSupport() {
- }
-
- public static class AsInsertParam {
- @Handler
- public InsertParam AsInsertParam(Exchange e) throws InvalidProtocolBufferException {
- Embedding embedding = e.getMessage().getHeader(LangchainEmbeddings.Headers.VECTOR, Embedding.class);
- List<InsertParam.Field> fields = new ArrayList<>();
- ArrayList list = new ArrayList<>();
- list.add(embedding.vectorAsList());
- fields.add(new InsertParam.Field("vector", list));
-
- InsertParam insertParam = InsertParam.newBuilder()
- .withCollectionName("embeddings")
- .withFields(fields)
- .build();
-
- return insertParam;
- }
- }
-}
diff --git a/components/camel-milvus/pom.xml b/components/camel-milvus/pom.xml
index 7e707de0f28..f8a8e088c79 100644
--- a/components/camel-milvus/pom.xml
+++ b/components/camel-milvus/pom.xml
@@ -48,6 +48,12 @@
<artifactId>camel-support</artifactId>
</dependency>
+ <dependency>
+ <groupId>dev.langchain4j</groupId>
+ <artifactId>langchain4j-core</artifactId>
+ <version>${langchain4j.version}</version>
+ </dependency>
+
<dependency>
<groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId>
diff --git a/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json b/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json
index 4e470ce0c9c..014942d2d91 100644
--- a/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json
+++ b/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json
@@ -34,7 +34,10 @@
"headers": {
"CamelMilvusAction": { "index": 0, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "enum": [ "CREATE_COLLECTION", "CREATE_INDEX", "UPSERT", "INSERT", "SEARCH", "DELETE" ], "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The action to be performed.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#ACTION" },
"CamelMilvusOperationStatus": { "index": 1, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Operation Status.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#OPERATION_STATUS" },
- "CamelMilvusOperationStatusValue": { "index": 2, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Operation Status Value.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#OPERATION_STATUS_VALUE" }
+ "CamelMilvusOperationStatusValue": { "index": 2, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Operation Status Value.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#OPERATION_STATUS_VALUE" },
+ "CamelMilvusTextFieldName": { "index": 3, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Text Field Name for Insert\/Upsert operation", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#TEXT_FIELD_NAME" },
+ "CamelMilvusVectorFieldName": { "index": 4, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Vector Field Name for Insert\/Upsert operation", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#VECTOR_FIELD_NAME" },
+ "CamelMilvusCollectionName": { "index": 5, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Collection Name for Insert\/Upsert operation", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#COLLECTION_NAME" }
},
"properties": {
"collection": { "index": 0, "kind": "path", "displayName": "Collection", "group": "producer", "label": "", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The collection Name" },
diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer.properties b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer.properties
new file mode 100644
index 00000000000..de216dd7f50
--- /dev/null
+++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer.properties
@@ -0,0 +1,7 @@
+# Generated by camel build tools - do NOT edit this file!
+transformers=milvus:embeddings
+groupId=org.apache.camel
+artifactId=camel-milvus
+version=4.5.0-SNAPSHOT
+projectName=Camel :: Milvus
+projectDescription=Camel Milvus support
diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings
new file mode 100644
index 00000000000..ccd7a4be1ed
--- /dev/null
+++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings
@@ -0,0 +1,2 @@
+# Generated by camel build tools - do NOT edit this file!
+class=org.apache.camel.component.milvus.transform.MilvusEmbeddingsDataTypeTransformer
diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings.json b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings.json
new file mode 100644
index 00000000000..e04d45f50c7
--- /dev/null
+++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings.json
@@ -0,0 +1,14 @@
+{
+ "transformer": {
+ "kind": "transformer",
+ "name": "milvus:embeddings",
+ "title": "Milvus (Embeddings)",
+ "description": "Prepares the message to become an object writable by Milvus component",
+ "deprecated": false,
+ "javaType": "org.apache.camel.component.milvus.transform.MilvusEmbeddingsDataTypeTransformer",
+ "groupId": "org.apache.camel",
+ "artifactId": "camel-milvus",
+ "version": "4.5.0-SNAPSHOT"
+ }
+}
+
diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings
new file mode 100644
index 00000000000..ccd7a4be1ed
--- /dev/null
+++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings
@@ -0,0 +1,2 @@
+# Generated by camel build tools - do NOT edit this file!
+class=org.apache.camel.component.milvus.transform.MilvusEmbeddingsDataTypeTransformer
diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings.json b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings.json
new file mode 100644
index 00000000000..2eaeb37bfa7
--- /dev/null
+++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings.json
@@ -0,0 +1,14 @@
+{
+ "transformer": {
+ "kind": "transformer",
+ "name": "qdrant:embeddings",
+ "title": "Qdrant (Embeddings)",
+ "description": "Prepares the message to become an object writable by Milvus component",
+ "deprecated": false,
+ "javaType": "org.apache.camel.component.milvus.transform.MilvusEmbeddingsDataTypeTransformer",
+ "groupId": "org.apache.camel",
+ "artifactId": "camel-milvus",
+ "version": "4.5.0-SNAPSHOT"
+ }
+}
+
diff --git a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java
index ebc92861673..7c2fc3bb7f1 100644
--- a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java
+++ b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java
@@ -34,5 +34,14 @@ public class Milvus {
@Metadata(description = "Operation Status Value.", javaType = "int")
public static final String OPERATION_STATUS_VALUE = "CamelMilvusOperationStatusValue";
+
+ @Metadata(description = "Text Field Name for Insert/Upsert operation", javaType = "String")
+ public static final String TEXT_FIELD_NAME = "CamelMilvusTextFieldName";
+
+ @Metadata(description = "Vector Field Name for Insert/Upsert operation", javaType = "String")
+ public static final String VECTOR_FIELD_NAME = "CamelMilvusVectorFieldName";
+
+ @Metadata(description = "Collection Name for Insert/Upsert operation", javaType = "String")
+ public static final String COLLECTION_NAME = "CamelMilvusCollectionName";
}
}
diff --git a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/transform/MilvusEmbeddingsDataTypeTransformer.java b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/transform/MilvusEmbeddingsDataTypeTransformer.java
new file mode 100644
index 00000000000..1aed3d370f7
--- /dev/null
+++ b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/transform/MilvusEmbeddingsDataTypeTransformer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.camel.component.milvus.transform;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import dev.langchain4j.data.embedding.Embedding;
+import dev.langchain4j.data.segment.TextSegment;
+import io.milvus.param.dml.InsertParam;
+import org.apache.camel.Message;
+import org.apache.camel.component.milvus.Milvus;
+import org.apache.camel.spi.DataType;
+import org.apache.camel.spi.DataTypeTransformer;
+import org.apache.camel.spi.Transformer;
+
+/**
+ * Maps a Langchain Embeddings to a Milvus InsertParam/Upsert Param to write an embeddings vector on a Milvus Database.
+ */
+@DataTypeTransformer(name = "milvus:embeddings",
+ description = "Prepares the message to become an object writable by Milvus component")
+public class MilvusEmbeddingsDataTypeTransformer extends Transformer {
+
+ @Override
+ public void transform(Message message, DataType fromType, DataType toType) {
+ Embedding embedding = message.getHeader("CamelLangchainEmbeddingsVector", Embedding.class);
+ String textFieldName = message.getHeader(Milvus.Headers.TEXT_FIELD_NAME, () -> "text", String.class);
+ String vectorFieldName = message.getHeader(Milvus.Headers.VECTOR_FIELD_NAME, () -> "vector", String.class);
+ String collectionName = message.getHeader(Milvus.Headers.COLLECTION_NAME, () -> "embeddings", String.class);
+ TextSegment text = message.getBody(TextSegment.class);
+ List<InsertParam.Field> fields = new ArrayList<>();
+ ArrayList list = new ArrayList<>();
+ list.add(embedding.vectorAsList());
+ fields.add(new InsertParam.Field(vectorFieldName, list));
+ fields.add(new InsertParam.Field(textFieldName, Collections.singletonList(text.text())));
+
+ InsertParam insertParam = InsertParam.newBuilder()
+ .withCollectionName(collectionName)
+ .withFields(fields)
+ .build();
+
+ message.setBody(insertParam);
+ }
+}