You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2024/03/18 12:11:09 UTC
(camel) branch milvus-supp-langchain created (now 930ff7f9e87)
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a change to branch milvus-supp-langchain
in repository https://gitbox.apache.org/repos/asf/camel.git
at 930ff7f9e87 Camel-AI: Langchain Embeddings component testing with Camel Milvus
This branch includes the following new commits:
new 930ff7f9e87 Camel-AI: Langchain Embeddings component testing with Camel Milvus
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(camel) 01/01: Camel-AI: Langchain Embeddings component testing with Camel Milvus
Posted by ac...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch milvus-supp-langchain
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 930ff7f9e870d1d788886b8357643f390628cadd
Author: Andrea Cosentino <an...@gmail.com>
AuthorDate: Mon Mar 18 13:10:19 2024 +0100
Camel-AI: Langchain Embeddings component testing with Camel Milvus
Signed-off-by: Andrea Cosentino <an...@gmail.com>
---
.../camel-ai/camel-langchain-embeddings/pom.xml | 12 ++
...LangchainEmbeddingsComponentMilvusTargetIT.java | 191 +++++++++++++++++++++
.../embeddings/LangchainEmbeddingsTestSupport.java | 23 +++
3 files changed, 226 insertions(+)
diff --git a/components/camel-ai/camel-langchain-embeddings/pom.xml b/components/camel-ai/camel-langchain-embeddings/pom.xml
index d4a68ff9704..09ae4d316fe 100644
--- a/components/camel-ai/camel-langchain-embeddings/pom.xml
+++ b/components/camel-ai/camel-langchain-embeddings/pom.xml
@@ -57,6 +57,11 @@
<artifactId>camel-qdrant</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-milvus</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.camel</groupId>
@@ -98,6 +103,13 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.camel</groupId>
+ <artifactId>camel-test-infra-milvus</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java
new file mode 100644
index 00000000000..728914d789e
--- /dev/null
+++ b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.langchain.embeddings;
+
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import dev.langchain4j.model.embedding.AllMiniLmL6V2EmbeddingModel;
+import io.milvus.common.clientenum.ConsistencyLevelEnum;
+import io.milvus.grpc.DataType;
+import io.milvus.param.IndexType;
+import io.milvus.param.MetricType;
+import io.milvus.param.collection.CreateCollectionParam;
+import io.milvus.param.collection.FieldType;
+import io.milvus.param.highlevel.dml.SearchSimpleParam;
+import io.milvus.param.highlevel.dml.response.SearchResponse;
+import io.milvus.param.index.CreateIndexParam;
+import org.apache.camel.CamelContext;
+import org.apache.camel.Exchange;
+import org.apache.camel.RoutesBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.milvus.Milvus;
+import org.apache.camel.component.milvus.MilvusAction;
+import org.apache.camel.component.milvus.MilvusComponent;
+import org.apache.camel.test.infra.milvus.services.MilvusService;
+import org.apache.camel.test.infra.milvus.services.MilvusServiceFactory;
+import org.apache.camel.test.junit5.CamelTestSupport;
+import org.assertj.core.util.Lists;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class LangchainEmbeddingsComponentMilvusTargetIT extends CamelTestSupport {
+ public static final String MILVUS_URI = "milvus:embeddings";
+
+ @RegisterExtension
+ static MilvusService MILVUS = MilvusServiceFactory.createSingletonService();
+
+ @Override
+
+ protected CamelContext createCamelContext() throws Exception {
+ CamelContext context = super.createCamelContext();
+
+ var milvus = context.getComponent(Milvus.SCHEME, MilvusComponent.class);
+ URL url = new URL(MILVUS.getMilvusEndpointUrl());
+ milvus.getConfiguration().setHost(url.getHost());
+ milvus.getConfiguration().setPort(url.getPort());
+
+ context.getRegistry().bind("embedding-model", new AllMiniLmL6V2EmbeddingModel());
+
+ return context;
+ }
+
+ @Test
+ @Order(1)
+ public void createCollectionAndIndex() {
+ FieldType fieldType1 = FieldType.newBuilder()
+ .withName("userID")
+ .withDescription("user identification")
+ .withDataType(DataType.Int64)
+ .withPrimaryKey(true)
+ .withAutoID(true)
+ .build();
+
+ FieldType fieldType2 = FieldType.newBuilder()
+ .withName("vector")
+ .withDescription("face embedding")
+ .withDataType(DataType.FloatVector)
+ .withDimension(384)
+ .build();
+
+ CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder()
+ .withCollectionName("embeddings")
+ .withDescription("customer info")
+ .withShardsNum(2)
+ .withEnableDynamicField(false)
+ .addFieldType(fieldType1)
+ .addFieldType(fieldType2)
+ .build();
+
+ Exchange result = fluentTemplate.to(MILVUS_URI)
+ .withHeader(Milvus.Headers.ACTION, MilvusAction.CREATE_COLLECTION)
+ .withBody(
+ createCollectionReq)
+ .request(Exchange.class);
+
+ assertThat(result).isNotNull();
+ assertThat(result.getException()).isNull();
+
+ CreateIndexParam createVectorIndexParam = CreateIndexParam.newBuilder()
+ .withCollectionName("embeddings")
+ .withFieldName("vector")
+ .withIndexName("vectorIndex")
+ .withIndexType(IndexType.IVF_FLAT)
+ .withMetricType(MetricType.L2)
+ .withExtraParam("{\"nlist\":128}")
+ .withSyncMode(Boolean.TRUE)
+ .build();
+
+ result = fluentTemplate.to(MILVUS_URI)
+ .withHeader(Milvus.Headers.ACTION, MilvusAction.CREATE_INDEX)
+ .withBody(
+ createVectorIndexParam)
+ .request(Exchange.class);
+
+ assertThat(result).isNotNull();
+ assertThat(result.getException()).isNull();
+ }
+
+ @Test
+ @Order(2)
+ public void insert() {
+
+ Exchange result = fluentTemplate.to("direct:in")
+ .withBody("hi")
+ .request(Exchange.class);
+
+ assertThat(result).isNotNull();
+ assertThat(result.getException()).isNull();
+ }
+
+ @Test
+ @Order(3)
+ public void search() {
+ SearchSimpleParam searchSimpleParam = SearchSimpleParam.newBuilder()
+ .withCollectionName("embeddings")
+ .withVectors(generateFloatVector())
+ .withFilter("userID>=1")
+ .withLimit(100L)
+ .withOffset(0L)
+ .withOutputFields(Lists.newArrayList("userID"))
+ .withConsistencyLevel(ConsistencyLevelEnum.STRONG)
+ .build();
+ Exchange result = fluentTemplate.to(MILVUS_URI)
+ .withHeader(Milvus.Headers.ACTION, MilvusAction.SEARCH)
+ .withBody(searchSimpleParam)
+ .request(Exchange.class);
+
+ assertThat(result).isNotNull();
+ assertThat(result.getException()).isNull();
+
+ assertThat(result.getIn().getBody()).isInstanceOfSatisfying(SearchResponse.class,
+ c -> assertThat(c.rowRecords.size() == 1));
+ }
+
+ @Override
+ protected RoutesBuilder createRouteBuilder() {
+ return new RouteBuilder() {
+ public void configure() {
+ from("direct:in")
+ .to("langchain-embeddings:test")
+ .bean(LangchainEmbeddingsTestSupport.AsInsertParam.class)
+ .setHeader(Milvus.Headers.ACTION).constant(MilvusAction.INSERT)
+ .bean(LangchainEmbeddingsTestSupport.AsInsertParam.class)
+ .to(MILVUS_URI);
+ }
+ };
+ }
+
+ private List<Float> generateFloatVector() {
+ Random ran = new Random();
+ List<Float> vector = new ArrayList<>();
+ for (int i = 0; i < 384; ++i) {
+ vector.add(ran.nextFloat());
+ }
+ return vector;
+ }
+}
diff --git a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java
index 16b22347c8a..326c456ef80 100644
--- a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java
+++ b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java
@@ -16,8 +16,13 @@
*/
package org.apache.camel.component.langchain.embeddings;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.protobuf.InvalidProtocolBufferException;
import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.segment.TextSegment;
+import io.milvus.param.dml.InsertParam;
import io.qdrant.client.ValueFactory;
import io.qdrant.client.VectorsFactory;
import io.qdrant.client.grpc.Points;
@@ -55,4 +60,22 @@ public class LangchainEmbeddingsTestSupport {
return builder.build();
}
}
+
+ public static class AsInsertParam {
+ @Handler
+ public InsertParam AsInsertParam(Exchange e) throws InvalidProtocolBufferException {
+ Embedding embedding = e.getMessage().getHeader(LangchainEmbeddings.Headers.VECTOR, Embedding.class);
+ List<InsertParam.Field> fields = new ArrayList<>();
+ ArrayList list = new ArrayList<>();
+ list.add(embedding.vectorAsList());
+ fields.add(new InsertParam.Field("vector", list));
+
+ InsertParam insertParam = InsertParam.newBuilder()
+ .withCollectionName("embeddings")
+ .withFields(fields)
+ .build();
+
+ return insertParam;
+ }
+ }
}