You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mk...@apache.org on 2025/05/08 10:26:11 UTC
(jackrabbit-oak) 06/08: OAK-11694: Added enricher status in document
This is an automated email from the ASF dual-hosted git repository.
mkataria pushed a commit to branch OAK-11694_1
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 6389c4161b6f60fc1147c7554cbaec627396f99c
Author: Mohit Kataria <ti...@gmail.com>
AuthorDate: Wed May 7 20:01:29 2025 +0530
OAK-11694: Added enricher status in document
---
.../index/elastic/index/ElasticIndexWriter.java | 38 +---
.../inference/ElasticInferenceUsingConfigTest.java | 249 +++++++++++++++++++++
2 files changed, 250 insertions(+), 37 deletions(-)
diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java
index a5313cd291..29011cf6d7 100644
--- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java
+++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java
@@ -160,43 +160,7 @@ class ElasticIndexWriter implements FulltextIndexWriter<ElasticDocument> {
if (InferenceConfig.getInstance().isInferenceEnabled()
&& InferenceConfig.getInstance().getInferenceIndexConfig(jcrIndexName).isEnabled()) {
doc.addProperty(InferenceConstants.ENRICH_NODE,
- Map.of(InferenceConstants.ENRICH_STATUS, InferenceConstants.ENRICH_STATUS_PENDING));
- }
- /*
-
- Once inference is enabled, it is not trivial to disable it. As inference configuration in Elasticsearch (ES)
- is persisted only during the creation of a new index
- or reindexing of an existing one. This means that the enricher configuration is updated only under
- these conditions. If we want to disable inference on instance, the existing enricher configuration
- remains unchanged, and the enricher will continue processing new documents.
-
- To stop the enricher from processing documents, we need to explicitly update the enricher status to
- `COMPLETED` in the ES document by adding the following structure:
- {
- :enrich {
- "status": "COMPLETED",
- "inferenceDisabled": true
- }
- }
-
- The `inferenceDisabled` flag is added to allow for potential evaluations at a later stage.
-
- This should happen in all cases where we try to disable inference i.e.
-
- 1. Inference is disabled in ElasticIndexProviderService but InferenceConfig is valid.
- 2. Inference is enabled and InferenceConfig is not equal to InferenceConfig.NOOP i.e.
- any of the properties is different from below:
- enricherConfig = "";
- isEnabled = false;
- inferenceModelConfigs = Map.of();
- Note: This is possible by not setting enricherConfig to empty string as other fields are set to default values.
- */
- else {
- Map<String, Object> enrichDocStatus = Map.of(
- InferenceConstants.ENRICH_STATUS, InferenceConstants.ENRICH_STATUS_COMPLETED,
- InferenceConstants.ENRICH_STATUS_INFERENCE_DISABLED, true
- );
- doc.addProperty(InferenceConstants.ENRICH_NODE, enrichDocStatus);
+ InferenceConfig.getInstance().getEnricherStatus());
}
bulkProcessorHandler.update(indexName, ElasticIndexUtils.idFromPath(path), doc);
}
diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/inference/ElasticInferenceUsingConfigTest.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/inference/ElasticInferenceUsingConfigTest.java
index 046ca29ee1..58c0d8ebfe 100644
--- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/inference/ElasticInferenceUsingConfigTest.java
+++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/inference/ElasticInferenceUsingConfigTest.java
@@ -336,4 +336,253 @@ public class ElasticInferenceUsingConfigTest extends ElasticAbstractQueryTest {
assertNotNull(carsDocUpdated.get(InferenceConstants.VECTOR_SPACES));
}
+
+ @Test
+ public void testEnricherStatus() throws Exception {
+ String jcrIndexName = UUID.randomUUID().toString();
+ String inferenceServiceUrl = "http://localhost:" + wireMock.port() + "/v1/embeddings";
+ String inferenceModelConfigName = "ada-test-model";
+ String inferenceModelName = "text-embedding-ada-002";
+
+ // Create inference config with enricher information
+ createInferenceConfig(jcrIndexName, true, enricherConfig, inferenceModelConfigName,
+ inferenceModelName, inferenceServiceUrl,
+ 0.8, 1L, true, true);
+
+ // Create and set up the node with enricher status information
+ NodeBuilder rootBuilder = nodeStore.getRoot().builder();
+ NodeBuilder nodeBuilder = rootBuilder;
+ for (String path : PathUtils.elements(INFERENCE_CONFIG_PATH)) {
+ nodeBuilder = nodeBuilder.child(path);
+ }
+ // Add enricher status node
+ NodeBuilder enrichNodeBuilder = nodeBuilder.child(InferenceConstants.ENRICH_NODE);
+ enrichNodeBuilder.setProperty("lastUpdated", System.currentTimeMillis());
+ enrichNodeBuilder.setProperty("status", "active");
+ enrichNodeBuilder.setProperty("documentsProcessed", 100);
+ nodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ IndexDefinitionBuilder builder = createIndex();
+ builder.includedPaths("/content")
+ .indexRule("nt:base")
+ .property("title").propertyIndex().analyzed().nodeScopeIndex()
+ .property("description").propertyIndex().analyzed().nodeScopeIndex();
+
+ Tree index = setIndex(jcrIndexName, builder);
+ root.commit();
+
+ // Add content
+ Tree content = root.getTree("/").addChild("content");
+ Tree document = content.addChild("document");
+ document.setProperty("title", "Test Document");
+ document.setProperty("description", "This is a test document to verify enricher status is included in document updates.");
+ root.commit();
+
+ // Let the index catch up
+ assertEventually(() -> assertEquals(2, countDocuments(index)));
+
+ // Get the document and check that it has the enricher status
+ ObjectNode docNode = getDocument(index, "/content/document");
+ assertNotNull(docNode);
+
+ // Add another property to trigger an update
+ document.setProperty("updatedAt", Instant.now().toString());
+ root.commit();
+
+ // Let the index catch up with the update
+ assertEventually(() -> {
+ ObjectNode updatedDoc = getDocument(index, "/content/document");
+ assertNotNull(updatedDoc.get(InferenceConstants.ENRICH_NODE));
+ JsonNode enrichNode = updatedDoc.get(InferenceConstants.ENRICH_NODE);
+ assertNotNull(enrichNode);
+ assertNotNull(enrichNode.get("lastUpdated"));
+ assertEquals("active", enrichNode.get("status").asText());
+ assertEquals(100, enrichNode.get("documentsProcessed").asInt());
+ });
+ }
+
+ @Test
+ public void testEnricherStatusPreservedWithVectorEmbeddings() throws Exception {
+ String jcrIndexName = UUID.randomUUID().toString();
+ String inferenceServiceUrl = "http://localhost:" + wireMock.port() + "/v1/embeddings";
+ String inferenceModelConfigName = "ada-test-model";
+ String inferenceModelName = "text-embedding-ada-002";
+
+ // Create inference config with enricher information
+ createInferenceConfig(jcrIndexName, true, enricherConfig, inferenceModelConfigName,
+ inferenceModelName, inferenceServiceUrl,
+ 0.8, 1L, true, true);
+
+ // Create and set up the node with enricher status information
+ NodeBuilder rootBuilder = nodeStore.getRoot().builder();
+ NodeBuilder nodeBuilder = rootBuilder;
+ for (String path : PathUtils.elements(INFERENCE_CONFIG_PATH)) {
+ nodeBuilder = nodeBuilder.child(path);
+ }
+ // Add enricher status node
+ NodeBuilder enrichNode = nodeBuilder.child(InferenceConstants.ENRICH_NODE);
+ enrichNode.setProperty("lastUpdated", System.currentTimeMillis());
+ enrichNode.setProperty("status", "active");
+ enrichNode.setProperty("documentsProcessed", 100);
+ nodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ IndexDefinitionBuilder builder = createIndex();
+ builder.includedPaths("/content")
+ .indexRule("nt:base")
+ .property("title").propertyIndex().analyzed().nodeScopeIndex()
+ .property("description").propertyIndex().analyzed().nodeScopeIndex()
+ .property("updatedBy").propertyIndex();
+
+ Tree index = setIndex(jcrIndexName, builder);
+ root.commit();
+
+ // Add content
+ Tree content = root.getTree("/").addChild("content");
+ Tree document = content.addChild("document");
+ document.setProperty("title", "Test Document with Embeddings");
+ document.setProperty("description", "This is a test document that will have vector embeddings and enricher status.");
+ root.commit();
+
+ // Let the index catch up
+ assertEventually(() -> assertEquals(2, countDocuments(index)));
+
+ // Create an update with vector embeddings
+ ObjectMapper mapper = new JsonMapper();
+ ObjectNode updateDoc = mapper.createObjectNode();
+ List<Float> embeddings = List.of(0.1f, 0.2f, 0.3f, 0.4f, 0.5f);
+ VectorDocument vectorDocument = new VectorDocument(UUID.randomUUID().toString(), embeddings,
+ Map.of("updatedAt", Instant.now().toEpochMilli(), "model", inferenceModelName));
+ ObjectNode vectorSpacesNode = updateDoc.putObject(InferenceConstants.VECTOR_SPACES);
+ ArrayNode inferenceModelConfigNode = vectorSpacesNode.putArray(inferenceModelConfigName);
+ inferenceModelConfigNode.addPOJO(vectorDocument);
+
+ // Update the document with vector embeddings
+ updateDocument(index, "/content/document", updateDoc);
+
+ // Verify the document has the embeddings
+ assertEventually(() -> {
+ ObjectNode docWithEmbeddings = getDocument(index, "/content/document");
+ assertNotNull(docWithEmbeddings.get(InferenceConstants.VECTOR_SPACES));
+ JsonNode vectorSpaces = docWithEmbeddings.get(InferenceConstants.VECTOR_SPACES);
+ assertNotNull(vectorSpaces.get(inferenceModelConfigName));
+ });
+
+ // Now update a property to trigger another update which should preserve the embeddings
+ // and also add the enricher status
+ document.setProperty("updatedBy", "Test User");
+ root.commit();
+
+ // Verify the document still has embeddings and now has enricher status
+ assertEventually(() -> {
+ ObjectNode updatedDoc = getDocument(index, "/content/document");
+
+ // Check that the vector embeddings are preserved
+ assertNotNull(updatedDoc.get(InferenceConstants.VECTOR_SPACES));
+ JsonNode vectorSpaces = updatedDoc.get(InferenceConstants.VECTOR_SPACES);
+ assertNotNull(vectorSpaces.get(inferenceModelConfigName));
+
+ // Check that the enricher status is present
+ assertNotNull(updatedDoc.get(InferenceConstants.ENRICH_NODE));
+ JsonNode enrichNodeData = updatedDoc.get(InferenceConstants.ENRICH_NODE);
+ assertNotNull(enrichNodeData.get("lastUpdated"));
+ assertEquals("active", enrichNodeData.get("status").asText());
+ assertEquals(100, enrichNodeData.get("documentsProcessed").asInt());
+
+ // Check that the updated property is present
+ assertEquals("Test User", updatedDoc.get("updatedBy").asText());
+ });
+ }
+
+ @Test
+ public void testEnricherStatusOnReinitialization() throws Exception {
+ String jcrIndexName = UUID.randomUUID().toString();
+ String inferenceServiceUrl = "http://localhost:" + wireMock.port() + "/v1/embeddings";
+ String inferenceModelConfigName = "ada-test-model";
+ String inferenceModelName = "text-embedding-ada-002";
+
+ // Create inference config
+ createInferenceConfig(jcrIndexName, true, enricherConfig, inferenceModelConfigName,
+ inferenceModelName, inferenceServiceUrl,
+ 0.8, 1L, true, true);
+
+ // Create and set up the node with initial enricher status
+ NodeBuilder rootBuilder = nodeStore.getRoot().builder();
+ NodeBuilder nodeBuilder = rootBuilder;
+ for (String path : PathUtils.elements(INFERENCE_CONFIG_PATH)) {
+ nodeBuilder = nodeBuilder.child(path);
+ }
+ // Add enricher status node with initial values
+ NodeBuilder enrichNode = nodeBuilder.child(InferenceConstants.ENRICH_NODE);
+ long initialTime = System.currentTimeMillis();
+ enrichNode.setProperty("lastUpdated", initialTime);
+ enrichNode.setProperty("status", "initializing");
+ enrichNode.setProperty("documentsProcessed", 0);
+ nodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // Force reinitialization of InferenceConfig
+ InferenceConfig.reInitialize();
+
+ // Verify initial enricher status
+ Map<String, Object> initialStatus = InferenceConfig.getInstance().getEnricherStatus();
+ assertNotNull(initialStatus);
+ assertEquals(initialTime, initialStatus.get("lastUpdated"));
+ assertEquals("initializing", initialStatus.get("status"));
+ assertEquals(0L, initialStatus.get("documentsProcessed"));
+
+ // Create and set up the node with updated enricher status
+ rootBuilder = nodeStore.getRoot().builder();
+ nodeBuilder = rootBuilder;
+ for (String path : PathUtils.elements(INFERENCE_CONFIG_PATH)) {
+ nodeBuilder = nodeBuilder.child(path);
+ }
+ // Update enricher status node with new values
+ enrichNode = nodeBuilder.child(InferenceConstants.ENRICH_NODE);
+ long updatedTime = System.currentTimeMillis() + 1000; // Ensure it's different
+ enrichNode.setProperty("lastUpdated", updatedTime);
+ enrichNode.setProperty("status", "active");
+ enrichNode.setProperty("documentsProcessed", 200);
+ // Add a new property
+ enrichNode.setProperty("errorCount", 5);
+ nodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // Force reinitialization of InferenceConfig
+ InferenceConfig.reInitialize();
+
+ // Verify updated enricher status
+ Map<String, Object> updatedStatus = InferenceConfig.getInstance().getEnricherStatus();
+ assertNotNull(updatedStatus);
+ assertEquals(updatedTime, updatedStatus.get("lastUpdated"));
+ assertEquals("active", updatedStatus.get("status"));
+ assertEquals(200L, updatedStatus.get("documentsProcessed"));
+ assertEquals(5L, updatedStatus.get("errorCount"));
+
+ // Create an index and verify the enricher status gets included in document updates
+ IndexDefinitionBuilder builder = createIndex();
+ builder.includedPaths("/content")
+ .indexRule("nt:base")
+ .property("title").propertyIndex().analyzed().nodeScopeIndex();
+
+ Tree index = setIndex(jcrIndexName, builder);
+ root.commit();
+
+ // Add content
+ Tree content = root.getTree("/").addChild("content");
+ Tree document = content.addChild("document");
+ document.setProperty("title", "Test Document for Reinitialization");
+ root.commit();
+
+ // Let the index catch up
+ assertEventually(() -> assertEquals(2, countDocuments(index)));
+
+ // Verify the enricher status in the indexed document
+ assertEventually(() -> {
+ ObjectNode docNode = getDocument(index, "/content/document");
+ assertNotNull(docNode.get(InferenceConstants.ENRICH_NODE));
+ JsonNode enrichNodeData = docNode.get(InferenceConstants.ENRICH_NODE);
+ assertEquals(updatedTime, enrichNodeData.get("lastUpdated").asLong());
+ assertEquals("active", enrichNodeData.get("status").asText());
+ assertEquals(200, enrichNodeData.get("documentsProcessed").asInt());
+ assertEquals(5, enrichNodeData.get("errorCount").asInt());
+ });
+ }
}