You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by pi...@apache.org on 2026/09/08 15:52:52 UTC
(atlas) branch ATLAS-4796_u created (now 02ed24a16)
This is an automated email from the ASF dual-hosted git repository.
pinal-shah pushed a change to branch ATLAS-4796_u
in repository https://gitbox.apache.org/repos/asf/atlas.git
at 02ed24a16 ATLAS-4796: Add OpenSearch integration tests and CI
This branch includes the following new commits:
new ec7d9204d ATLAS-4796: Support OpenSearch as mixed index backend
new 02ed24a16 ATLAS-4796: Add OpenSearch integration tests and CI
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(atlas) 02/02: ATLAS-4796: Add OpenSearch integration tests and CI
Posted by pi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pinal-shah pushed a commit to branch ATLAS-4796_u
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 02ed24a16ccb4fbcccc45c4d2b505b9d027c294a
Author: Pinal Shah <pi...@apache.org>
AuthorDate: Tue Sep 8 21:19:31 2026 +0530
ATLAS-4796: Add OpenSearch integration tests and CI
Add Testcontainers-based ITs, test utilities, updated unit tests,
and CI workflow for OpenSearch 3.7/3.8 verification.
---
.github/workflows/ci.yml | 80 +++
.../janus/AtlasJanusGraphIndexClientTest.java | 86 +++
.../janus/AtlasJanusGraphManagementTest.java | 139 ++++
.../janus/AtlasOpenSearchIndexClientTest.java | 302 +++++++++
.../janus/AtlasOpenSearchQueryBuilderTest.java | 367 +++++++++++
.../janus/OpenSearchAtlasJanusTestSupport.java | 53 ++
.../janus/OpenSearchClassificationFilterIT.java | 219 +++++++
.../janus/OpenSearchQueryCaptureDriver.java | 142 ++++
.../graphdb/janus/OpenSearchRuntimeInitTest.java | 163 +++++
.../janus/OpenSearchStateKeywordMappingIT.java | 176 +++++
.../AtlasOpenSearchIndexInstanceTest.java | 66 ++
.../atlas-opensearch-application.properties | 33 +
.../OpenSearchIndexIdempotentCreateTest.java | 129 ++++
.../OpenSearchIndexScrollWiringTest.java | 160 +++++
.../opensearch/OpenSearchScrollCleanupTest.java | 163 +++++
.../rest/OpenSearchBulkWireFormatVerifier.java | 83 +++
.../OpenSearchConcurrentIndexCreationDriver.java | 132 ++++
.../smoke/OpenSearchConcurrentIndexCreationIT.java | 32 +
.../smoke/OpenSearchInitSmokeDriver.java | 103 +++
.../smoke/OpenSearchMixedIndexLifecycleDriver.java | 120 ++++
.../smoke/OpenSearchMixedIndexLifecycleIT.java | 32 +
.../OpenSearchMixedIndexMutationQueryDriver.java | 499 ++++++++++++++
.../smoke/OpenSearchMixedIndexMutationQueryIT.java | 32 +
.../opensearch/smoke/OpenSearchSmokeSupport.java | 130 ++++
.../discovery/smoke/OpenSearchAggregationsIT.java | 34 +
.../OpenSearchAggregationsValidationDriver.java | 599 +++++++++++++++++
.../discovery/smoke/OpenSearchQuickSearchIT.java | 34 +
.../smoke/OpenSearchQuickSearchSmokeSupport.java | 227 +++++++
.../OpenSearchQuickSearchValidationDriver.java | 475 ++++++++++++++
.../OpenSearchSearchWeightBaselineDriver.java | 720 +++++++++++++++++++++
.../discovery/smoke/OpenSearchSearchWeightIT.java | 34 +
.../OpenSearchSearchWeightValidationDriver.java | 466 +++++++++++++
.../discovery/smoke/OpenSearchSuggestionsIT.java | 34 +
.../OpenSearchSuggestionsValidationDriver.java | 416 ++++++++++++
.../atlas-opensearch-c5-application.properties | 27 +
.../org/apache/atlas/runner/OpenSearchITBase.java | 40 ++
.../runner/OpenSearchTestContainerRunner.java | 135 ++++
.../runner/OpenSearchTestContainerRunnerTest.java | 105 +++
38 files changed, 6787 insertions(+)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 909843307..0c2ced04e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -158,3 +158,83 @@ jobs:
docker stop $(docker ps -q) && docker rm $(docker ps -aq);
exit 1;
fi
+
+ # OpenSearch integration tests run against a real OpenSearch server provisioned by Testcontainers (Docker).
+ # This job keeps the Solr-based docker-build job above unchanged and adds OpenSearch backend coverage.
+ opensearch-it:
+ runs-on: ubuntu-22.04
+ timeout-minutes: 120
+
+ steps:
+ - uses: actions/checkout@v6
+
+ - name: Delete huge unnecessary tools folder
+ run: rm -rf /opt/hostedtoolcache
+
+ - name: Cache for maven dependencies
+ uses: actions/cache@v5
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/atlas
+ key: maven-repo-osit-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ maven-repo-osit-
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v5
+ with:
+ java-version: 17
+ distribution: 'temurin'
+
+ - name: Set MAVEN_OPTS for JDK 17
+ run: |
+ echo "MAVEN_OPTS=-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED" >> $GITHUB_ENV
+
+ # Fail fast if the runner has no Docker daemon: OpenSearch ITs require Testcontainers and must NOT be
+ # silently skipped in CI.
+ - name: Verify Docker is available
+ run: |
+ docker version
+ docker info
+
+ - name: Build modules under test (skip tests)
+ run: |
+ mvn -pl graphdb/janusgraph-opensearch,graphdb/janus,repository -am \
+ -DskipTests -DskipDocs -Dcheckstyle.skip=true clean install --no-transfer-progress -B -V
+
+ # Default OpenSearch server version is 3.7.0; override the matrix here to also cover 3.8.x.
+ - name: Run OpenSearch integration tests (OpenSearch ${{ matrix.opensearch }})
+ run: |
+ mvn -Popensearch-it -pl graphdb/janusgraph-opensearch,graphdb/janus,repository \
+ -Dopensearch.docker.version=${{ matrix.opensearch }} \
+ -Dcheckstyle.skip=true test --no-transfer-progress -B
+
+ # A green build with zero OpenSearch tests executed is a false pass: assert reports exist and none failed.
+ - name: Assert OpenSearch integration tests actually executed
+ if: always()
+ run: |
+ set -euo pipefail
+ reports=$(find . -path '*/surefire-reports/*OpenSearch*IT.txt' | wc -l | tr -d ' ')
+ echo "OpenSearch IT surefire report files found: ${reports}"
+ if [ "${reports}" -eq 0 ]; then
+ echo "ERROR: no OpenSearch integration tests were executed"
+ exit 1
+ fi
+ skipped_all=true
+ while IFS= read -r f; do
+ if grep -qE 'Tests run: [1-9]' "$f" && ! grep -qE 'Skipped: [1-9]+, Tests run: 0' "$f"; then
+ skipped_all=false
+ fi
+ cat "$f"
+ done < <(find . -path '*/surefire-reports/*OpenSearch*IT.txt')
+ if [ "${skipped_all}" = true ]; then
+ echo "ERROR: OpenSearch integration tests were all skipped (Docker/container unavailable?)"
+ exit 1
+ fi
+
+ strategy:
+ fail-fast: false
+ matrix:
+ # Co-primary OpenSearch targets. 2.x compatibility can be added here once validated on a runner.
+ opensearch: [ "3.7.0", "3.8.0" ]
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClientTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClientTest.java
index 6a3662fd5..511d21ab9 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClientTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClientTest.java
@@ -21,6 +21,8 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AggregationContext;
+import org.apache.atlas.repository.graphdb.QuickSearchContext;
+import org.apache.atlas.repository.graphdb.QuickSearchResult;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.configuration2.Configuration;
import org.apache.solr.client.solrj.SolrClient;
@@ -36,6 +38,8 @@ import org.apache.solr.client.solrj.response.V2Response;
import org.apache.solr.common.util.NamedList;
import org.janusgraph.diskstorage.es.ElasticSearch7Index;
import org.janusgraph.diskstorage.es.ElasticSearchClient;
+import org.janusgraph.diskstorage.opensearch.AtlasOpenSearchIndex;
+import org.janusgraph.diskstorage.opensearch.OpenSearchClient;
import org.janusgraph.diskstorage.solr.Solr6Index;
import org.mockito.Mock;
import org.mockito.MockedStatic;
@@ -57,6 +61,7 @@ import java.util.Set;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
@@ -84,11 +89,14 @@ public class AtlasJanusGraphIndexClientTest {
@Mock
private ElasticSearchClient mockElasticSearchClient;
@Mock
+ private OpenSearchClient mockOpenSearchClient;
+ @Mock
private V2Response mockV2Response;
private AtlasJanusGraphIndexClient indexClient;
private MockedStatic<Solr6Index> mockedSolr6Index;
private MockedStatic<ElasticSearch7Index> mockedElasticSearch7Index;
+ private MockedStatic<AtlasOpenSearchIndex> mockedAtlasOpenSearchIndex;
private MockedStatic<ApplicationProperties> mockedApplicationProperties;
@BeforeMethod
@@ -99,6 +107,7 @@ public class AtlasJanusGraphIndexClientTest {
// Setup static mocks
mockedSolr6Index = Mockito.mockStatic(Solr6Index.class);
mockedElasticSearch7Index = Mockito.mockStatic(ElasticSearch7Index.class);
+ mockedAtlasOpenSearchIndex = Mockito.mockStatic(AtlasOpenSearchIndex.class);
mockedApplicationProperties = Mockito.mockStatic(ApplicationProperties.class);
}
@@ -110,6 +119,9 @@ public class AtlasJanusGraphIndexClientTest {
if (mockedElasticSearch7Index != null) {
mockedElasticSearch7Index.close();
}
+ if (mockedAtlasOpenSearchIndex != null) {
+ mockedAtlasOpenSearchIndex.close();
+ }
if (mockedApplicationProperties != null) {
mockedApplicationProperties.close();
}
@@ -290,6 +302,32 @@ public class AtlasJanusGraphIndexClientTest {
assertEquals(result.size(), 0);
}
+ @Test
+ public void testApplySearchWeightWithOpenSearchBackend() {
+ when(mockConfiguration.getString(ApplicationProperties.INDEX_BACKEND_CONF))
+ .thenReturn(ApplicationProperties.INDEX_BACKEND_OPENSEARCH);
+
+ Map<String, Integer> weightMap = Collections.singletonMap("field1", 10);
+ indexClient.applySearchWeight("testCollection", weightMap);
+
+ mockedSolr6Index.verify(Solr6Index::getSolrClient, never());
+ assertEquals(AtlasOpenSearchIndexClient.getSearchWeightByIndexField(), weightMap);
+ }
+
+ @Test
+ public void testQuickSearchWithOpenSearchBackendReturnsEmptyWhenClientNull() throws org.apache.atlas.exception.AtlasBaseException {
+ when(mockConfiguration.getString(ApplicationProperties.INDEX_BACKEND_CONF))
+ .thenReturn(ApplicationProperties.INDEX_BACKEND_OPENSEARCH);
+ mockedAtlasOpenSearchIndex.when(AtlasOpenSearchIndex::getOpenSearchClient).thenReturn(null);
+
+ QuickSearchContext ctx = new QuickSearchContext("test*", null, Collections.emptySet(),
+ Collections.emptySet(), Collections.emptyMap(), true, true, 0, 10);
+ QuickSearchResult result = indexClient.quickSearch(ctx);
+
+ assertEquals(result.getEntityGuids().size(), 0);
+ assertEquals(result.getTotalCount(), 0L);
+ }
+
@Test
public void testApplySearchWeightWithNullSolrClient() {
mockedSolr6Index.when(Solr6Index::getSolrClient).thenReturn(null);
@@ -389,6 +427,54 @@ public class AtlasJanusGraphIndexClientTest {
}
}
+ @Test
+ public void testIsHealthyWithOpenSearchBackendHealthy() throws Exception {
+ when(mockConfiguration.getString(ApplicationProperties.INDEX_BACKEND_CONF)).thenReturn(ApplicationProperties.INDEX_BACKEND_OPENSEARCH);
+ mockedAtlasOpenSearchIndex.when(AtlasOpenSearchIndex::getOpenSearchClient).thenReturn(mockOpenSearchClient);
+ when(mockOpenSearchClient.indexExists("janusgraph_vertex_index")).thenReturn(true);
+
+ assertTrue(indexClient.isHealthy());
+
+ verify(mockOpenSearchClient).indexExists("janusgraph_vertex_index");
+ mockedAtlasOpenSearchIndex.verify(AtlasOpenSearchIndex::getOpenSearchClient);
+ }
+
+ @Test
+ public void testIsHealthyWithOpenSearchBackendUnhealthy() throws Exception {
+ when(mockConfiguration.getString(ApplicationProperties.INDEX_BACKEND_CONF)).thenReturn(ApplicationProperties.INDEX_BACKEND_OPENSEARCH);
+ mockedAtlasOpenSearchIndex.when(AtlasOpenSearchIndex::getOpenSearchClient).thenReturn(mockOpenSearchClient);
+ when(mockOpenSearchClient.indexExists("janusgraph_vertex_index")).thenReturn(false);
+
+ assertFalse(indexClient.isHealthy());
+ }
+
+ @Test
+ public void testIsHealthyWithNullOpenSearchClient() {
+ when(mockConfiguration.getString(ApplicationProperties.INDEX_BACKEND_CONF)).thenReturn(ApplicationProperties.INDEX_BACKEND_OPENSEARCH);
+ mockedAtlasOpenSearchIndex.when(AtlasOpenSearchIndex::getOpenSearchClient).thenReturn(null);
+
+ assertFalse(indexClient.isHealthy());
+ }
+
+ @Test
+ public void testIsHealthyWithSolrBackendUnchanged() throws Exception {
+ when(mockConfiguration.getString(ApplicationProperties.INDEX_BACKEND_CONF)).thenReturn(ApplicationProperties.INDEX_BACKEND_SOLR);
+ mockedSolr6Index.when(Solr6Index::getSolrClient).thenReturn(mockSolrClient);
+ when(mockSolrClient.ping(Constants.VERTEX_INDEX)).thenReturn(mockPingResponse);
+ when(mockPingResponse.getStatus()).thenReturn(0);
+
+ assertTrue(indexClient.isHealthy());
+
+ verify(mockSolrClient).ping(Constants.VERTEX_INDEX);
+ }
+
+ @Test
+ public void testIsHealthyWithUnsupportedBackend() {
+ when(mockConfiguration.getString(ApplicationProperties.INDEX_BACKEND_CONF)).thenReturn("unknown-backend");
+
+ assertFalse(indexClient.isHealthy());
+ }
+
@Test
public void testGetTopTermsAsendingInput() {
Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = generateTerms(10, 12, 15);
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagementTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagementTest.java
index 0b186afbc..648e24a25 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagementTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagementTest.java
@@ -17,7 +17,9 @@
*/
package org.apache.atlas.repository.graphdb.janus;
+import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.graph.GraphSandboxUtil;
+import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasEdgeLabel;
@@ -34,7 +36,10 @@ import org.janusgraph.core.PropertyKey;
import org.janusgraph.core.schema.ConsistencyModifier;
import org.janusgraph.core.schema.JanusGraphIndex;
import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.Mapping;
import org.janusgraph.core.schema.SchemaAction;
+import org.apache.commons.configuration2.Configuration;
+import org.mockito.MockedStatic;
import org.janusgraph.core.schema.SchemaStatus;
import org.janusgraph.diskstorage.BackendTransaction;
import org.janusgraph.diskstorage.keycolumnvalue.scan.ScanJobFuture;
@@ -89,6 +94,10 @@ public class AtlasJanusGraphManagementTest {
public static void setupClass() throws Exception {
GraphSandboxUtil.create();
+ // Force Constants static init before any MockedStatic<ApplicationProperties> usage.
+ assertNotNull(Constants.STATE_PROPERTY_KEY);
+ assertNotNull(Constants.ENTITY_TYPE_PROPERTY_KEY);
+
if (useLocalSolr()) {
LocalSolrRunner.start();
}
@@ -104,6 +113,7 @@ public class AtlasJanusGraphManagementTest {
@BeforeMethod
public void setup() {
MockitoAnnotations.openMocks(this);
+ AtlasOpenSearchIndexClient.clearKeywordSubfieldFieldsForTests();
database = new AtlasJanusGraphDatabase();
atlasGraph = database.getGraph();
management = new AtlasJanusGraphManagement(mockAtlasGraph, mockJanusManagement);
@@ -456,6 +466,135 @@ public class AtlasJanusGraphManagementTest {
assertEquals(result, "testProperty_string_encoded");
}
+ @Test
+ public void testAddMixedIndexOpenSearchStateUsesTextWithKeywordSubfield() throws Exception {
+ Configuration mockConfig = mock(Configuration.class);
+ when(mockConfig.getString(ApplicationProperties.INDEX_BACKEND_CONF)).thenReturn("opensearch");
+
+ try (MockedStatic<ApplicationProperties> mockedAppProps = org.mockito.Mockito.mockStatic(ApplicationProperties.class)) {
+ mockedAppProps.when(ApplicationProperties::get).thenReturn(mockConfig);
+
+ AtlasJanusPropertyKey mockPropertyKey = mock(AtlasJanusPropertyKey.class);
+ org.janusgraph.core.PropertyKey mockJanusPropertyKey = mock(org.janusgraph.core.PropertyKey.class);
+ when(mockPropertyKey.getWrappedPropertyKey()).thenReturn(mockJanusPropertyKey);
+ when(mockPropertyKey.getName()).thenReturn(Constants.STATE_PROPERTY_KEY);
+ JanusGraphIndex mockJanusGraphIndex = mock(JanusGraphIndex.class);
+
+ when(mockJanusManagement.getGraphIndex("testIndex")).thenReturn(mockJanusGraphIndex);
+ when(mockAtlasGraph.getIndexFieldName(any(), any(), any())).thenReturn("__state");
+
+ String result = management.addMixedIndex("testIndex", mockPropertyKey, false, true);
+
+ assertEquals(result, "__state");
+ verify(mockJanusManagement).addIndexKey(eq(mockJanusGraphIndex), eq(mockJanusPropertyKey), any());
+ verify(mockJanusManagement, never()).addIndexKey(eq(mockJanusGraphIndex), eq(mockJanusPropertyKey),
+ eq(Mapping.STRING.asParameter()));
+ assertTrue(AtlasOpenSearchIndexClient.usesKeywordSubfield("__state"));
+ }
+ }
+
+ @Test
+ public void testAddMixedIndexOpenSearchTypeNameUsesTextWithKeywordSubfield() throws Exception {
+ Configuration mockConfig = mock(Configuration.class);
+ when(mockConfig.getString(ApplicationProperties.INDEX_BACKEND_CONF)).thenReturn("opensearch");
+
+ try (MockedStatic<ApplicationProperties> mockedAppProps = org.mockito.Mockito.mockStatic(ApplicationProperties.class)) {
+ mockedAppProps.when(ApplicationProperties::get).thenReturn(mockConfig);
+
+ AtlasJanusPropertyKey mockPropertyKey = mock(AtlasJanusPropertyKey.class);
+ org.janusgraph.core.PropertyKey mockJanusPropertyKey = mock(org.janusgraph.core.PropertyKey.class);
+ when(mockPropertyKey.getWrappedPropertyKey()).thenReturn(mockJanusPropertyKey);
+ when(mockPropertyKey.getName()).thenReturn(Constants.ENTITY_TYPE_PROPERTY_KEY);
+ JanusGraphIndex mockJanusGraphIndex = mock(JanusGraphIndex.class);
+
+ when(mockJanusManagement.getGraphIndex("testIndex")).thenReturn(mockJanusGraphIndex);
+ when(mockAtlasGraph.getIndexFieldName(any(), any(), any())).thenReturn("__typeName");
+
+ String result = management.addMixedIndex("testIndex", mockPropertyKey, false, true);
+
+ assertEquals(result, "__typeName");
+ verify(mockJanusManagement).addIndexKey(eq(mockJanusGraphIndex), eq(mockJanusPropertyKey), any());
+ verify(mockJanusManagement, never()).addIndexKey(eq(mockJanusGraphIndex), eq(mockJanusPropertyKey),
+ eq(Mapping.STRING.asParameter()));
+ assertTrue(AtlasOpenSearchIndexClient.usesKeywordSubfield("__typeName"));
+ }
+ }
+
+ @Test
+ public void testAddMixedIndexSolrStateKeepsDefaultMapping() throws Exception {
+ Configuration mockConfig = mock(Configuration.class);
+ when(mockConfig.getString(ApplicationProperties.INDEX_BACKEND_CONF)).thenReturn("solr");
+
+ try (MockedStatic<ApplicationProperties> mockedAppProps = org.mockito.Mockito.mockStatic(ApplicationProperties.class)) {
+ mockedAppProps.when(ApplicationProperties::get).thenReturn(mockConfig);
+
+ AtlasJanusPropertyKey mockPropertyKey = mock(AtlasJanusPropertyKey.class);
+ org.janusgraph.core.PropertyKey mockJanusPropertyKey = mock(org.janusgraph.core.PropertyKey.class);
+ when(mockPropertyKey.getWrappedPropertyKey()).thenReturn(mockJanusPropertyKey);
+ when(mockPropertyKey.getName()).thenReturn(Constants.STATE_PROPERTY_KEY);
+ JanusGraphIndex mockJanusGraphIndex = mock(JanusGraphIndex.class);
+
+ when(mockJanusManagement.getGraphIndex("testIndex")).thenReturn(mockJanusGraphIndex);
+ when(mockAtlasGraph.getIndexFieldName(any(), any())).thenReturn("__state");
+
+ String result = management.addMixedIndex("testIndex", mockPropertyKey, false);
+
+ assertEquals(result, "__state");
+ verify(mockJanusManagement).addIndexKey(mockJanusGraphIndex, mockJanusPropertyKey);
+ verify(mockJanusManagement, never()).addIndexKey(eq(mockJanusGraphIndex), eq(mockJanusPropertyKey),
+ eq(Mapping.STRING.asParameter()));
+ }
+ }
+
+ @Test
+ public void testAddMixedIndexElasticsearchStateKeepsDefaultMapping() throws Exception {
+ Configuration mockConfig = mock(Configuration.class);
+ when(mockConfig.getString(ApplicationProperties.INDEX_BACKEND_CONF)).thenReturn("elasticsearch");
+
+ try (MockedStatic<ApplicationProperties> mockedAppProps = org.mockito.Mockito.mockStatic(ApplicationProperties.class)) {
+ mockedAppProps.when(ApplicationProperties::get).thenReturn(mockConfig);
+
+ AtlasJanusPropertyKey mockPropertyKey = mock(AtlasJanusPropertyKey.class);
+ org.janusgraph.core.PropertyKey mockJanusPropertyKey = mock(org.janusgraph.core.PropertyKey.class);
+ when(mockPropertyKey.getWrappedPropertyKey()).thenReturn(mockJanusPropertyKey);
+ when(mockPropertyKey.getName()).thenReturn(Constants.STATE_PROPERTY_KEY);
+ JanusGraphIndex mockJanusGraphIndex = mock(JanusGraphIndex.class);
+
+ when(mockJanusManagement.getGraphIndex("testIndex")).thenReturn(mockJanusGraphIndex);
+ when(mockAtlasGraph.getIndexFieldName(any(), any())).thenReturn("__state");
+
+ String result = management.addMixedIndex("testIndex", mockPropertyKey, false);
+
+ assertEquals(result, "__state");
+ verify(mockJanusManagement).addIndexKey(mockJanusGraphIndex, mockJanusPropertyKey);
+ verify(mockJanusManagement, never()).addIndexKey(eq(mockJanusGraphIndex), eq(mockJanusPropertyKey),
+ eq(Mapping.STRING.asParameter()));
+ }
+ }
+
+ @Test
+ public void testGetIndexFieldNameOpenSearchStateUsesTextWithKeywordSubfield() throws Exception {
+ Configuration mockConfig = mock(Configuration.class);
+ when(mockConfig.getString(ApplicationProperties.INDEX_BACKEND_CONF)).thenReturn("opensearch");
+
+ try (MockedStatic<ApplicationProperties> mockedAppProps = org.mockito.Mockito.mockStatic(ApplicationProperties.class)) {
+ mockedAppProps.when(ApplicationProperties::get).thenReturn(mockConfig);
+
+ AtlasJanusPropertyKey mockPropertyKey = mock(AtlasJanusPropertyKey.class);
+ org.janusgraph.core.PropertyKey mockJanusPropertyKey = mock(org.janusgraph.core.PropertyKey.class);
+ when(mockPropertyKey.getWrappedPropertyKey()).thenReturn(mockJanusPropertyKey);
+ when(mockPropertyKey.getName()).thenReturn(Constants.STATE_PROPERTY_KEY);
+ JanusGraphIndex mockJanusGraphIndex = mock(JanusGraphIndex.class);
+
+ when(mockJanusManagement.getGraphIndex("testIndex")).thenReturn(mockJanusGraphIndex);
+ when(mockAtlasGraph.getIndexFieldName(any(), any(), any())).thenReturn("__state");
+
+ String result = management.getIndexFieldName("testIndex", mockPropertyKey, false, true);
+
+ assertEquals(result, "__state");
+ }
+ }
+
@Test
public void testGetIndexFieldName() {
AtlasJanusPropertyKey mockPropertyKey = mock(AtlasJanusPropertyKey.class);
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchIndexClientTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchIndexClientTest.java
new file mode 100644
index 000000000..1cf55c77a
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchIndexClientTest.java
@@ -0,0 +1,302 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.QuickSearchContext;
+import org.janusgraph.diskstorage.opensearch.AtlasOpenSearchIndex;
+import org.janusgraph.diskstorage.opensearch.OpenSearchClient;
+import org.janusgraph.diskstorage.opensearch.rest.RestSearchResponse;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+public class AtlasOpenSearchIndexClientTest {
+
+ @BeforeMethod
+ public void setUp() {
+ AtlasOpenSearchIndexClient.clearKeywordSubfieldFieldsForTests();
+ AtlasOpenSearchIndexClient.applySuggestionFields(Collections.emptyList());
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ AtlasOpenSearchIndexClient.clearKeywordSubfieldFieldsForTests();
+ AtlasOpenSearchIndexClient.applySuggestionFields(Collections.emptyList());
+ }
+
+ @Test
+ public void toTermsIncludePatternEscapesRegexMetacharacters() {
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("cust"), "cust.*");
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("cust-"), "cust\\-.*");
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("cust_"), "cust\\_.*");
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("cust."), "cust\\..*");
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("cust+"), "cust\\+.*");
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("cust*"), "cust\\*.*");
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("cust?"), "cust\\?.*");
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("cust("), "cust\\(.*");
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("cust["), "cust\\[.*");
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("cust\\"), "cust\\\\.*");
+ }
+
+ @Test
+ public void toTermsIncludePatternPreservesCase() {
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("Customer"), "Customer.*");
+ assertEquals(AtlasOpenSearchIndexClient.toTermsIncludePattern("CUST"), "CUST.*");
+ }
+
+ @Test
+ public void resolveTermsAggregationFieldUsesKeywordSubfieldWhenRegistered() {
+ AtlasOpenSearchIndexClient.registerKeywordSubfieldField("storm_node.description");
+
+ assertEquals(AtlasOpenSearchIndexClient.resolveTermsAggregationFieldName("storm_node.description"),
+ "storm_node\u2022description.keyword");
+ }
+
+ @Test
+ public void resolveTermsAggregationFieldUsesBaseFieldForStringMapping() {
+ assertEquals(AtlasOpenSearchIndexClient.resolveTermsAggregationFieldName("c55_asset\u2022__s_owner"),
+ "c55_asset\u2022__s_owner");
+ }
+
+ @Test
+ public void resolveTermsAggregationFieldUsesKeywordSubfieldWhenRegisteredForEntityType() {
+ AtlasOpenSearchIndexClient.registerKeywordSubfieldField(Constants.ENTITY_TYPE_PROPERTY_KEY);
+
+ assertEquals(AtlasOpenSearchIndexClient.resolveTermsAggregationFieldName(Constants.ENTITY_TYPE_PROPERTY_KEY),
+ Constants.ENTITY_TYPE_PROPERTY_KEY + ".keyword");
+ }
+
+ @Test
+ public void resolveTermsAggregationFieldUsesNativeKeywordWhenSubfieldNotRegistered() {
+ AtlasOpenSearchIndexClient.clearKeywordSubfieldFieldsForTests();
+
+ assertEquals(AtlasOpenSearchIndexClient.resolveTermsAggregationFieldName(Constants.ENTITY_TYPE_PROPERTY_KEY),
+ Constants.ENTITY_TYPE_PROPERTY_KEY);
+ }
+
+ @Test
+ public void buildSuggestionsTermsAggsCreatesOneAggPerField() {
+ List<String> fields = Arrays.asList("field_a", "field_b", "field_c");
+ Map<String, Object> aggs = AtlasOpenSearchIndexClient.buildSuggestionsTermsAggs(fields, "cust");
+
+ assertEquals(aggs.size(), 3);
+ assertTrue(aggs.containsKey("sugg_0"));
+ assertTrue(aggs.containsKey("sugg_1"));
+ assertTrue(aggs.containsKey("sugg_2"));
+
+ Map<String, Object> firstTerms = (Map<String, Object>) aggs.get("sugg_0");
+ Map<String, Object> termsSpec = (Map<String, Object>) firstTerms.get("terms");
+
+ assertEquals(termsSpec.get("include"), "cust.*");
+ assertEquals(termsSpec.get("size"), AtlasJanusGraphIndexClient.DEFAULT_SUGGESTION_COUNT * 4);
+ }
+
+ @Test
+ public void buildSuggestionsFilterQueryExcludesDeletedEntities() {
+ AtlasOpenSearchIndexClient.registerKeywordSubfieldField(Constants.STATE_PROPERTY_KEY);
+
+ Map<String, Object> query = AtlasOpenSearchIndexClient.buildSuggestionsFilterQuery();
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+ List<Map<String, Object>> mustNot = (List<Map<String, Object>>) bool.get("must_not");
+ Map<String, Object> termClause = mustNot.get(0);
+ Map<String, Object> term = (Map<String, Object>) termClause.get("term");
+
+ assertEquals(term.get("__state.keyword"), AtlasEntity.Status.DELETED.name());
+ }
+
+ @Test
+ public void mergeTermBucketsDeduplicatesAndSumsFrequencies() {
+ Map<String, AtlasJanusGraphIndexClient.TermFreq> termsMap = new HashMap<>();
+
+ List<Map<String, Object>> nameBuckets = Arrays.asList(
+ bucket("customer", 10L),
+ bucket("customer_data", 7L));
+ List<Map<String, Object>> ownerBuckets = Arrays.asList(
+ bucket("customer", 5L),
+ bucket("customer_team", 3L));
+
+ AtlasOpenSearchIndexClient.mergeTermBuckets(termsMap, nameBuckets);
+ AtlasOpenSearchIndexClient.mergeTermBuckets(termsMap, ownerBuckets);
+
+ List<String> top = AtlasJanusGraphIndexClient.getTopTerms(termsMap);
+
+ assertEquals(top.size(), 3);
+ assertEquals(top.get(0), "customer");
+ assertEquals(termsMap.get("customer").getFreq(), 15L);
+ assertEquals(termsMap.get("customer_data").getFreq(), 7L);
+ assertEquals(termsMap.get("customer_team").getFreq(), 3L);
+ }
+
+ @Test
+ public void collectTermsFromAggregationsMergesAcrossNamedAggs() {
+ Map<String, Object> aggregations = new HashMap<>();
+
+ aggregations.put("sugg_0", aggResult(bucket("alpha", 3L), bucket("beta", 1L)));
+ aggregations.put("sugg_1", aggResult(bucket("alpha", 2L), bucket("gamma", 4L)));
+
+ Map<String, AtlasJanusGraphIndexClient.TermFreq> terms =
+ AtlasOpenSearchIndexClient.collectTermsFromAggregations(
+ aggregations, new LinkedHashSet<>(Arrays.asList("sugg_0", "sugg_1")));
+
+ assertEquals(terms.get("alpha").getFreq(), 5L);
+ assertEquals(terms.get("gamma").getFreq(), 4L);
+ }
+
+ @Test
+ public void getSuggestionsIssuesSingleOpenSearchRequestForMultipleFields() throws Exception {
+ OpenSearchClient mockClient = mock(OpenSearchClient.class);
+ RestSearchResponse mockResponse = mock(RestSearchResponse.class);
+
+ Map<String, Object> aggregations = new HashMap<>();
+ aggregations.put("sugg_0", aggResult(bucket("team-alpha", 3L)));
+ aggregations.put("sugg_1", aggResult(bucket("team-beta", 2L)));
+
+ when(mockClient.search(any(), any(), eq(false))).thenReturn(mockResponse);
+ when(mockResponse.getAggregations()).thenReturn(aggregations);
+
+ try (MockedStatic<AtlasOpenSearchIndex> mockedIndex = Mockito.mockStatic(AtlasOpenSearchIndex.class)) {
+ mockedIndex.when(AtlasOpenSearchIndex::getOpenSearchClient).thenReturn(mockClient);
+
+ AtlasOpenSearchIndexClient.applySuggestionFields(
+ Arrays.asList("owner_field", "name_field"));
+
+ List<String> result = AtlasOpenSearchIndexClient.getSuggestions("team", null, null);
+
+ verify(mockClient, times(1)).search(any(), any(), eq(false));
+ assertFalse(result.isEmpty());
+ }
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class)
+ public void quickSearchPropagatesBackendFailureInsteadOfReturningEmpty() throws Exception {
+ // An OpenSearch outage must surface as an exception, not be converted into a 0-result search.
+ OpenSearchClient mockClient = mock(OpenSearchClient.class);
+
+ when(mockClient.search(any(), any(), eq(false))).thenThrow(new java.io.IOException("OpenSearch unavailable"));
+
+ try (MockedStatic<AtlasOpenSearchIndex> mockedIndex = Mockito.mockStatic(AtlasOpenSearchIndex.class)) {
+ mockedIndex.when(AtlasOpenSearchIndex::getOpenSearchClient).thenReturn(mockClient);
+
+ QuickSearchContext ctx = new QuickSearchContext("atlas", null, Collections.emptySet(),
+ Collections.emptySet(), new HashMap<>(), false, false, 0, 10);
+
+ AtlasOpenSearchIndexClient.quickSearch(ctx, null);
+ }
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class)
+ public void quickSearchPropagatesQueryBuildFailureInsteadOfReturningEmpty() throws Exception {
+ // A query-builder failure (here: exclude-deleted requested but no __state index field mapped)
+ // must propagate rather than silently becoming an empty result.
+ OpenSearchClient mockClient = mock(OpenSearchClient.class);
+
+ try (MockedStatic<AtlasOpenSearchIndex> mockedIndex = Mockito.mockStatic(AtlasOpenSearchIndex.class)) {
+ mockedIndex.when(AtlasOpenSearchIndex::getOpenSearchClient).thenReturn(mockClient);
+
+ QuickSearchContext ctx = new QuickSearchContext("atlas", null, Collections.emptySet(),
+ Collections.emptySet(), new HashMap<>(), true, false, 0, 10);
+
+ AtlasOpenSearchIndexClient.quickSearch(ctx, null);
+ }
+ }
+
+ @Test
+ public void quickSearchReturnsEmptyWhenBackendIsNotOpenSearch() throws Exception {
+ // Documented contract: when the deployment is not OpenSearch-backed (null client), an empty result is
+ // returned — this is NOT a backend failure and must not throw.
+ try (MockedStatic<AtlasOpenSearchIndex> mockedIndex = Mockito.mockStatic(AtlasOpenSearchIndex.class)) {
+ mockedIndex.when(AtlasOpenSearchIndex::getOpenSearchClient).thenReturn(null);
+
+ QuickSearchContext ctx = new QuickSearchContext("atlas", null, Collections.emptySet(),
+ Collections.emptySet(), new HashMap<>(), false, false, 0, 10);
+
+ assertTrue(AtlasOpenSearchIndexClient.quickSearch(ctx, null).getEntityGuids().isEmpty());
+ }
+ }
+
+ @Test
+ public void getSuggestionsReturnsEmptyOnBackendFailure() throws Exception {
+ // Suggestions intentionally return empty on backend errors (parity with Solr), and must not throw.
+ OpenSearchClient mockClient = mock(OpenSearchClient.class);
+
+ when(mockClient.search(any(), any(), eq(false))).thenThrow(new java.io.IOException("OpenSearch unavailable"));
+
+ try (MockedStatic<AtlasOpenSearchIndex> mockedIndex = Mockito.mockStatic(AtlasOpenSearchIndex.class)) {
+ mockedIndex.when(AtlasOpenSearchIndex::getOpenSearchClient).thenReturn(mockClient);
+
+ AtlasOpenSearchIndexClient.applySuggestionFields(Arrays.asList("owner_field"));
+
+ List<String> result = AtlasOpenSearchIndexClient.getSuggestions("team", null, null);
+
+ assertTrue(result.isEmpty());
+ }
+ }
+
+ @Test
+ public void getTopTermsReturnsAtMostFiveSuggestions() {
+ Map<String, AtlasJanusGraphIndexClient.TermFreq> terms = new HashMap<>();
+
+ for (int i = 0; i < 10; i++) {
+ terms.put("term-" + i, new AtlasJanusGraphIndexClient.TermFreq("term-" + i, 100 - i));
+ }
+
+ List<String> top = AtlasJanusGraphIndexClient.getTopTerms(terms);
+
+ assertEquals(top.size(), AtlasJanusGraphIndexClient.DEFAULT_SUGGESTION_COUNT);
+ }
+
+ private static Map<String, Object> bucket(String key, long docCount) {
+ Map<String, Object> bucket = new HashMap<>();
+
+ bucket.put("key", key);
+ bucket.put("doc_count", docCount);
+
+ return bucket;
+ }
+
+ private static Map<String, Object> aggResult(Map<String, Object>... buckets) {
+ Map<String, Object> agg = new HashMap<>();
+
+ agg.put("buckets", Arrays.asList(buckets));
+
+ return agg;
+ }
+}
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchQueryBuilderTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchQueryBuilderTest.java
new file mode 100644
index 000000000..673860b3e
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchQueryBuilderTest.java
@@ -0,0 +1,367 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.Operator;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+public class AtlasOpenSearchQueryBuilderTest {
+
+ private static final String TYPE_DATASET = "test_dataset";
+
+ private AtlasTypeRegistry typeRegistry;
+ private Map<String, String> indexFieldNameCache;
+ private Map<String, Integer> searchWeights;
+ private Set<AtlasEntityType> entityTypes;
+
+ @org.testng.annotations.AfterMethod
+ public void tearDownKeywordFields() {
+ AtlasOpenSearchIndexClient.clearKeywordSubfieldFieldsForTests();
+ }
+
+ @BeforeMethod
+ public void setUp() throws org.apache.atlas.exception.AtlasBaseException {
+ AtlasOpenSearchIndexClient.clearKeywordSubfieldFieldsForTests();
+
+ typeRegistry = new AtlasTypeRegistry();
+
+ AtlasEntityDef datasetDef = new AtlasEntityDef();
+ datasetDef.setName(TYPE_DATASET);
+
+ List<AtlasAttributeDef> attrs = new java.util.ArrayList<>();
+ AtlasAttributeDef ownerAttr = new AtlasAttributeDef("owner", "string");
+ ownerAttr.setIndexType(AtlasAttributeDef.IndexType.STRING);
+ attrs.add(ownerAttr);
+ datasetDef.setAttributeDefs(attrs);
+
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+ typesDef.getEntityDefs().add(datasetDef);
+ typeRegistry.updateTypes(typesDef);
+
+ AtlasEntityType datasetType = typeRegistry.getEntityTypeByName(TYPE_DATASET);
+ datasetType.getAttribute("owner").setIndexFieldName("owner_index");
+
+ indexFieldNameCache = new HashMap<>();
+ indexFieldNameCache.put(Constants.ENTITY_TYPE_PROPERTY_KEY, "__typeName");
+ indexFieldNameCache.put(Constants.STATE_PROPERTY_KEY, "__state");
+ indexFieldNameCache.put("owner", "owner_index");
+
+ searchWeights = new HashMap<>();
+ searchWeights.put("name_index", 10);
+ searchWeights.put("comment_index", 5);
+
+ entityTypes = new HashSet<>();
+ entityTypes.add(datasetType);
+ }
+
+ @Test
+ public void plainTermUsesDisMaxWhenWeightsConfigured() throws AtlasBaseException {
+ Map<String, Object> query = builder("atlas").buildDiscoveryQuery();
+
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+ List<Map<String, Object>> must = (List<Map<String, Object>>) bool.get("must");
+
+ assertNotNull(must);
+ assertTrue(must.get(0).containsKey("dis_max"));
+
+ Map<String, Object> disMax = (Map<String, Object>) must.get(0).get("dis_max");
+ List<Map<String, Object>> queries = (List<Map<String, Object>>) disMax.get("queries");
+
+ assertNotNull(queries);
+ assertFalse(queries.isEmpty());
+ assertTrue(queries.get(0).containsKey("match"));
+ assertTrue(queries.stream().anyMatch(q -> q.containsKey("match")
+ && ((Map<?, ?>) q.get("match")).containsKey("all")));
+ }
+
+ @Test
+ public void wildcardQueryUsesDisMaxWithQueryStringSubQueries() throws AtlasBaseException {
+ Map<String, Object> query = builder("custo*").buildDiscoveryQuery();
+
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+ List<Map<String, Object>> must = (List<Map<String, Object>>) bool.get("must");
+
+ assertNotNull(must);
+ assertTrue(must.get(0).containsKey("dis_max"));
+
+ Map<String, Object> disMax = (Map<String, Object>) must.get(0).get("dis_max");
+ List<Map<String, Object>> queries = (List<Map<String, Object>>) disMax.get("queries");
+
+ assertNotNull(queries);
+ assertTrue(queries.get(0).containsKey("prefix"));
+
+ Map<String, Object> prefix = (Map<String, Object>) queries.get(0).get("prefix");
+ assertNotNull(prefix);
+ }
+
+ @Test
+ public void colonQueryUsesQuotedQueryStringSubQueries() throws AtlasBaseException {
+ Map<String, Object> query = builder("A:B").buildDiscoveryQuery();
+
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+ List<Map<String, Object>> must = (List<Map<String, Object>>) bool.get("must");
+
+ assertNotNull(must);
+ assertTrue(must.get(0).containsKey("dis_max"));
+
+ Map<String, Object> disMax = (Map<String, Object>) must.get(0).get("dis_max");
+ List<Map<String, Object>> queries = (List<Map<String, Object>>) disMax.get("queries");
+
+ assertNotNull(queries);
+ assertTrue(queries.get(0).containsKey("query_string"));
+
+ Map<String, Object> queryString = (Map<String, Object>) queries.get(0).get("query_string");
+ assertEquals(queryString.get("query"), "\"A\\:B\"");
+ }
+
+ @Test
+ public void discoveryQueryAndWeightedQuickSearchAreEquivalent() throws AtlasBaseException {
+ AtlasOpenSearchQueryBuilder builder = builder("atlas*");
+
+ assertEquals(builder.buildDiscoveryQuery(), builder.buildWeightedQuickSearch());
+ }
+
+ @Test
+ public void entityFilterUsesFilterClause() throws AtlasBaseException {
+ FilterCriteria filter = new FilterCriteria();
+ filter.setAttributeName("owner");
+ filter.setOperator(Operator.EQ);
+ filter.setAttributeValue("team-alpha");
+
+ Map<String, Object> query = builder("atlas").withCriteria(filter).buildDiscoveryQuery();
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+
+ assertNotNull(bool.get("filter"));
+ assertNotNull(bool.get("must_not"));
+ }
+
+ @Test
+ public void excludeDeletedUsesMustNotOnState() throws AtlasBaseException {
+ Map<String, Object> query = builder("atlas").buildDiscoveryQuery();
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+ List<Map<String, Object>> mustNot = (List<Map<String, Object>>) bool.get("must_not");
+
+ assertNotNull(mustNot);
+ assertFalse(mustNot.isEmpty());
+ }
+
+ @Test
+ public void excludeDeletedUsesKeywordSubfieldWhenStateRegistered() throws AtlasBaseException {
+ // __state mapped as text+keyword: exact-match term must target __state.keyword, otherwise the analyzed
+ // (lower-cased) text field would not match the "DELETED" term and deleted entities would leak into results.
+ AtlasOpenSearchIndexClient.registerKeywordSubfieldField("__state");
+
+ Map<String, Object> query = builder("atlas").buildDiscoveryQuery();
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+ List<Map<String, Object>> mustNot = (List<Map<String, Object>>) bool.get("must_not");
+ Map<String, Object> term = (Map<String, Object>) mustNot.get(0).get("term");
+
+ assertTrue(term.containsKey("__state.keyword"), "expected __state.keyword, got: " + term);
+ assertEquals(term.get("__state.keyword"), "DELETED");
+ }
+
+ @Test
+ public void excludeDeletedUsesBareFieldWhenStateNotRegistered() throws AtlasBaseException {
+ // Native-keyword/string mapping (not registered): the bare field is correct; behavior must be unchanged.
+ Map<String, Object> query = builder("atlas").buildDiscoveryQuery();
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+ List<Map<String, Object>> mustNot = (List<Map<String, Object>>) bool.get("must_not");
+ Map<String, Object> term = (Map<String, Object>) mustNot.get(0).get("term");
+
+ assertTrue(term.containsKey("__state"), "expected bare __state, got: " + term);
+ assertFalse(term.containsKey("__state.keyword"));
+ }
+
+ @Test
+ public void entityTypeFilterUsesKeywordSubfieldWhenTypeNameRegistered() throws AtlasBaseException {
+ AtlasOpenSearchIndexClient.registerKeywordSubfieldField("__typeName");
+
+ Map<String, Object> query = builder("atlas").buildDiscoveryQuery();
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+ List<Map<String, Object>> filter = (List<Map<String, Object>>) bool.get("filter");
+ Map<String, Object> terms = (Map<String, Object>) filter.get(0).get("terms");
+
+ assertTrue(terms.containsKey("__typeName.keyword"), "expected __typeName.keyword, got: " + terms);
+ }
+
+ @Test
+ public void eqAndNeqUseKeywordSubfieldWhileWildcardAndExistsUseAnalyzedField() throws AtlasBaseException {
+ // owner_index mapped as text+keyword.
+ AtlasOpenSearchIndexClient.registerKeywordSubfieldField("owner_index");
+
+ // EQ -> term on owner_index.keyword
+ FilterCriteria eq = leaf("owner", Operator.EQ, "team-alpha");
+ Map<String, Object> eqClause = leafClause(builder("atlas").withCriteria(eq).buildDiscoveryQuery());
+ Map<String, Object> term = (Map<String, Object>) eqClause.get("term");
+ assertNotNull(term, "EQ should produce a term clause: " + eqClause);
+ assertTrue(term.containsKey("owner_index.keyword"), "EQ must use keyword field, got: " + term);
+
+ // NEQ -> must_not term on owner_index.keyword
+ FilterCriteria neq = leaf("owner", Operator.NEQ, "team-alpha");
+ Map<String, Object> neqClause = leafClause(builder("atlas").withCriteria(neq).buildDiscoveryQuery());
+ Map<String, Object> neqBool = (Map<String, Object>) neqClause.get("bool");
+ Map<String, Object> neqTerm = (Map<String, Object>) ((Map<String, Object>) neqBool.get("must_not")).get("term");
+ assertTrue(neqTerm.containsKey("owner_index.keyword"), "NEQ must use keyword field, got: " + neqTerm);
+
+ // STARTS_WITH -> wildcard on the analyzed field (NOT .keyword)
+ FilterCriteria sw = leaf("owner", Operator.STARTS_WITH, "team");
+ Map<String, Object> swClause = leafClause(builder("atlas").withCriteria(sw).buildDiscoveryQuery());
+ Map<String, Object> wildcard = (Map<String, Object>) swClause.get("wildcard");
+ assertNotNull(wildcard, "STARTS_WITH should produce a wildcard clause: " + swClause);
+ assertTrue(wildcard.containsKey("owner_index"), "wildcard must use analyzed field, got: " + wildcard);
+ assertFalse(wildcard.containsKey("owner_index.keyword"));
+
+ // NOT_NULL -> exists on the analyzed field (NOT .keyword)
+ FilterCriteria nn = leaf("owner", Operator.NOT_NULL, null);
+ Map<String, Object> nnClause = leafClause(builder("atlas").withCriteria(nn).buildDiscoveryQuery());
+ Map<String, Object> exists = (Map<String, Object>) nnClause.get("exists");
+ assertNotNull(exists, "NOT_NULL should produce an exists clause: " + nnClause);
+ assertEquals(exists.get("field"), "owner_index");
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class)
+ public void unsupportedOperatorThrowsForParityWithSolr() throws AtlasBaseException {
+ // IN/LIKE/CONTAINS_ANY/CONTAINS_ALL are unsupported by AtlasSolrQueryBuilder too; OpenSearch must not invent.
+ FilterCriteria in = leaf("owner", Operator.IN, "a,b");
+
+ builder("atlas").withCriteria(in).buildDiscoveryQuery();
+ }
+
+ @Test
+ public void classificationFilterUsesShouldOverBothClassificationFieldsWithKeywordSubfield() throws AtlasBaseException {
+ indexFieldNameCache.put(Constants.CLASSIFICATION_NAMES_KEY, "__classificationNames");
+ indexFieldNameCache.put(Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY, "__propagatedClassificationNames");
+
+ // Common case: classification-name fields are registered as text+keyword (see
+ // AtlasJanusGraphManagement#registerKeywordSubfieldField), so the exact-match "terms" filter must resolve
+ // to the .keyword subfield, not the analyzed text field.
+ AtlasOpenSearchIndexClient.registerKeywordSubfieldField("__classificationNames");
+ AtlasOpenSearchIndexClient.registerKeywordSubfieldField("__propagatedClassificationNames");
+
+ Set<String> classifications = new HashSet<>();
+ classifications.add("PII");
+
+ Map<String, Object> query = builder("atlas").withClassificationTypeNames(classifications).buildDiscoveryQuery();
+
+ assertClassificationShouldClause(query, "__classificationNames.keyword", "__propagatedClassificationNames.keyword");
+ }
+
+ @Test
+ public void classificationFilterFallsBackToBareFieldWhenNotKeywordRegistered() throws AtlasBaseException {
+ indexFieldNameCache.put(Constants.CLASSIFICATION_NAMES_KEY, "__classificationNames");
+ indexFieldNameCache.put(Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY, "__propagatedClassificationNames");
+
+ // Deliberately do NOT register a .keyword subfield for these fields — exercises the "native"/legacy
+ // fallback path (e.g. classification fields mapped without a keyword subfield), preserving prior behavior.
+ Set<String> classifications = new HashSet<>();
+ classifications.add("PII");
+
+ Map<String, Object> query = builder("atlas").withClassificationTypeNames(classifications).buildDiscoveryQuery();
+
+ assertClassificationShouldClause(query, "__classificationNames", "__propagatedClassificationNames");
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void assertClassificationShouldClause(Map<String, Object> query, String expectedClassField,
+ String expectedPropagatedField) {
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+ List<Map<String, Object>> filter = (List<Map<String, Object>>) bool.get("filter");
+
+ Map<String, Object> classificationBool = filter.stream()
+ .map(f -> (Map<String, Object>) f.get("bool"))
+ .filter(b -> b != null && b.containsKey("should"))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("expected a classification bool/should clause: " + filter));
+
+ List<Map<String, Object>> shouldClauses = (List<Map<String, Object>>) classificationBool.get("should");
+ assertEquals(shouldClauses.size(), 2,
+ "classification filter must OR the direct and propagated classification fields: " + shouldClauses);
+
+ Map<String, Object> classTerms = (Map<String, Object>) shouldClauses.get(0).get("terms");
+ Map<String, Object> propagatedTerms = (Map<String, Object>) shouldClauses.get(1).get("terms");
+
+ assertNotNull(classTerms, "expected first should-clause to be a terms filter: " + shouldClauses);
+ assertNotNull(propagatedTerms, "expected second should-clause to be a terms filter: " + shouldClauses);
+
+ assertTrue(classTerms.containsKey(expectedClassField),
+ "expected terms filter keyed on '" + expectedClassField + "' but got: " + classTerms.keySet());
+ assertTrue(propagatedTerms.containsKey(expectedPropagatedField),
+ "expected terms filter keyed on '" + expectedPropagatedField + "' but got: " + propagatedTerms.keySet());
+
+ assertEquals(classTerms.get(expectedClassField), Collections.singletonList("PII"),
+ "expected the classification value PII to be passed through to the terms filter");
+ assertEquals(propagatedTerms.get(expectedPropagatedField), Collections.singletonList("PII"),
+ "expected the classification value PII to be passed through to the propagated terms filter");
+ }
+
+ private static FilterCriteria leaf(String attr, Operator op, String value) {
+ FilterCriteria criteria = new FilterCriteria();
+ criteria.setAttributeName(attr);
+ criteria.setOperator(op);
+ criteria.setAttributeValue(value);
+ return criteria;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, Object> leafClause(Map<String, Object> query) {
+ Map<String, Object> bool = (Map<String, Object>) query.get("bool");
+ List<Map<String, Object>> filter = (List<Map<String, Object>>) bool.get("filter");
+ // The criteria clause is the last filter entry (after entity-type filter).
+ return filter.get(filter.size() - 1);
+ }
+
+ @Test
+ public void containsWildcardDetectsUserWildcards() {
+ assertTrue(AtlasOpenSearchQueryBuilder.containsWildcard("custo*"));
+ assertTrue(AtlasOpenSearchQueryBuilder.containsWildcard("atlas?"));
+ assertFalse(AtlasOpenSearchQueryBuilder.containsWildcard("atlas"));
+ assertFalse(AtlasOpenSearchQueryBuilder.containsWildcard("atlas\\*"));
+ }
+
+ private AtlasOpenSearchQueryBuilder builder(String queryString) {
+ return new AtlasOpenSearchQueryBuilder()
+ .withEntityTypes(entityTypes)
+ .withQueryString(queryString)
+ .withExcludedDeletedEntities(true)
+ .withIncludeSubTypes(true)
+ .withCommonIndexFieldNames(indexFieldNameCache)
+ .withSearchWeights(searchWeights);
+ }
+}
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchAtlasJanusTestSupport.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchAtlasJanusTestSupport.java
new file mode 100644
index 000000000..5a72d5b01
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchAtlasJanusTestSupport.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.ConfigurationConverter;
+
+import java.util.Properties;
+
+/**
+ * Builds JanusGraph {@link Configuration} objects for OpenSearch integration tests using the
+ * same flat {@code index.search.*} property layout as the janusgraph-opensearch smoke drivers.
+ */
+final class OpenSearchAtlasJanusTestSupport {
+
+ private OpenSearchAtlasJanusTestSupport() {
+ }
+
+ static Configuration buildJanusGraphConfiguration(Configuration atlasJanusSubset, String host, int port, String indexName) {
+ Properties properties = new Properties();
+
+ properties.setProperty("storage.backend", atlasJanusSubset.getString("storage.backend", "berkeleyje"));
+ properties.setProperty("storage.directory", atlasJanusSubset.getString("storage.directory"));
+ properties.setProperty("storage.transactions", String.valueOf(atlasJanusSubset.getBoolean("storage.transactions", true)));
+ properties.setProperty("index.search.backend", "opensearch");
+ properties.setProperty("index.search.hostname", host);
+ properties.setProperty("index.search.port", String.valueOf(port));
+ properties.setProperty("index.search.max-result-set-size",
+ String.valueOf(atlasJanusSubset.getInt("index.search.max-result-set-size", 150)));
+ properties.setProperty("index.search.opensearch.setup-max-open-scroll-contexts", "false");
+
+ if (indexName != null) {
+ properties.setProperty("index.search.index-name", indexName);
+ }
+
+ return ConfigurationConverter.getConfiguration(properties);
+ }
+}
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchClassificationFilterIT.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchClassificationFilterIT.java
new file mode 100644
index 000000000..6cf948be5
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchClassificationFilterIT.java
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.graph.GraphSandboxUtil;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasCardinality;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.repository.graphdb.QuickSearchContext;
+import org.apache.atlas.repository.graphdb.QuickSearchResult;
+import org.apache.atlas.runner.OpenSearchITBase;
+import org.apache.commons.configuration2.Configuration;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphVertex;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+/**
+ * Live end-to-end coverage that classification filtering actually restricts results against a real
+ * OpenSearch server. Indexes vertices with a direct classification, a propagated-only classification, a different
+ * (non-matching) classification, and no classification at all, then drives the exact production entry point —
+ * {@link AtlasOpenSearchIndexClient#quickSearch(QuickSearchContext, Configuration)}, which builds the query via
+ * {@link AtlasOpenSearchQueryBuilder#withClassificationTypeNames} — and asserts only the direct/propagated matches
+ * come back. This complements (does not replace) the DSL-shape unit coverage in
+ * {@code AtlasOpenSearchQueryBuilderTest#classificationFilterUsesShouldOverBothClassificationFieldsWithKeywordSubfield}.
+ */
+public class OpenSearchClassificationFilterIT extends OpenSearchITBase {
+
+ private static final String GRAPH_INDEX_NAME = "classificationfilter";
+ private static final String PHYSICAL_INDEX = GRAPH_INDEX_NAME + "_vertex_index";
+
+ private JanusGraph janusGraph;
+ private Configuration atlasConfig;
+
+ @BeforeClass(dependsOnMethods = "startOpenSearchContainer")
+ public void setUp() throws Exception {
+ if (Runtime.version().feature() >= 25) {
+ throw new org.testng.SkipException(
+ "AtlasJanusGraphDatabase static init is not supported on JDK 25+ in this test");
+ }
+
+ String opensearchHost = org.apache.atlas.runner.OpenSearchTestContainerRunner.getHost();
+ int opensearchPort = org.apache.atlas.runner.OpenSearchTestContainerRunner.getPort();
+
+ File dataDir = new File(System.getProperty("java.io.tmpdir"), "atlas-opensearch-classification-filter-test");
+ deleteRecursively(dataDir.toPath());
+ System.setProperty("atlas.data", dataDir.getAbsolutePath());
+ System.setProperty("atlas.properties", "atlas-opensearch-application.properties");
+
+ ApplicationProperties.forceReload();
+ GraphSandboxUtil.create("opensearch-classification-filter");
+ AtlasJanusGraphDatabase.unload();
+
+ atlasConfig = ApplicationProperties.get();
+ atlasConfig.setProperty("atlas.graph.index.search.hostname", opensearchHost);
+ atlasConfig.setProperty("atlas.graph.index.search.port", opensearchPort);
+ atlasConfig.setProperty("atlas.graph.index.search.index-name", GRAPH_INDEX_NAME);
+ atlasConfig.setProperty(ApplicationProperties.INDEX_RECOVERY_CONF, false);
+
+ deletePhysicalIndexIfPresent(opensearchHost, opensearchPort);
+
+ Class.forName(AtlasJanusGraphDatabase.class.getName());
+ Configuration atlasJanusConfig = AtlasJanusGraphDatabase.getConfiguration();
+ Configuration janusConfig = OpenSearchAtlasJanusTestSupport.buildJanusGraphConfiguration(
+ atlasJanusConfig, opensearchHost, opensearchPort, GRAPH_INDEX_NAME);
+ janusGraph = AtlasJanusGraphDatabase.initJanusGraph(janusConfig);
+
+ AtlasJanusGraph atlasGraph = new AtlasJanusGraph(janusGraph);
+
+ try (AtlasGraphManagement mgmt = atlasGraph.getManagementSystem()) {
+ mgmt.createVertexMixedIndex(Constants.VERTEX_INDEX, Constants.BACKING_INDEX, Collections.emptyList());
+
+ AtlasPropertyKey guidKey = mgmt.makePropertyKey(Constants.GUID_PROPERTY_KEY, String.class, AtlasCardinality.SINGLE);
+ AtlasPropertyKey classKey = mgmt.makePropertyKey(Constants.CLASSIFICATION_NAMES_KEY, String.class, AtlasCardinality.SINGLE);
+ AtlasPropertyKey propagatedKey = mgmt.makePropertyKey(Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY, String.class, AtlasCardinality.SINGLE);
+
+ // guid: stored only (no search needed on it).
+ mgmt.addMixedIndex(Constants.VERTEX_INDEX, guidKey, true, false);
+ // classification fields: text + keyword subfield, mirroring the production GraphBackedSearchIndexer
+ // path — this is what causes AtlasOpenSearchIndexClient.usesKeywordSubfield(...) to be true below.
+ mgmt.addMixedIndex(Constants.VERTEX_INDEX, classKey, false, true);
+ mgmt.addMixedIndex(Constants.VERTEX_INDEX, propagatedKey, false, true);
+
+ mgmt.updateSchemaStatus();
+ mgmt.setIsSuccess(true);
+ }
+ atlasGraph.commit();
+
+ addVertex("guid-direct-pii", Constants.CLASSIFICATION_NAMES_KEY, "PII");
+ addVertex("guid-propagated-pii", Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY, "PII");
+ addVertex("guid-other-classification", Constants.CLASSIFICATION_NAMES_KEY, "CONFIDENTIAL");
+ addVertex("guid-no-classification", null, null);
+ janusGraph.tx().commit();
+
+ Thread.sleep(1500);
+ }
+
+ private void addVertex(String guid, String classificationField, String classificationValue) {
+ JanusGraphVertex vertex = janusGraph.addVertex();
+
+ vertex.property(Constants.GUID_PROPERTY_KEY, guid);
+
+ if (classificationField != null) {
+ vertex.property(classificationField, classificationValue);
+ }
+ }
+
+ @AfterClass
+ public void tearDown() {
+ if (janusGraph != null && janusGraph.isOpen()) {
+ janusGraph.close();
+ }
+
+ AtlasJanusGraphDatabase.unload();
+ ApplicationProperties.forceReload();
+ }
+
+ @Test
+ public void classificationFilterRestrictsResultsToDirectOrPropagatedMatches() throws Exception {
+ assertTrue(AtlasOpenSearchIndexClient.usesKeywordSubfield(Constants.CLASSIFICATION_NAMES_KEY),
+ "precondition: classification field must be keyword-registered like production");
+ assertTrue(AtlasOpenSearchIndexClient.usesKeywordSubfield(Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY),
+ "precondition: propagated classification field must be keyword-registered like production");
+
+ Map<String, String> indexFieldNameCache = new HashMap<>();
+ indexFieldNameCache.put(Constants.CLASSIFICATION_NAMES_KEY, Constants.CLASSIFICATION_NAMES_KEY);
+ indexFieldNameCache.put(Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY, Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY);
+
+ Set<String> classificationTypeNames = new HashSet<>();
+ classificationTypeNames.add("PII");
+
+ QuickSearchContext context = new QuickSearchContext(null, null, null, classificationTypeNames,
+ indexFieldNameCache, false, false, 0, 50);
+
+ QuickSearchResult result = AtlasOpenSearchIndexClient.quickSearch(context, atlasConfig);
+ List<String> guids = result.getEntityGuids();
+
+ assertEquals(result.getTotalCount(), 2,
+ "classification filter for PII must restrict to exactly the direct + propagated matches: " + guids);
+ assertTrue(guids.contains("guid-direct-pii"), "expected direct PII match in " + guids);
+ assertTrue(guids.contains("guid-propagated-pii"), "expected propagated PII match in " + guids);
+ assertFalse(guids.contains("guid-other-classification"), "must not match a different classification: " + guids);
+ assertFalse(guids.contains("guid-no-classification"), "must not match an unclassified vertex: " + guids);
+ }
+
+ @Test(dependsOnMethods = "classificationFilterRestrictsResultsToDirectOrPropagatedMatches")
+ public void noClassificationFilterReturnsAllVertices() throws Exception {
+ // Sanity check that the restriction above comes from the classification filter itself, not from some
+ // unrelated cause (e.g. all documents being excluded) — with no classification filter, all 4 come back.
+ QuickSearchContext context = new QuickSearchContext(null, null, null, Collections.emptySet(),
+ new HashMap<>(), false, false, 0, 50);
+
+ QuickSearchResult result = AtlasOpenSearchIndexClient.quickSearch(context, atlasConfig);
+
+ assertEquals(result.getTotalCount(), 4,
+ "expected all 4 seeded vertices when no classification filter is applied: " + result.getEntityGuids());
+ }
+
+ private static void deleteRecursively(Path path) throws IOException {
+ if (!Files.exists(path)) {
+ return;
+ }
+
+ Files.walk(path)
+ .sorted(java.util.Comparator.reverseOrder())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ private static void deletePhysicalIndexIfPresent(String host, int port) throws IOException {
+ URL url = new URL("http://" + host + ":" + port + "/" + PHYSICAL_INDEX);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(5000);
+ connection.setRequestMethod("DELETE");
+ connection.getResponseCode();
+ }
+}
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchQueryCaptureDriver.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchQueryCaptureDriver.java
new file mode 100644
index 000000000..42b453c96
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchQueryCaptureDriver.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Emits OpenSearch Query DSL (and related request fragments) for documentation and debugging.
+ * Output format: one JSON object per line with keys scenarioId, atlasQuery, opensearchQuery, notes.
+ */
+public final class OpenSearchQueryCaptureDriver {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
+
+ private OpenSearchQueryCaptureDriver() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
+ AtlasEntityDef datasetDef = new AtlasEntityDef();
+ datasetDef.setName("test_dataset");
+ AtlasAttributeDef ownerAttr = new AtlasAttributeDef("owner", "string");
+ ownerAttr.setIndexType(AtlasAttributeDef.IndexType.STRING);
+ datasetDef.setAttributeDefs(List.of(ownerAttr));
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+ typesDef.getEntityDefs().add(datasetDef);
+ typeRegistry.updateTypes(typesDef);
+ AtlasEntityType datasetType = typeRegistry.getEntityTypeByName("test_dataset");
+ datasetType.getAttribute("owner").setIndexFieldName("owner_index");
+
+ Map<String, String> indexFieldNameCache = new HashMap<>();
+ indexFieldNameCache.put(Constants.ENTITY_TYPE_PROPERTY_KEY, "__typeName");
+ indexFieldNameCache.put(Constants.STATE_PROPERTY_KEY, "__state");
+ indexFieldNameCache.put("owner", "owner_index");
+ indexFieldNameCache.put("name", "name_index");
+ indexFieldNameCache.put("labels", "labels_index");
+
+ Map<String, Integer> searchWeights = new LinkedHashMap<>();
+ searchWeights.put("name_index", 10);
+ searchWeights.put("owner_index", 3);
+ searchWeights.put("labels_index", 10);
+
+ Set<AtlasEntityType> entityTypes = new HashSet<>(Collections.singletonList(datasetType));
+
+ captureQuickSearch("quick-search-atlas-term", "atlas", indexFieldNameCache, searchWeights, entityTypes);
+ captureQuickSearch("quick-search-customer-wildcard", "customer*", indexFieldNameCache, searchWeights, entityTypes);
+ captureQuickSearch("quick-search-custo-prefix", "custo*", indexFieldNameCache, searchWeights, entityTypes);
+ captureQuickSearch("quick-search-custo-prefix-filtered", "custo*", indexFieldNameCache, searchWeights, entityTypes);
+ captureQuickSearch("quick-search-colon-in-query", "A:B", indexFieldNameCache, searchWeights, entityTypes);
+ captureQuickSearch("quick-search-colon-special-char", "A:B", indexFieldNameCache, searchWeights, entityTypes);
+
+ Map<String, Object> aggQuery = new AtlasOpenSearchQueryBuilder()
+ .withEntityTypes(entityTypes)
+ .withQueryString("atlas")
+ .withCommonIndexFieldNames(indexFieldNameCache)
+ .withSearchWeights(searchWeights)
+ .withExcludedDeletedEntities(true)
+ .buildDiscoveryQuery();
+ Map<String, Object> aggBody = new LinkedHashMap<>();
+ aggBody.put("size", 0);
+ aggBody.put("query", aggQuery);
+ aggBody.put("aggs", Map.of("agg_0", Map.of("terms", Map.of("field", "__typeName", "size", 100))));
+ emit("aggregation-type-name-filter", "atlas + owner=team-alpha filter", aggBody, "POST /search/quick aggregation on __typeName");
+
+ Map<String, Object> suggestBody = new LinkedHashMap<>();
+ suggestBody.put("query", AtlasOpenSearchIndexClient.buildSuggestionsFilterQuery());
+ suggestBody.put("aggs", AtlasOpenSearchIndexClient.buildSuggestionsTermsAggs(
+ List.of("owner_index", "name_index"), "cust"));
+ emit("suggestions-cust-prefix", "cust", suggestBody, "POST /search/suggestions prefixString=cust");
+
+ emitTermsPattern("suggestions-terms-include-pattern", "cust_", AtlasOpenSearchIndexClient.toTermsIncludePattern("cust_"));
+ }
+
+ private static void captureQuickSearch(String scenarioId, String atlasQuery,
+ Map<String, String> indexFieldNameCache,
+ Map<String, Integer> searchWeights,
+ Set<AtlasEntityType> entityTypes) throws Exception {
+ Map<String, Object> query = new AtlasOpenSearchQueryBuilder()
+ .withEntityTypes(entityTypes)
+ .withQueryString(atlasQuery)
+ .withCommonIndexFieldNames(indexFieldNameCache)
+ .withSearchWeights(searchWeights)
+ .withExcludedDeletedEntities(true)
+ .buildDiscoveryQuery();
+
+ Map<String, Object> body = new LinkedHashMap<>();
+ body.put("query", query);
+ body.put("from", 0);
+ body.put("size", 10);
+ body.put("track_total_hits", true);
+
+ emit(scenarioId, atlasQuery, body, "GET /search/quick weighted quick-search");
+ }
+
+ private static void emitTermsPattern(String scenarioId, String prefix, String pattern) throws Exception {
+ Map<String, Object> row = new LinkedHashMap<>();
+ row.put("scenarioId", scenarioId);
+ row.put("atlasQuery", prefix);
+ row.put("opensearchQuery", Map.of("terms.include", pattern));
+ row.put("notes", "terms aggregation include regex for suggestions");
+ System.out.println(MAPPER.writeValueAsString(row));
+ }
+
+ private static void emit(String scenarioId, String atlasQuery, Object opensearchBody, String notes) throws Exception {
+ Map<String, Object> row = new LinkedHashMap<>();
+ row.put("scenarioId", scenarioId);
+ row.put("atlasQuery", atlasQuery);
+ row.put("opensearchQuery", opensearchBody);
+ row.put("notes", notes);
+ System.out.println(MAPPER.writeValueAsString(row));
+ }
+}
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchRuntimeInitTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchRuntimeInitTest.java
new file mode 100644
index 000000000..ad5d08365
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchRuntimeInitTest.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.graph.GraphSandboxUtil;
+import org.apache.atlas.runner.OpenSearchITBase;
+import org.apache.commons.configuration2.Configuration;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.diskstorage.StandardIndexProvider;
+import org.janusgraph.diskstorage.opensearch.OpenSearchMajorVersion;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+/**
+ * Validates Atlas JanusGraph initialization against OpenSearch 2.x via Testcontainers.
+ */
+public class OpenSearchRuntimeInitTest extends OpenSearchITBase {
+
+ private static final String OPENSEARCH_INDEX_CLASS = "org.janusgraph.diskstorage.opensearch.AtlasOpenSearchIndex";
+
+ private JanusGraph graph;
+
+ @org.testng.annotations.BeforeClass(dependsOnMethods = "startOpenSearchContainer")
+ public void setUp() throws Exception {
+ if (Runtime.version().feature() >= 25) {
+ throw new org.testng.SkipException(
+ "AtlasJanusGraphDatabase static init is not supported on JDK 25+ in this test");
+ }
+
+ String opensearchHost = org.apache.atlas.runner.OpenSearchTestContainerRunner.getHost();
+ int opensearchPort = org.apache.atlas.runner.OpenSearchTestContainerRunner.getPort();
+
+ File dataDir = new File(System.getProperty("java.io.tmpdir"), "atlas-opensearch-init-test");
+ deleteRecursively(dataDir.toPath());
+ System.setProperty("atlas.data", dataDir.getAbsolutePath());
+ System.setProperty("atlas.properties", "atlas-opensearch-application.properties");
+
+ ApplicationProperties.forceReload();
+ GraphSandboxUtil.create("opensearch-runtime-init");
+ AtlasJanusGraphDatabase.unload();
+
+ Configuration config = ApplicationProperties.get();
+ config.setProperty("atlas.graph.index.search.hostname", opensearchHost);
+ config.setProperty("atlas.graph.index.search.port", opensearchPort);
+ config.setProperty(ApplicationProperties.INDEX_RECOVERY_CONF, false);
+
+ Class.forName(AtlasJanusGraphDatabase.class.getName());
+ }
+
+ @AfterClass
+ public void tearDown() {
+ if (graph != null && graph.isOpen()) {
+ graph.close();
+ }
+ AtlasJanusGraphDatabase.unload();
+ ApplicationProperties.forceReload();
+ }
+
+ @Test
+ public void standardIndexProviderRegistersOpenSearchBeforeGraphOpen() {
+ Map<String, String> providers = StandardIndexProvider.getAllProviderClasses();
+
+ assertEquals(providers.get("opensearch"), OPENSEARCH_INDEX_CLASS,
+ "StandardIndexProvider must map opensearch backend to AtlasOpenSearchIndex");
+ assertEquals(providers.get("elasticsearch"),
+ "org.janusgraph.diskstorage.es.ElasticSearch7Index",
+ "Existing elasticsearch backend registration must remain unchanged");
+ }
+
+ @Test(dependsOnMethods = "standardIndexProviderRegistersOpenSearchBeforeGraphOpen")
+ public void janusGraphOpensWithOpenSearchBackend() throws Exception {
+ Configuration atlasJanusConfig = AtlasJanusGraphDatabase.getConfiguration();
+
+ assertEquals(atlasJanusConfig.getString(AtlasJanusGraphDatabase.INDEX_BACKEND_CONF), "opensearch");
+
+ String host = org.apache.atlas.runner.OpenSearchTestContainerRunner.getHost();
+ int port = org.apache.atlas.runner.OpenSearchTestContainerRunner.getPort();
+ Configuration janusConfig = OpenSearchAtlasJanusTestSupport.buildJanusGraphConfiguration(
+ atlasJanusConfig, host, port, null);
+
+ graph = AtlasJanusGraphDatabase.initJanusGraph(janusConfig);
+
+ assertNotNull(graph);
+ assertTrue(graph.isOpen());
+ OpenSearchMajorVersion majorVersion = readOpenSearchMajorVersion();
+ assertTrue(majorVersion == OpenSearchMajorVersion.TWO || majorVersion == OpenSearchMajorVersion.THREE,
+ "Expected OpenSearch 2.x or 3.x, got " + majorVersion);
+ }
+
+ @Test(dependsOnMethods = "standardIndexProviderRegistersOpenSearchBeforeGraphOpen")
+ public void janusGraphFactoryOpenResolvesOpenSearchIndexDirectly() throws Exception {
+ String host = org.apache.atlas.runner.OpenSearchTestContainerRunner.getHost();
+ int port = org.apache.atlas.runner.OpenSearchTestContainerRunner.getPort();
+ Configuration conf2 = OpenSearchAtlasJanusTestSupport.buildJanusGraphConfiguration(
+ AtlasJanusGraphDatabase.getConfiguration(), host, port, null);
+
+ try (JanusGraph directGraph = JanusGraphFactory.open(conf2)) {
+ assertNotNull(directGraph);
+ assertTrue(directGraph.isOpen());
+ }
+ }
+
+ private static void deleteRecursively(Path path) throws IOException {
+ if (!Files.exists(path)) {
+ return;
+ }
+
+ Files.walk(path)
+ .sorted(java.util.Comparator.reverseOrder())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ private static OpenSearchMajorVersion readOpenSearchMajorVersion() throws IOException {
+ String host = org.apache.atlas.runner.OpenSearchTestContainerRunner.getHost();
+ int port = org.apache.atlas.runner.OpenSearchTestContainerRunner.getPort();
+ URL url = new URL("http://" + host + ":" + port + "/");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(2000);
+ connection.setReadTimeout(2000);
+ connection.setRequestMethod("GET");
+ try (java.io.InputStream in = connection.getInputStream()) {
+ String body = new String(in.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8);
+ int numberStart = body.indexOf("\"number\" : \"") + 12;
+ int numberEnd = body.indexOf('"', numberStart);
+ return OpenSearchMajorVersion.parse(body.substring(numberStart, numberEnd));
+ }
+ }
+}
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchStateKeywordMappingIT.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchStateKeywordMappingIT.java
new file mode 100644
index 000000000..0370e3480
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/OpenSearchStateKeywordMappingIT.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.graph.GraphSandboxUtil;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasCardinality;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.runner.OpenSearchITBase;
+import org.apache.commons.configuration2.Configuration;
+import org.janusgraph.core.JanusGraph;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.testng.Assert.assertTrue;
+
+/**
+ * Verifies that {@link Constants#STATE_PROPERTY_KEY} and {@link Constants#ENTITY_TYPE_PROPERTY_KEY}
+ * are mapped as OpenSearch text with a {@code keyword} subfield when registered through
+ * {@link AtlasJanusGraphManagement#addMixedIndex} with {@code withKeywordSubfield=true},
+ * matching the GraphBackedSearchIndexer production path.
+ */
+public class OpenSearchStateKeywordMappingIT extends OpenSearchITBase {
+
+ private static final String GRAPH_INDEX_NAME = "statemapping";
+ private static final String PHYSICAL_INDEX = GRAPH_INDEX_NAME + "_vertex_index";
+
+ private JanusGraph janusGraph;
+
+ @BeforeClass(dependsOnMethods = "startOpenSearchContainer")
+ public void setUp() throws Exception {
+ if (Runtime.version().feature() >= 25) {
+ throw new org.testng.SkipException(
+ "AtlasJanusGraphDatabase static init is not supported on JDK 25+ in this test");
+ }
+
+ String opensearchHost = org.apache.atlas.runner.OpenSearchTestContainerRunner.getHost();
+ int opensearchPort = org.apache.atlas.runner.OpenSearchTestContainerRunner.getPort();
+
+ File dataDir = new File(System.getProperty("java.io.tmpdir"), "atlas-opensearch-state-mapping-test");
+ deleteRecursively(dataDir.toPath());
+ System.setProperty("atlas.data", dataDir.getAbsolutePath());
+ System.setProperty("atlas.properties", "atlas-opensearch-application.properties");
+
+ ApplicationProperties.forceReload();
+ GraphSandboxUtil.create("opensearch-state-mapping");
+ AtlasJanusGraphDatabase.unload();
+
+ Configuration config = ApplicationProperties.get();
+ config.setProperty("atlas.graph.index.search.hostname", opensearchHost);
+ config.setProperty("atlas.graph.index.search.port", opensearchPort);
+ config.setProperty("atlas.graph.index.search.index-name", GRAPH_INDEX_NAME);
+ config.setProperty(ApplicationProperties.INDEX_RECOVERY_CONF, false);
+
+ deletePhysicalIndexIfPresent(opensearchHost, opensearchPort);
+
+ Class.forName(AtlasJanusGraphDatabase.class.getName());
+ Configuration atlasJanusConfig = AtlasJanusGraphDatabase.getConfiguration();
+ Configuration janusConfig = OpenSearchAtlasJanusTestSupport.buildJanusGraphConfiguration(
+ atlasJanusConfig, opensearchHost, opensearchPort, GRAPH_INDEX_NAME);
+ janusGraph = AtlasJanusGraphDatabase.initJanusGraph(janusConfig);
+
+ AtlasJanusGraph atlasGraph = new AtlasJanusGraph(janusGraph);
+
+ try (AtlasGraphManagement mgmt = atlasGraph.getManagementSystem()) {
+ mgmt.createVertexMixedIndex(Constants.VERTEX_INDEX, Constants.BACKING_INDEX, java.util.Collections.emptyList());
+
+ AtlasPropertyKey stateKey = mgmt.makePropertyKey(Constants.STATE_PROPERTY_KEY, String.class, AtlasCardinality.SINGLE);
+ AtlasPropertyKey typeKey = mgmt.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, AtlasCardinality.SINGLE);
+ mgmt.addMixedIndex(Constants.VERTEX_INDEX, stateKey, false, true);
+ mgmt.addMixedIndex(Constants.VERTEX_INDEX, typeKey, false, true);
+
+ mgmt.updateSchemaStatus();
+ mgmt.setIsSuccess(true);
+ }
+ atlasGraph.commit();
+
+ Thread.sleep(1500);
+ }
+
+ @AfterClass
+ public void tearDown() {
+ if (janusGraph != null && janusGraph.isOpen()) {
+ janusGraph.close();
+ }
+ AtlasJanusGraphDatabase.unload();
+ ApplicationProperties.forceReload();
+ }
+
+ @Test
+ public void stateAndTypeNameFieldsHaveKeywordSubfield() throws Exception {
+ String host = org.apache.atlas.runner.OpenSearchTestContainerRunner.getHost();
+ int port = org.apache.atlas.runner.OpenSearchTestContainerRunner.getPort();
+ String mapping = httpGet(host, port, "/" + PHYSICAL_INDEX + "/_mapping");
+
+ assertTrue(mapping.contains("\"__state\""), "mapping should contain __state: " + mapping);
+ assertTrue(mapping.contains("\"__typeName\""), "mapping should contain __typeName: " + mapping);
+ assertTextWithKeywordSubfield(mapping, "__state");
+ assertTextWithKeywordSubfield(mapping, "__typeName");
+ assertTrue(AtlasOpenSearchIndexClient.usesKeywordSubfield("__state"));
+ assertTrue(AtlasOpenSearchIndexClient.usesKeywordSubfield("__typeName"));
+ }
+
+ private static void deleteRecursively(Path path) throws IOException {
+ if (!Files.exists(path)) {
+ return;
+ }
+
+ Files.walk(path)
+ .sorted(java.util.Comparator.reverseOrder())
+ .forEach(p -> {
+ try {
+ Files.deleteIfExists(p);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ private static void assertTextWithKeywordSubfield(String mapping, String fieldName) {
+ int fieldStart = mapping.indexOf("\"" + fieldName + "\"");
+ assertTrue(fieldStart >= 0, "mapping should contain " + fieldName + ": " + mapping);
+
+ String fieldSection = mapping.substring(fieldStart, Math.min(mapping.length(), fieldStart + 400));
+ assertTrue(fieldSection.contains("\"type\":\"text\"") || fieldSection.contains("\"type\" : \"text\""),
+ fieldName + " should be text, got: " + fieldSection);
+ assertTrue(fieldSection.contains("\"keyword\""), fieldName + " should have keyword subfield, got: " + fieldSection);
+ }
+
+ private static void deletePhysicalIndexIfPresent(String host, int port) throws IOException {
+ URL url = new URL("http://" + host + ":" + port + "/" + PHYSICAL_INDEX);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(5000);
+ connection.setRequestMethod("DELETE");
+ connection.getResponseCode();
+ }
+
+ private static String httpGet(String host, int port, String path) throws IOException {
+ URL url = new URL("http://" + host + ":" + port + path);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(5000);
+ connection.setRequestMethod("GET");
+ try (InputStream in = connection.getInputStream()) {
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+}
diff --git a/graphdb/janus/src/test/java/org/janusgraph/diskstorage/opensearch/AtlasOpenSearchIndexInstanceTest.java b/graphdb/janus/src/test/java/org/janusgraph/diskstorage/opensearch/AtlasOpenSearchIndexInstanceTest.java
new file mode 100644
index 000000000..ad595834a
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/janusgraph/diskstorage/opensearch/AtlasOpenSearchIndexInstanceTest.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch;
+
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertSame;
+import static org.testng.Assert.assertTrue;
+
+/**
+ * Verifies the shared static-client lifecycle guard: closing an OpenSearch index instance clears the
+ * shared static reference only when it still points to that instance, so a short-lived (e.g. bulk-loading) index
+ * closing cannot wipe a different graph's live client reference.
+ */
+public class AtlasOpenSearchIndexInstanceTest {
+
+ @AfterMethod
+ public void resetInstance() {
+ AtlasOpenSearchIndex.setInstanceForTests(null);
+ }
+
+ @Test
+ public void clearOnlyWhenStaticStillPointsToClosingInstance() {
+ AtlasOpenSearchIndex primary = mock(AtlasOpenSearchIndex.class);
+ AtlasOpenSearchIndex bulk = mock(AtlasOpenSearchIndex.class);
+
+ // Primary graph owns the static reference.
+ AtlasOpenSearchIndex.setInstanceForTests(primary);
+
+ // A different (bulk) instance closing must NOT clear the primary's reference.
+ assertFalse(AtlasOpenSearchIndex.clearInstanceIfCurrent(bulk));
+ assertSame(AtlasOpenSearchIndex.getInstanceForTests(), primary);
+
+ // The owning instance closing clears the reference.
+ assertTrue(AtlasOpenSearchIndex.clearInstanceIfCurrent(primary));
+ assertNull(AtlasOpenSearchIndex.getInstanceForTests());
+ }
+
+ @Test
+ public void clearIsIdempotentWhenAlreadyNull() {
+ AtlasOpenSearchIndex.setInstanceForTests(null);
+ AtlasOpenSearchIndex some = mock(AtlasOpenSearchIndex.class);
+
+ assertFalse(AtlasOpenSearchIndex.clearInstanceIfCurrent(some));
+ assertNull(AtlasOpenSearchIndex.getInstanceForTests());
+ }
+}
diff --git a/graphdb/janus/src/test/resources/atlas-opensearch-application.properties b/graphdb/janus/src/test/resources/atlas-opensearch-application.properties
new file mode 100644
index 000000000..b7e8d1cf3
--- /dev/null
+++ b/graphdb/janus/src/test/resources/atlas-opensearch-application.properties
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Minimal Atlas config for OpenSearch 2.x runtime initialization validation.
+#
+
+atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
+
+atlas.graph.storage.backend=berkeleyje
+atlas.graph.storage.directory=${sys:atlas.data}/berkeley
+atlas.graph.storage.transactions=true
+
+atlas.graph.index.search.backend=opensearch
+atlas.graph.index.search.hostname=localhost
+atlas.graph.index.search.port=9200
+atlas.graph.index.search.max-result-set-size=150
+atlas.graph.index.search.opensearch.setup-max-open-scroll-contexts=false
+
+atlas.graph.index.recovery.enable=false
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/OpenSearchIndexIdempotentCreateTest.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/OpenSearchIndexIdempotentCreateTest.java
new file mode 100644
index 000000000..6dd82ce42
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/OpenSearchIndexIdempotentCreateTest.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch;
+
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+/**
+ * Unit tests for {@link OpenSearchIndex#createIndexIdempotent(OpenSearchClient, String, Map)} — the HA / concurrent
+ * index-creation safety fix. Two Atlas nodes starting concurrently may both attempt to create the same index; the
+ * loser must treat an already-existing index as success without masking genuine creation failures.
+ */
+public class OpenSearchIndexIdempotentCreateTest {
+
+ private static final String INDEX = "janusgraph_vertex_index";
+ private static final Map<String, Object> SETTINGS = Collections.emptyMap();
+
+ @Test
+ public void createSucceedsWhenIndexIsAbsent() throws IOException {
+ FakeClient client = new FakeClient();
+ client.existsAfterCreate = true;
+
+ boolean created = OpenSearchIndex.createIndexIdempotent(client, INDEX, SETTINGS);
+
+ assertTrue(created, "create should report success when it created the index");
+ assertEquals(client.createCalls, 1);
+ }
+
+ @Test
+ public void concurrentAlreadyExistsIsTreatedAsSuccess() throws IOException {
+ // Simulate the race loser: createIndex fails (resource_already_exists), but the index exists afterwards
+ // because another node created it. This must NOT be surfaced as an error.
+ FakeClient client = new FakeClient();
+ client.failCreate = true;
+ client.existsAfterCreate = true;
+
+ boolean created = OpenSearchIndex.createIndexIdempotent(client, INDEX, SETTINGS);
+
+ assertFalse(created, "create should report it did not create the index (another node did)");
+ assertEquals(client.createCalls, 1);
+ assertEquals(client.existsCalls, 1, "should re-check existence exactly once after a failed create");
+ }
+
+ @Test
+ public void genuineFailureIsRethrownWhenIndexStillAbsent() {
+ // createIndex fails and the index still does not exist afterwards: this is a real error and must propagate.
+ FakeClient client = new FakeClient();
+ client.failCreate = true;
+ client.existsAfterCreate = false;
+
+ try {
+ OpenSearchIndex.createIndexIdempotent(client, INDEX, SETTINGS);
+ fail("expected the original IOException to be rethrown");
+ } catch (IOException expected) {
+ assertEquals(expected.getMessage(), "resource_already_exists_exception");
+ }
+ }
+
+ /** Minimal hand-written {@link OpenSearchClient} for the create/exists interactions under test. */
+ private static final class FakeClient implements OpenSearchClient {
+ boolean failCreate;
+ boolean existsAfterCreate;
+ int createCalls;
+ int existsCalls;
+
+ @Override
+ public void createIndex(String indexName, Map<String, Object> settings) throws IOException {
+ createCalls++;
+ if (failCreate) {
+ throw new IOException("resource_already_exists_exception");
+ }
+ }
+
+ @Override
+ public boolean indexExists(String indexName) {
+ existsCalls++;
+ return existsAfterCreate;
+ }
+
+ // ---- unused interface methods ----
+ @Override public OpenSearchMajorVersion getMajorVersion() { return null; }
+ @Override public void clusterHealthRequest(String timeout) { }
+ @Override public boolean isIndex(String indexName) { return false; }
+ @Override public boolean isAlias(String aliasName) { return false; }
+ @Override public void createStoredScript(String scriptName, Map<String, Object> script) { }
+ @Override public org.janusgraph.diskstorage.opensearch.script.OSScriptResponse getStoredScript(String scriptName) { return null; }
+ @Override public void updateIndexSettings(String indexName, Map<String, Object> settings) { }
+ @Override public void updateClusterSettings(Map<String, Object> settings) { }
+ @Override public Map getIndexSettings(String indexName) { return null; }
+ @Override public void createMapping(String indexName, String typeName, Map<String, Object> mapping) { }
+ @Override public org.janusgraph.diskstorage.opensearch.mapping.IndexMapping getMapping(String indexName, String typeName) { return null; }
+ @Override public void deleteIndex(String indexName) { }
+ @Override public void clearStore(String indexName, String storeName) { }
+ @Override public void bulkRequest(java.util.List<OpenSearchMutation> requests, String ingestPipeline) { }
+ @Override public long countTotal(String indexName, Map<String, Object> requestData) { return 0; }
+ @Override public Number min(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) { return null; }
+ @Override public Number max(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) { return null; }
+ @Override public double avg(String indexName, Map<String, Object> requestData, String fieldName) { return 0; }
+ @Override public Number sum(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) { return null; }
+ @Override public OpenSearchResponse search(String indexName, Map<String, Object> request, boolean useScroll) { return null; }
+ @Override public OpenSearchResponse search(String scrollId) { return null; }
+ @Override public void deleteScroll(String scrollId) { }
+ @Override public void addAlias(String alias, String index) { }
+ @Override public void close() { }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/OpenSearchIndexScrollWiringTest.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/OpenSearchIndexScrollWiringTest.java
new file mode 100644
index 000000000..afead0b1a
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/OpenSearchIndexScrollWiringTest.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch;
+
+import org.janusgraph.diskstorage.indexing.RawQuery;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+
+/**
+ * Unit tests for {@link OpenSearchIndex#withScrollCleanup(Iterator, Stream)} — the shared wiring used by every
+ * {@code query(...)} overload that may back its {@code Stream} with an {@link OpenSearchScroll}. This is the exact
+ * seam that was previously missing from the {@code query(RawQuery, ...)} path (used by Atlas' native/raw
+ * {@code graph.indexQuery(indexName, queryString)} calls), which allowed the server-side scroll context to leak on
+ * early termination. Docker-free.
+ */
+public class OpenSearchIndexScrollWiringTest {
+
+ private static final int BATCH_SIZE = 2;
+
+ @Test
+ public void wiresCloseToScrollWhenIteratorIsScroll() {
+ RecordingClient client = new RecordingClient();
+ // Full initial batch => not finished => scroll not deleted yet on construction.
+ OpenSearchScroll scroll = new OpenSearchScroll(client, response("scroll-1", BATCH_SIZE), BATCH_SIZE);
+
+ Stream<RawQuery.Result<String>> stream = OpenSearchIndex.withScrollCleanup(scroll, streamOf(scroll));
+
+ assertEquals(client.deleteScrollCalls, 0, "scroll must not be released before the stream is closed");
+
+ stream.close();
+
+ assertEquals(client.deleteScrollCalls, 1, "closing the wrapped stream must release the scroll context");
+ assertEquals(client.lastDeletedScrollId, "scroll-1");
+ }
+
+ @Test
+ public void closingBeforeLimitConsumesStreamStillReleasesScroll() {
+ // Mirrors query(RawQuery, ...) / query(IndexQuery, ...): a caller applies .limit(n) with n smaller than the
+ // batch and then closes the stream (e.g. via try-with-resources) without exhausting it.
+ RecordingClient client = new RecordingClient();
+ OpenSearchScroll scroll = new OpenSearchScroll(client, response("scroll-2", BATCH_SIZE), BATCH_SIZE);
+
+ try (Stream<RawQuery.Result<String>> limited =
+ OpenSearchIndex.withScrollCleanup(scroll, streamOf(scroll)).limit(1)) {
+ List<RawQuery.Result<String>> collected = new ArrayList<>();
+ limited.forEach(collected::add);
+ assertEquals(collected.size(), 1);
+ }
+
+ assertEquals(client.deleteScrollCalls, 1,
+ "early termination behind a limit() must still release the scroll context on stream close");
+ }
+
+ @Test
+ public void isNoOpForNonScrollIterator() {
+ // Single-page (non-scroll) result path: withScrollCleanup must not alter behavior or attempt any deletion.
+ RecordingClient client = new RecordingClient();
+ Iterator<RawQuery.Result<String>> plainIterator =
+ Collections.singletonList(new RawQuery.Result<>("doc-0", 1.0f)).iterator();
+ Stream<RawQuery.Result<String>> plainStream =
+ StreamSupport.stream(Spliterators.spliteratorUnknownSize(plainIterator, Spliterator.ORDERED), false);
+
+ Stream<RawQuery.Result<String>> wrapped = OpenSearchIndex.withScrollCleanup(plainIterator, plainStream);
+ List<RawQuery.Result<String>> collected = new ArrayList<>();
+ wrapped.forEach(collected::add);
+ wrapped.close();
+
+ assertEquals(collected.size(), 1);
+ assertEquals(client.deleteScrollCalls, 0, "non-scroll path must never call deleteScroll");
+ assertFalse(client.searchCalled, "non-scroll path must never touch the OpenSearch client at all");
+ }
+
+ private static Stream<RawQuery.Result<String>> streamOf(Iterator<RawQuery.Result<String>> iterator) {
+ return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
+ }
+
+ private static OpenSearchResponse response(String scrollId, int hits) {
+ OpenSearchResponse response = new OpenSearchResponse();
+ List<RawQuery.Result<String>> results = new ArrayList<>();
+ for (int i = 0; i < hits; i++) {
+ results.add(new RawQuery.Result<>(scrollId + "-doc-" + i, 1.0f));
+ }
+ response.setResults(results);
+ response.setScrollId(scrollId);
+ return response;
+ }
+
+ /** Minimal hand-written {@link OpenSearchClient} fake; tracks scroll delete/search interactions only. */
+ private static final class RecordingClient implements OpenSearchClient {
+ boolean searchCalled;
+ int deleteScrollCalls;
+ String lastDeletedScrollId;
+
+ @Override
+ public OpenSearchResponse search(String scrollId) throws IOException {
+ searchCalled = true;
+ return response(scrollId, 0);
+ }
+
+ @Override
+ public void deleteScroll(String scrollId) throws IOException {
+ deleteScrollCalls++;
+ lastDeletedScrollId = scrollId;
+ }
+
+ // ---- unused interface methods ----
+ @Override public OpenSearchMajorVersion getMajorVersion() { return null; }
+ @Override public void clusterHealthRequest(String timeout) { }
+ @Override public boolean indexExists(String indexName) { return false; }
+ @Override public boolean isIndex(String indexName) { return false; }
+ @Override public boolean isAlias(String aliasName) { return false; }
+ @Override public void createStoredScript(String scriptName, Map<String, Object> script) { }
+ @Override public org.janusgraph.diskstorage.opensearch.script.OSScriptResponse getStoredScript(String scriptName) { return null; }
+ @Override public void createIndex(String indexName, Map<String, Object> settings) { }
+ @Override public void updateIndexSettings(String indexName, Map<String, Object> settings) { }
+ @Override public void updateClusterSettings(Map<String, Object> settings) { }
+ @Override public Map getIndexSettings(String indexName) { return null; }
+ @Override public void createMapping(String indexName, String typeName, Map<String, Object> mapping) { }
+ @Override public org.janusgraph.diskstorage.opensearch.mapping.IndexMapping getMapping(String indexName, String typeName) { return null; }
+ @Override public void deleteIndex(String indexName) { }
+ @Override public void clearStore(String indexName, String storeName) { }
+ @Override public void bulkRequest(List<OpenSearchMutation> requests, String ingestPipeline) { }
+ @Override public long countTotal(String indexName, Map<String, Object> requestData) { return 0; }
+ @Override public Number min(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) { return null; }
+ @Override public Number max(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) { return null; }
+ @Override public double avg(String indexName, Map<String, Object> requestData, String fieldName) { return 0; }
+ @Override public Number sum(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) { return null; }
+ @Override public OpenSearchResponse search(String indexName, Map<String, Object> request, boolean useScroll) { return null; }
+ @Override public void addAlias(String alias, String index) { }
+ @Override public void close() { }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/OpenSearchScrollCleanupTest.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/OpenSearchScrollCleanupTest.java
new file mode 100644
index 000000000..1307b8cd0
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/OpenSearchScrollCleanupTest.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch;
+
+import org.janusgraph.diskstorage.indexing.RawQuery;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+/**
+ * Verifies the scroll-context cleanup behavior of {@link OpenSearchScroll}: the server-side scroll
+ * context must be released on normal exhaustion, on early termination via {@link OpenSearchScroll#close()}, and on
+ * a fetch exception — and cleanup must be best-effort and idempotent.
+ */
+public class OpenSearchScrollCleanupTest {
+
+ private static final int BATCH_SIZE = 2;
+
+ @Test
+ public void deletesScrollOnNormalExhaustion() {
+ // Initial batch smaller than batchSize => finished => scroll deleted eagerly in the constructor.
+ RecordingClient client = new RecordingClient();
+ OpenSearchScroll scroll = new OpenSearchScroll(client, response("scroll-1", 1), BATCH_SIZE);
+
+ assertTrue(scroll.hasNext());
+ scroll.next();
+ assertEquals(client.deleteScrollCalls, 1);
+
+ // Idempotent: closing an already-exhausted scroll does not re-delete.
+ scroll.close();
+ assertEquals(client.deleteScrollCalls, 1);
+ }
+
+ @Test
+ public void deletesScrollOnEarlyTerminationViaClose() {
+ // Full initial batch => not finished => scroll NOT deleted yet; simulates a consumer stopping early (limit).
+ RecordingClient client = new RecordingClient();
+ OpenSearchScroll scroll = new OpenSearchScroll(client, response("scroll-2", BATCH_SIZE), BATCH_SIZE);
+
+ assertEquals(client.deleteScrollCalls, 0, "not finished yet, so no delete on construction");
+
+ scroll.close();
+
+ assertEquals(client.deleteScrollCalls, 1, "close() must release the scroll context");
+ assertEquals(client.lastDeletedScrollId, "scroll-2");
+ }
+
+ @Test
+ public void deletesScrollOnFetchExceptionAndSurfacesOriginalError() {
+ RecordingClient client = new RecordingClient();
+ client.failNextSearch = true;
+ OpenSearchScroll scroll = new OpenSearchScroll(client, response("scroll-3", BATCH_SIZE), BATCH_SIZE);
+
+ // Drain the initial batch so hasNext() must fetch the next page, which fails.
+ scroll.next();
+ scroll.next();
+
+ try {
+ scroll.hasNext();
+ fail("expected UncheckedIOException from the failing fetch");
+ } catch (UncheckedIOException expected) {
+ assertEquals(client.deleteScrollCalls, 1, "scroll must be released even when the next fetch fails");
+ }
+ }
+
+ @Test
+ public void cleanupFailureDoesNotMaskFlow() {
+ RecordingClient client = new RecordingClient();
+ client.failDeleteScroll = true;
+ OpenSearchScroll scroll = new OpenSearchScroll(client, response("scroll-4", BATCH_SIZE), BATCH_SIZE);
+
+ // A failing deleteScroll must be swallowed (best-effort) — close() must not throw.
+ scroll.close();
+
+ assertEquals(client.deleteScrollCalls, 1);
+ }
+
+ private static OpenSearchResponse response(String scrollId, int hits) {
+ OpenSearchResponse response = new OpenSearchResponse();
+ List<RawQuery.Result<String>> results = new ArrayList<>();
+ for (int i = 0; i < hits; i++) {
+ results.add(new RawQuery.Result<>(scrollId + "-doc-" + i, 1.0f));
+ }
+ response.setResults(results);
+ response.setScrollId(scrollId);
+ return response;
+ }
+
+ /** Records scroll interactions; can be configured to fail the next search or the deleteScroll call. */
+ private static final class RecordingClient implements OpenSearchClient {
+ boolean failNextSearch;
+ boolean failDeleteScroll;
+ int deleteScrollCalls;
+ String lastDeletedScrollId;
+
+ @Override
+ public OpenSearchResponse search(String scrollId) throws IOException {
+ if (failNextSearch) {
+ throw new IOException("scroll fetch failed");
+ }
+ // Return an empty, finished page.
+ return response(scrollId, 0);
+ }
+
+ @Override
+ public void deleteScroll(String scrollId) throws IOException {
+ deleteScrollCalls++;
+ lastDeletedScrollId = scrollId;
+ if (failDeleteScroll) {
+ throw new IOException("delete scroll failed");
+ }
+ }
+
+ // ---- unused interface methods ----
+ @Override public OpenSearchMajorVersion getMajorVersion() { return null; }
+ @Override public void clusterHealthRequest(String timeout) { }
+ @Override public boolean indexExists(String indexName) { return false; }
+ @Override public boolean isIndex(String indexName) { return false; }
+ @Override public boolean isAlias(String aliasName) { return false; }
+ @Override public void createStoredScript(String scriptName, Map<String, Object> script) { }
+ @Override public org.janusgraph.diskstorage.opensearch.script.OSScriptResponse getStoredScript(String scriptName) { return null; }
+ @Override public void createIndex(String indexName, Map<String, Object> settings) { }
+ @Override public void updateIndexSettings(String indexName, Map<String, Object> settings) { }
+ @Override public void updateClusterSettings(Map<String, Object> settings) { }
+ @Override public Map getIndexSettings(String indexName) { return null; }
+ @Override public void createMapping(String indexName, String typeName, Map<String, Object> mapping) { }
+ @Override public org.janusgraph.diskstorage.opensearch.mapping.IndexMapping getMapping(String indexName, String typeName) { return null; }
+ @Override public void deleteIndex(String indexName) { }
+ @Override public void clearStore(String indexName, String storeName) { }
+ @Override public void bulkRequest(List<OpenSearchMutation> requests, String ingestPipeline) { }
+ @Override public long countTotal(String indexName, Map<String, Object> requestData) { return 0; }
+ @Override public Number min(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) { return null; }
+ @Override public Number max(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) { return null; }
+ @Override public double avg(String indexName, Map<String, Object> requestData, String fieldName) { return 0; }
+ @Override public Number sum(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) { return null; }
+ @Override public OpenSearchResponse search(String indexName, Map<String, Object> request, boolean useScroll) { return null; }
+ @Override public void addAlias(String alias, String index) { }
+ @Override public void close() { }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/rest/OpenSearchBulkWireFormatVerifier.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/rest/OpenSearchBulkWireFormatVerifier.java
new file mode 100644
index 000000000..e052a8242
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/rest/OpenSearchBulkWireFormatVerifier.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hc.core5.http.HttpHost;
+import org.janusgraph.diskstorage.opensearch.OpenSearchMutation;
+import org.opensearch.client.RestClient;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Bulk wire-format checks: typeless metadata and retry_on_conflict on updates.
+ */
+public final class OpenSearchBulkWireFormatVerifier {
+
+ private OpenSearchBulkWireFormatVerifier() {
+ }
+
+ public static void verify(String host, int port) throws Exception {
+ try (RestClient restClient = RestClient.builder(new HttpHost("http", host, port)).build();
+ RestOpenSearchClient client = new RestOpenSearchClient(
+ restClient, 60, false, 0, Collections.emptySet(), 1L, 1000L, 100_000_000)) {
+
+ client.setRetryOnConflict(3);
+
+ Map<String, Object> source = ImmutableMap.of("name", "Alice", "age", 30);
+ OpenSearchMutation indexMutation = OpenSearchMutation.createIndexRequest(
+ "c4janus_c4mixed", "c4mixed", "doc-1", source);
+ RestOpenSearchClient.RequestBytes indexBytes = client.new RequestBytes(indexMutation);
+ String indexLine = new String(indexBytes.requestBytes, StandardCharsets.UTF_8);
+ assertNoType(indexLine, "index");
+ if (!indexLine.contains("\"_id\":\"doc-1\"")) {
+ throw new IllegalStateException("Index bulk metadata missing _id: " + indexLine);
+ }
+ System.out.println("[OK] Bulk index metadata is typeless: " + indexLine.trim());
+
+ OpenSearchMutation updateMutation = OpenSearchMutation.createUpdateRequest(
+ "c4janus_c4mixed", "c4mixed", "doc-1", ImmutableMap.of("doc", ImmutableMap.of("age", 31)));
+ RestOpenSearchClient.RequestBytes updateBytes = client.new RequestBytes(updateMutation);
+ String updateLine = new String(updateBytes.requestBytes, StandardCharsets.UTF_8);
+ assertNoType(updateLine, "update");
+ if (!updateLine.contains("retry_on_conflict")) {
+ throw new IllegalStateException("Update bulk metadata missing retry_on_conflict: " + updateLine);
+ }
+ if (updateLine.contains("_retry_on_conflict")) {
+ throw new IllegalStateException("Update bulk metadata uses legacy _retry_on_conflict: " + updateLine);
+ }
+ System.out.println("[OK] Bulk update uses retry_on_conflict: " + updateLine.trim());
+
+ OpenSearchMutation deleteMutation = OpenSearchMutation.createDeleteRequest(
+ "c4janus_c4mixed", "c4mixed", "doc-1");
+ RestOpenSearchClient.RequestBytes deleteBytes = client.new RequestBytes(deleteMutation);
+ String deleteLine = new String(deleteBytes.requestBytes, StandardCharsets.UTF_8);
+ assertNoType(deleteLine, "delete");
+ System.out.println("[OK] Bulk delete metadata is typeless: " + deleteLine.trim());
+ }
+ }
+
+ private static void assertNoType(String bulkMetadataLine, String operation) {
+ if (bulkMetadataLine.contains("_type")) {
+ throw new IllegalStateException(
+ "Bulk " + operation + " metadata must not contain _type: " + bulkMetadataLine);
+ }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchConcurrentIndexCreationDriver.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchConcurrentIndexCreationDriver.java
new file mode 100644
index 000000000..d59804173
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchConcurrentIndexCreationDriver.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch.smoke;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.Mapping;
+
+import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.List;
+
+/**
+ * Validates that two Atlas/JanusGraph nodes creating the same OpenSearch-backed mixed index concurrently must
+ * both succeed. Before the idempotent-create fix, the node that lost the check-then-create race failed with
+ * {@code resource_already_exists_exception}. This driver reproduces that race against a real OpenSearch container.
+ */
+public final class OpenSearchConcurrentIndexCreationDriver {
+
+ public static final String GRAPH_INDEX_NAME = "concjanus";
+ public static final String MIXED_INDEX_NAME = "concmixed";
+ public static final String BACKING_INDEX_NAME = "search";
+ public static final String PHYSICAL_INDEX = GRAPH_INDEX_NAME + "_" + MIXED_INDEX_NAME.toLowerCase();
+
+ private OpenSearchConcurrentIndexCreationDriver() {
+ }
+
+ public static void execute() throws Exception {
+ OpenSearchSmokeSupport.verifyOpenSearchReachable();
+ OpenSearchSmokeSupport.registerOpenSearchBackend();
+ OpenSearchSmokeSupport.deletePhysicalIndex(PHYSICAL_INDEX);
+
+ int nodeCount = 2;
+ CountDownLatch startGate = new CountDownLatch(1);
+ CountDownLatch doneGate = new CountDownLatch(nodeCount);
+ List<Throwable> failures = new CopyOnWriteArrayList<>();
+
+ for (int i = 0; i < nodeCount; i++) {
+ final int nodeId = i;
+ Thread thread = new Thread(() -> {
+ try {
+ startGate.await(); // release both threads as simultaneously as possible to force the race
+ createMixedIndex();
+ } catch (Throwable t) {
+ failures.add(t);
+ } finally {
+ doneGate.countDown();
+ }
+ }, "os-index-creator-" + nodeId);
+ thread.start();
+ }
+
+ startGate.countDown();
+ doneGate.await();
+
+ if (!failures.isEmpty()) {
+ IllegalStateException error = new IllegalStateException(
+ "Concurrent OpenSearch index creation must succeed on all nodes but " + failures.size()
+ + " failed; first failure: " + failures.get(0));
+ error.initCause(failures.get(0));
+ throw error;
+ }
+
+ verifyPhysicalIndexCreated();
+ }
+
+ private static void createMixedIndex() {
+ Properties properties = new Properties();
+ properties.setProperty("storage.backend", "inmemory");
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".backend", "opensearch");
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".hostname", OpenSearchSmokeSupport.getOpenSearchHost());
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".port",
+ String.valueOf(OpenSearchSmokeSupport.getOpenSearchPort()));
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".index-name", GRAPH_INDEX_NAME);
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".opensearch.setup-max-open-scroll-contexts", "false");
+
+ org.apache.commons.configuration2.Configuration config =
+ org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties);
+
+ try (JanusGraph graph = JanusGraphFactory.open(config)) {
+ JanusGraphManagement mgmt = graph.openManagement();
+
+ if (mgmt.getGraphIndex(MIXED_INDEX_NAME) == null) {
+ PropertyKey nameField = mgmt.containsPropertyKey("nameField")
+ ? mgmt.getPropertyKey("nameField")
+ : mgmt.makePropertyKey("nameField").dataType(String.class).make();
+
+ mgmt.buildIndex(MIXED_INDEX_NAME, Vertex.class)
+ .addKey(nameField, Mapping.STRING.asParameter())
+ .buildMixedIndex(BACKING_INDEX_NAME);
+ }
+
+ mgmt.commit();
+ }
+ }
+
+ private static void verifyPhysicalIndexCreated() throws Exception {
+ String mapping = OpenSearchSmokeSupport.httpGet("/" + PHYSICAL_INDEX + "/_mapping");
+ if (!mapping.contains("nameField")) {
+ throw new IllegalStateException("Physical index mapping missing expected field 'nameField': " + mapping);
+ }
+
+ String settings = OpenSearchSmokeSupport.httpGet("/" + PHYSICAL_INDEX + "/_settings");
+ if (!settings.contains(PHYSICAL_INDEX)) {
+ throw new IllegalStateException("Physical index settings missing for " + PHYSICAL_INDEX + ": " + settings);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ execute();
+ System.out.println("[OK] Concurrent index creation succeeded on all nodes for " + PHYSICAL_INDEX);
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchConcurrentIndexCreationIT.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchConcurrentIndexCreationIT.java
new file mode 100644
index 000000000..478c0844f
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchConcurrentIndexCreationIT.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch.smoke;
+
+import org.apache.atlas.runner.OpenSearchITBase;
+import org.testng.annotations.Test;
+
+/**
+ * Concurrent index-creation (HA) safety integration test against a real OpenSearch container.
+ */
+public class OpenSearchConcurrentIndexCreationIT extends OpenSearchITBase {
+
+ @Test
+ public void concurrentIndexCreationSucceedsOnAllNodes() throws Exception {
+ OpenSearchConcurrentIndexCreationDriver.execute();
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchInitSmokeDriver.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchInitSmokeDriver.java
new file mode 100644
index 000000000..4cfb45a68
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchInitSmokeDriver.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch.smoke;
+
+import org.apache.hc.core5.http.HttpHost;
+import org.janusgraph.diskstorage.opensearch.OpenSearchMajorVersion;
+import org.janusgraph.diskstorage.opensearch.rest.RestOpenSearchClient;
+import org.opensearch.client.RestClient;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+
+/**
+ * Standalone smoke driver for OpenSearch REST client initialization. The client/version-parsing logic itself is
+ * not 3.x-specific, but the CI matrix (dedicated {@code opensearch-it} job) currently only validates against
+ * OpenSearch 3.7 and 3.8 — OpenSearch 2.x is NOT part of the CI matrix and has not been verified there.
+ *
+ * Run after starting OpenSearch on localhost:9200:
+ * <pre>
+ * mvn -pl graphdb/janusgraph-opensearch test-compile exec:java \
+ * -Dexec.classpathScope=test \
+ * -Dexec.mainClass=org.janusgraph.diskstorage.opensearch.smoke.OpenSearchInitSmokeDriver
+ * </pre>
+ */
+public final class OpenSearchInitSmokeDriver {
+
+ private static final String OPENSEARCH_HOST = System.getProperty("opensearch.host", "localhost");
+ private static final int OPENSEARCH_PORT = Integer.getInteger("opensearch.port", 9200);
+
+ private OpenSearchInitSmokeDriver() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ verifyOpenSearchReachable();
+ String versionNumber = verifyVersionParsing();
+
+ try (RestClient restClient = RestClient.builder(new HttpHost("http", OPENSEARCH_HOST, OPENSEARCH_PORT)).build();
+ RestOpenSearchClient client = new RestOpenSearchClient(
+ restClient, 60, false, 0, Collections.emptySet(), 1L, 1000L, 100_000_000)) {
+
+ OpenSearchMajorVersion version = client.getMajorVersion();
+ if (OpenSearchMajorVersion.TWO != version && OpenSearchMajorVersion.THREE != version) {
+ throw new IllegalStateException("RestOpenSearchClient detected " + version + " but expected TWO or THREE");
+ }
+ System.out.println("[OK] RestOpenSearchClient.getMajorVersion(): " + version);
+
+ client.clusterHealthRequest("30s");
+ System.out.println("[OK] GET /_cluster/health?wait_for_status=yellow&timeout=30s");
+ }
+
+ System.out.println("OpenSearch " + versionNumber + " runtime initialization smoke test passed.");
+ }
+
+ private static void verifyOpenSearchReachable() throws IOException {
+ URL url = new URL("http://" + OPENSEARCH_HOST + ":" + OPENSEARCH_PORT + "/");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(5000);
+ connection.setRequestMethod("GET");
+ if (connection.getResponseCode() != 200) {
+ throw new IOException("OpenSearch GET / returned HTTP " + connection.getResponseCode());
+ }
+ System.out.println("[OK] OpenSearch GET / reachable at " + OPENSEARCH_HOST + ":" + OPENSEARCH_PORT);
+ }
+
+ private static String verifyVersionParsing() throws IOException {
+ URL url = new URL("http://" + OPENSEARCH_HOST + ":" + OPENSEARCH_PORT + "/");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(5000);
+ connection.setRequestMethod("GET");
+ try (java.io.InputStream in = connection.getInputStream()) {
+ String body = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ int numberStart = body.indexOf("\"number\" : \"") + 12;
+ int numberEnd = body.indexOf('"', numberStart);
+ String number = body.substring(numberStart, numberEnd);
+ OpenSearchMajorVersion version = OpenSearchMajorVersion.parse(number);
+ if (OpenSearchMajorVersion.TWO != version && OpenSearchMajorVersion.THREE != version) {
+ throw new IllegalStateException("Expected TWO or THREE from version.number but got " + version);
+ }
+ System.out.println("[OK] OpenSearchMajorVersion.parse(\"" + number + "\") -> " + version);
+ return number;
+ }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexLifecycleDriver.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexLifecycleDriver.java
new file mode 100644
index 000000000..10e03f336
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexLifecycleDriver.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch.smoke;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphIndex;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.Mapping;
+
+import java.time.Instant;
+import java.util.Date;
+import java.util.Properties;
+
+/**
+ * Validates mixed-index creation and mapping lifecycle against a real OpenSearch server. The server version
+ * is configurable (system property {@code opensearch.docker.version} / {@code opensearch.docker.image}) — the
+ * dedicated {@code opensearch-it} CI job validates OpenSearch 3.7 and 3.8. OpenSearch 2.x is NOT currently part
+ * of the CI matrix; this driver does not depend on any 3.x-only API, but has not been verified against 2.x in CI.
+ */
+public final class OpenSearchMixedIndexLifecycleDriver {
+
+ public static final String GRAPH_INDEX_NAME = "c3janus";
+ public static final String MIXED_INDEX_NAME = "c3mixed";
+ public static final String BACKING_INDEX_NAME = "search";
+ public static final String PHYSICAL_INDEX = GRAPH_INDEX_NAME + "_" + MIXED_INDEX_NAME.toLowerCase();
+
+ private OpenSearchMixedIndexLifecycleDriver() {
+ }
+
+ public static void execute() throws Exception {
+ OpenSearchSmokeSupport.verifyOpenSearchReachable();
+ OpenSearchSmokeSupport.registerOpenSearchBackend();
+ OpenSearchSmokeSupport.deletePhysicalIndex(PHYSICAL_INDEX);
+
+ Properties properties = new Properties();
+ properties.setProperty("storage.backend", "inmemory");
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".backend", "opensearch");
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".hostname", OpenSearchSmokeSupport.getOpenSearchHost());
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".port",
+ String.valueOf(OpenSearchSmokeSupport.getOpenSearchPort()));
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".index-name", GRAPH_INDEX_NAME);
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".opensearch.setup-max-open-scroll-contexts", "false");
+
+ org.apache.commons.configuration2.Configuration config =
+ org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties);
+
+ try (JanusGraph graph = JanusGraphFactory.open(config)) {
+ JanusGraphManagement mgmt = graph.openManagement();
+
+ PropertyKey keywordField = mgmt.makePropertyKey("keywordField").dataType(String.class).make();
+ PropertyKey textField = mgmt.makePropertyKey("textField").dataType(String.class).make();
+ PropertyKey textStringField = mgmt.makePropertyKey("textStringField").dataType(String.class).make();
+ PropertyKey intField = mgmt.makePropertyKey("intField").dataType(Integer.class).make();
+ PropertyKey boolField = mgmt.makePropertyKey("boolField").dataType(Boolean.class).make();
+ PropertyKey dateField = mgmt.makePropertyKey("dateField").dataType(Date.class).make();
+ PropertyKey instantField = mgmt.makePropertyKey("instantField").dataType(Instant.class).make();
+
+ mgmt.buildIndex(MIXED_INDEX_NAME, Vertex.class)
+ .addKey(keywordField, Mapping.STRING.asParameter())
+ .addKey(textField, Mapping.TEXT.asParameter())
+ .addKey(textStringField, Mapping.TEXTSTRING.asParameter())
+ .addKey(intField)
+ .addKey(boolField)
+ .addKey(dateField)
+ .addKey(instantField)
+ .buildMixedIndex(BACKING_INDEX_NAME);
+
+ mgmt.commit();
+
+ mgmt = graph.openManagement();
+ PropertyKey longField = mgmt.makePropertyKey("longField").dataType(Long.class).make();
+ JanusGraphIndex existingMixedIndex = mgmt.getGraphIndex(MIXED_INDEX_NAME);
+ mgmt.addIndexKey(existingMixedIndex, longField);
+ mgmt.commit();
+ }
+
+ verifyPhysicalIndexCreated();
+ }
+
+ public static void main(String[] args) throws Exception {
+ execute();
+
+ System.out.println("[OK] Mixed index committed: " + MIXED_INDEX_NAME);
+ System.out.println("[INFO] Expected physical OpenSearch index: " + PHYSICAL_INDEX);
+ System.out.println("Mixed-index lifecycle driver finished. Inspect OpenSearch with:");
+ System.out.println(" curl -s http://" + OpenSearchSmokeSupport.getOpenSearchHost() + ":"
+ + OpenSearchSmokeSupport.getOpenSearchPort() + "/" + PHYSICAL_INDEX);
+ }
+
+ private static void verifyPhysicalIndexCreated() throws Exception {
+ String mapping = OpenSearchSmokeSupport.httpGet("/" + PHYSICAL_INDEX + "/_mapping");
+ if (!mapping.contains("keywordField") || !mapping.contains("longField")) {
+ throw new IllegalStateException("Physical index mapping missing expected fields: " + mapping);
+ }
+
+ String alias = OpenSearchSmokeSupport.httpGet("/_alias/" + GRAPH_INDEX_NAME);
+ if (!alias.contains(PHYSICAL_INDEX)) {
+ throw new IllegalStateException("Expected alias " + GRAPH_INDEX_NAME + " -> " + PHYSICAL_INDEX
+ + " but got: " + alias);
+ }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexLifecycleIT.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexLifecycleIT.java
new file mode 100644
index 000000000..ef1990d58
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexLifecycleIT.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch.smoke;
+
+import org.apache.atlas.runner.OpenSearchITBase;
+import org.testng.annotations.Test;
+
+/**
+ * Integration test: mixed-index creation and mapping lifecycle via Testcontainers OpenSearch.
+ */
+public class OpenSearchMixedIndexLifecycleIT extends OpenSearchITBase {
+
+ @Test
+ public void mixedIndexLifecycle() throws Exception {
+ OpenSearchMixedIndexLifecycleDriver.execute();
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexMutationQueryDriver.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexMutationQueryDriver.java
new file mode 100644
index 000000000..24882c11e
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexMutationQueryDriver.java
@@ -0,0 +1,499 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch.smoke;
+
+import org.apache.tinkerpop.gremlin.process.traversal.Order;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.Cardinality;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.JanusGraphVertex;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.Mapping;
+import org.janusgraph.diskstorage.opensearch.rest.OpenSearchBulkWireFormatVerifier;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Validates mixed-index document mutations and basic query execution against a real OpenSearch server
+ * through the JanusGraph → OpenSearchIndex path (no Atlas discovery APIs). The server version is configurable
+ * (system property {@code opensearch.docker.version} / {@code opensearch.docker.image}) — the dedicated
+ * {@code opensearch-it} CI job validates OpenSearch 3.7 and 3.8. OpenSearch 2.x is NOT currently part of the CI
+ * matrix; this driver does not depend on any 3.x-only API, but has not been verified against 2.x in CI.
+ *
+ * <pre>
+ * mvn -pl graphdb/janusgraph-opensearch test-compile exec:java \
+ * -Dexec.classpathScope=test \
+ * -Dexec.mainClass=org.janusgraph.diskstorage.opensearch.smoke.OpenSearchMixedIndexMutationQueryDriver \
+ * -Drat.skip=true -Dcheckstyle.skip=true -Dsortpom.skip=true
+ * </pre>
+ */
+public final class OpenSearchMixedIndexMutationQueryDriver {
+
+ public static final String GRAPH_INDEX_NAME = "c4janus";
+ public static final String MIXED_INDEX_NAME = "c4mixed";
+ public static final String BACKING_INDEX_NAME = "search";
+ public static final String PHYSICAL_INDEX = GRAPH_INDEX_NAME + "_" + MIXED_INDEX_NAME.toLowerCase();
+
+ /** Small batch size forces scroll when querying more results than this. */
+ private static final int SCROLL_BATCH_SIZE = 5;
+
+ private static final String SCROLL_GROUP = "c4scroll";
+ private static final int SCROLL_DOC_COUNT = 18;
+
+ private OpenSearchMixedIndexMutationQueryDriver() {
+ }
+
+ public static void execute() throws Exception {
+ OpenSearchSmokeSupport.verifyOpenSearchReachable();
+ OpenSearchSmokeSupport.registerOpenSearchBackend();
+ OpenSearchBulkWireFormatVerifier.verify(
+ OpenSearchSmokeSupport.getOpenSearchHost(), OpenSearchSmokeSupport.getOpenSearchPort());
+ OpenSearchSmokeSupport.deletePhysicalIndex(PHYSICAL_INDEX);
+
+ org.janusgraph.diskstorage.configuration.ReadConfiguration config =
+ OpenSearchSmokeSupport.buildConfiguration(GRAPH_INDEX_NAME, BACKING_INDEX_NAME, SCROLL_BATCH_SIZE);
+
+ try (JanusGraph graph = JanusGraphFactory.open(config)) {
+ Schema schema = createSchema(graph);
+ runInsertTests(graph, schema);
+ runUpdateTests(graph, schema);
+ runDeleteTests(graph, schema);
+ runCardinalityTests(graph, schema);
+ runBasicQueryTests(graph, schema);
+ runPaginationAndScrollTests(graph, schema);
+ runMutationQueryConsistencyTests(graph, schema);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ execute();
+
+ System.out.println();
+ System.out.println("Mixed-index mutation and query driver finished successfully.");
+ System.out.println("Inspect OpenSearch with:");
+ System.out.println(" curl -s http://" + OpenSearchSmokeSupport.getOpenSearchHost() + ":"
+ + OpenSearchSmokeSupport.getOpenSearchPort() + "/" + PHYSICAL_INDEX + "/_count");
+ }
+
+ // -------------------------------------------------------------------------
+ // Schema
+ // -------------------------------------------------------------------------
+
+ private static Schema createSchema(JanusGraph graph) {
+ JanusGraphManagement mgmt = graph.openManagement();
+
+ PropertyKey name = mgmt.makePropertyKey("name").dataType(String.class).make();
+ PropertyKey textField = mgmt.makePropertyKey("textField").dataType(String.class).make();
+ PropertyKey textStringField = mgmt.makePropertyKey("textStringField").dataType(String.class).make();
+ PropertyKey age = mgmt.makePropertyKey("age").dataType(Integer.class).make();
+ PropertyKey longField = mgmt.makePropertyKey("longField").dataType(Long.class).make();
+ PropertyKey active = mgmt.makePropertyKey("active").dataType(Boolean.class).make();
+ PropertyKey birthDate = mgmt.makePropertyKey("birthDate").dataType(Date.class).make();
+ PropertyKey created = mgmt.makePropertyKey("created").dataType(Instant.class).make();
+ PropertyKey batchGroup = mgmt.makePropertyKey("batchGroup").dataType(String.class).make();
+ PropertyKey listTags = mgmt.makePropertyKey("listTags").dataType(String.class)
+ .cardinality(Cardinality.LIST).make();
+ PropertyKey setTags = mgmt.makePropertyKey("setTags").dataType(String.class)
+ .cardinality(Cardinality.SET).make();
+
+ mgmt.buildIndex(MIXED_INDEX_NAME, Vertex.class)
+ .addKey(name, Mapping.STRING.asParameter())
+ .addKey(textField, Mapping.TEXT.asParameter())
+ .addKey(textStringField, Mapping.TEXTSTRING.asParameter())
+ .addKey(age)
+ .addKey(longField)
+ .addKey(active)
+ .addKey(birthDate)
+ .addKey(created)
+ .addKey(batchGroup, Mapping.STRING.asParameter())
+ .addKey(listTags, Mapping.STRING.asParameter())
+ .addKey(setTags, Mapping.STRING.asParameter())
+ .buildMixedIndex(BACKING_INDEX_NAME);
+
+ mgmt.commit();
+ System.out.println("[OK] Schema committed (mixed index: " + MIXED_INDEX_NAME + ")");
+ return new Schema(name, textField, textStringField, age, longField, active, birthDate, created,
+ batchGroup, listTags, setTags);
+ }
+
+ // -------------------------------------------------------------------------
+ // Insert
+ // -------------------------------------------------------------------------
+
+ private static void runInsertTests(JanusGraph graph, Schema s) {
+ JanusGraphVertex alice = graph.addVertex();
+ alice.property(s.name.name(), "Alice");
+ alice.property(s.textField.name(), "hello atlas world");
+ alice.property(s.textStringField.name(), "exactValue");
+ alice.property(s.age.name(), 30);
+ alice.property(s.longField.name(), 9_000_000_000L);
+ alice.property(s.active.name(), true);
+ alice.property(s.birthDate.name(), new Date(631152000000L)); // 1990-01-01
+ alice.property(s.created.name(), Instant.parse("2020-01-01T00:00:00Z"));
+ alice.property(s.batchGroup.name(), "c4insert");
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:Alice", 1);
+ System.out.println("[OK] Single document insert indexed (all field types)");
+
+ List<JanusGraphVertex> bulk = new ArrayList<>();
+ for (int i = 0; i < 5; i++) {
+ JanusGraphVertex v = graph.addVertex();
+ v.property(s.name.name(), "Bulk" + i);
+ v.property(s.age.name(), 20 + i);
+ v.property(s.batchGroup.name(), "c4bulk");
+ bulk.add(v);
+ }
+ graph.tx().commit();
+ assertVertexCount(graph, "v.batchGroup:c4bulk", 5);
+ System.out.println("[OK] Bulk insert indexed 5 documents");
+ }
+
+ // -------------------------------------------------------------------------
+ // Update
+ // -------------------------------------------------------------------------
+
+ private static void runUpdateTests(JanusGraph graph, Schema s) {
+ JanusGraphVertex bob = graph.addVertex();
+ bob.property(s.name.name(), "Bob");
+ bob.property(s.age.name(), 25);
+ bob.property(s.batchGroup.name(), "c4update");
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:Bob AND v.age:25", 1);
+
+ bob.property(s.age.name(), 35);
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:Bob AND v.age:35", 1);
+ assertVertexCount(graph, "v.name:Bob AND v.age:25", 0);
+ System.out.println("[OK] Property update visible via indexQuery (retry_on_conflict wired at bulk layer)");
+ }
+
+ // -------------------------------------------------------------------------
+ // Delete
+ // -------------------------------------------------------------------------
+
+ private static void runDeleteTests(JanusGraph graph, Schema s) {
+ JanusGraphVertex doomed = graph.addVertex();
+ doomed.property(s.name.name(), "ToDelete");
+ doomed.property(s.batchGroup.name(), "c4delete");
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:ToDelete", 1);
+
+ doomed.remove();
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:ToDelete", 0);
+ System.out.println("[OK] Single delete removes document from index");
+
+ List<JanusGraphVertex> bulkDelete = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ JanusGraphVertex v = graph.addVertex();
+ v.property(s.name.name(), "BulkDel" + i);
+ v.property(s.batchGroup.name(), "c4bulkdelete");
+ bulkDelete.add(v);
+ }
+ graph.tx().commit();
+ assertVertexCount(graph, "v.batchGroup:c4bulkdelete", 3);
+
+ for (JanusGraphVertex v : bulkDelete) {
+ v.remove();
+ }
+ graph.tx().commit();
+ assertVertexCount(graph, "v.batchGroup:c4bulkdelete", 0);
+ System.out.println("[OK] Bulk delete removes multiple documents");
+ }
+
+ // -------------------------------------------------------------------------
+ // Cardinality / stored-script paths
+ // -------------------------------------------------------------------------
+
+ private static void runCardinalityTests(JanusGraph graph, Schema s) {
+ JanusGraphVertex single = graph.addVertex();
+ single.property(s.name.name(), "SingleCard");
+ single.property(s.batchGroup.name(), "c4card");
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:SingleCard", 1);
+ System.out.println("[OK] SINGLE cardinality indexed");
+
+ JanusGraphVertex listVertex = graph.addVertex();
+ listVertex.property(s.name.name(), "ListCard");
+ listVertex.property(s.batchGroup.name(), "c4card");
+ listVertex.property(s.listTags.name(), "alpha");
+ listVertex.property(s.listTags.name(), "beta");
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:ListCard AND v.listTags:alpha", 1);
+ listVertex.property(s.listTags.name(), "gamma");
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:ListCard AND v.listTags:gamma", 1);
+ System.out.println("[OK] LIST cardinality insert and stored-script add");
+
+ JanusGraphVertex setVertex = graph.addVertex();
+ setVertex.property(s.name.name(), "SetCard");
+ setVertex.property(s.batchGroup.name(), "c4card");
+ setVertex.property(s.setTags.name(), "red");
+ setVertex.property(s.setTags.name(), "blue");
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:SetCard AND v.setTags:blue", 1);
+ setVertex.property(s.setTags.name(), "green");
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:SetCard AND v.setTags:green", 1);
+ System.out.println("[OK] SET cardinality insert and stored-script add");
+ }
+
+ // -------------------------------------------------------------------------
+ // Basic queries (OpenSearchIndex.query via graph.indexQuery)
+ // -------------------------------------------------------------------------
+
+ private static void runBasicQueryTests(JanusGraph graph, Schema s) {
+ seedQueryFixtures(graph, s);
+ final String qfix = "v.batchGroup:queryfix";
+
+ assertVertexCount(graph, "v.name:QueryAlice", 1);
+ System.out.println("[OK] Equality query");
+
+ assertVertexCount(graph, qfix + " AND v.textField:(atlas search)", 1);
+ System.out.println("[OK] Text query");
+
+ assertVertexCount(graph, qfix + " AND v.age:30", 1);
+ assertVertexCount(graph, qfix + " AND v.age:>20", 2);
+ assertVertexCount(graph, qfix + " AND v.age:<40", 2);
+ assertVertexCount(graph, qfix + " AND v.age:>=30", 1);
+ assertVertexCount(graph, qfix + " AND v.age:<=30", 2);
+ System.out.println("[OK] Numeric range queries (>, <, >=, <=, ==)");
+
+ assertVertexCount(graph, qfix + " AND -v.age:30", 1);
+ System.out.println("[OK] Not-equal query");
+
+ assertVertexCount(graph, qfix + " AND _exists_:v.longField", 1);
+ System.out.println("[OK] Exists query");
+
+ assertVertexCount(graph, qfix + " AND v.age:[25 TO 35]", 2);
+ assertVertexCount(graph, qfix + " AND v.active:true", 1);
+ assertVertexCount(graph, "v.name:QueryAlice OR v.name:QueryBob", 2);
+ System.out.println("[OK] Compound AND/OR queries");
+
+ assertVertexCount(graph, qfix + " AND v.textStringField__STRING:dualValue", 1);
+ System.out.println("[OK] TEXTSTRING dual-mapping query");
+
+ List<String> ordered = graph.indexQuery(MIXED_INDEX_NAME, "v.batchGroup:queryfix")
+ .orderBy(s.age.name(), Order.asc)
+ .limit(10)
+ .vertexStream()
+ .map(r -> (String) r.getElement().value(s.name.name()))
+ .collect(Collectors.toList());
+ if (!ordered.get(0).equals("QueryBob") || !ordered.get(1).equals("QueryAlice")) {
+ throw new IllegalStateException("Unexpected order: " + ordered);
+ }
+ System.out.println("[OK] Ordering");
+
+ long limited = graph.indexQuery(MIXED_INDEX_NAME, "v.batchGroup:queryfix")
+ .limit(1)
+ .vertexStream()
+ .count();
+ if (limited != 1) {
+ throw new IllegalStateException("Expected limit 1 but got " + limited);
+ }
+ System.out.println("[OK] Limit");
+ }
+
+ private static void seedQueryFixtures(JanusGraph graph, Schema s) {
+ JanusGraphVertex alice = graph.addVertex();
+ alice.property(s.name.name(), "QueryAlice");
+ alice.property(s.textField.name(), "atlas search engine");
+ alice.property(s.textStringField.name(), "dualValue");
+ alice.property(s.age.name(), 30);
+ alice.property(s.longField.name(), 42L);
+ alice.property(s.active.name(), true);
+ alice.property(s.batchGroup.name(), "queryfix");
+
+ JanusGraphVertex bob = graph.addVertex();
+ bob.property(s.name.name(), "QueryBob");
+ bob.property(s.textField.name(), "other content");
+ bob.property(s.age.name(), 25);
+ bob.property(s.active.name(), false);
+ bob.property(s.batchGroup.name(), "queryfix");
+
+ graph.tx().commit();
+ }
+
+ // -------------------------------------------------------------------------
+ // Pagination / scroll
+ // -------------------------------------------------------------------------
+
+ private static void runPaginationAndScrollTests(JanusGraph graph, Schema s) throws Exception {
+ for (int i = 0; i < SCROLL_DOC_COUNT; i++) {
+ JanusGraphVertex v = graph.addVertex();
+ v.property(s.name.name(), SCROLL_GROUP + i);
+ v.property(s.age.name(), i);
+ v.property(s.batchGroup.name(), SCROLL_GROUP);
+ }
+ graph.tx().commit();
+
+ long total = graph.indexQuery(MIXED_INDEX_NAME, "v.batchGroup:" + SCROLL_GROUP).vertexTotals();
+ if (total != SCROLL_DOC_COUNT) {
+ throw new IllegalStateException("Expected " + SCROLL_DOC_COUNT + " scroll docs but vertexTotals=" + total);
+ }
+
+ List<String> allViaScroll = graph.indexQuery(MIXED_INDEX_NAME, "v.batchGroup:" + SCROLL_GROUP)
+ .limit(SCROLL_DOC_COUNT)
+ .vertexStream()
+ .map(r -> (String) r.getElement().value(s.name.name()))
+ .collect(Collectors.toList());
+ if (allViaScroll.size() != SCROLL_DOC_COUNT) {
+ throw new IllegalStateException("Scroll query returned " + allViaScroll.size() + " docs, expected "
+ + SCROLL_DOC_COUNT);
+ }
+ Set<String> unique = new HashSet<>(allViaScroll);
+ if (unique.size() != SCROLL_DOC_COUNT) {
+ throw new IllegalStateException("Scroll query returned duplicate document IDs");
+ }
+ System.out.println("[OK] Scroll retrieved all " + SCROLL_DOC_COUNT + " documents without duplicates");
+
+ // offset(5).limit(5) against SCROLL_BATCH_SIZE=5 forces a scroll (window size 5) that fetches exactly
+ // offset+limit=10 of the 18 total docs (page 1 skipped, page 2 taken) — i.e. it is NOT naturally exhausted
+ // (8 docs remain unfetched server-side). Must be explicitly closed, same as the early-terminated query
+ // below, or its OpenSearch scroll context is left open until the scroll-keep-alive TTL expires.
+ List<String> page = collectAndClose(
+ graph.indexQuery(MIXED_INDEX_NAME, "v.batchGroup:" + SCROLL_GROUP)
+ .offset(5)
+ .limit(5)
+ .vertexStream()
+ .map(r -> (String) r.getElement().value(s.name.name())));
+ if (page.size() != 5) {
+ throw new IllegalStateException("Offset/limit page expected 5 results but got " + page.size());
+ }
+ System.out.println("[OK] Offset/limit pagination");
+
+ // Early termination: request fewer documents (10) than exist (18) but enough to force scroll usage
+ // (limit >= SCROLL_BATCH_SIZE), then explicitly close the stream before the scroll is naturally exhausted.
+ // This is the exact scenario that used to leak an OpenSearch scroll context on the RawQuery path
+ // (graph.indexQuery(indexName, queryString) — used throughout this driver) before the onClose(scroll::close)
+ // wiring fix; it is intentionally NOT collected to completion so the fix's hard-assertion below is meaningful.
+ final int earlyLimit = SCROLL_BATCH_SIZE * 2;
+ List<String> earlyTerminated = collectAndClose(
+ graph.indexQuery(MIXED_INDEX_NAME, "v.batchGroup:" + SCROLL_GROUP)
+ .limit(earlyLimit)
+ .vertexStream()
+ .map(r -> (String) r.getElement().value(s.name.name())));
+ if (earlyTerminated.size() != earlyLimit) {
+ throw new IllegalStateException("Early-terminated scroll query expected " + earlyLimit
+ + " results but got " + earlyTerminated.size());
+ }
+ System.out.println("[OK] Early-terminated scroll query (limit " + earlyLimit + " of " + SCROLL_DOC_COUNT
+ + ") returned the expected subset");
+
+ String scrollStats = OpenSearchSmokeSupport.httpGet("/_nodes/stats/indices/search?filter_path=nodes.*.indices.search.open_contexts");
+ if (scrollStats.contains("\"open_contexts\":") && !scrollStats.contains("\"open_contexts\":0")) {
+ throw new IllegalStateException("OpenSearch still reports open scroll contexts after all scroll "
+ + "queries (including the early-terminated one) were closed — scroll context leak: "
+ + scrollStats.trim());
+ }
+ System.out.println("[OK] Scroll contexts cleaned up (open_contexts=0 or unavailable)");
+ }
+
+ /**
+ * Collects a {@code Stream} and always closes it, releasing any {@code onClose} handler attached to its
+ * source (e.g. an OpenSearch scroll context) — including when the stream is not fully exhausted (a limit()
+ * short-circuits collection). Generic so it works regardless of the concrete JanusGraph result-stream type.
+ */
+ private static <T> List<T> collectAndClose(Stream<T> stream) {
+ try (Stream<T> closeable = stream) {
+ return closeable.collect(Collectors.toList());
+ }
+ }
+
+ // -------------------------------------------------------------------------
+ // Mutation / query consistency
+ // -------------------------------------------------------------------------
+
+ private static void runMutationQueryConsistencyTests(JanusGraph graph, Schema s) {
+ JanusGraphVertex keep = graph.addVertex();
+ keep.property(s.name.name(), "KeepMe");
+ keep.property(s.age.name(), 10);
+ keep.property(s.batchGroup.name(), "c4consistency");
+
+ JanusGraphVertex target = graph.addVertex();
+ target.property(s.name.name(), "Target");
+ target.property(s.age.name(), 50);
+ target.property(s.batchGroup.name(), "c4consistency");
+ graph.tx().commit();
+
+ assertVertexCount(graph, "v.name:Target", 1);
+ assertVertexCount(graph, "v.batchGroup:c4consistency", 2);
+
+ target.property(s.age.name(), 55);
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:Target AND v.age:55", 1);
+ assertVertexCount(graph, "v.name:Target AND v.age:50", 0);
+ assertVertexCount(graph, "v.name:KeepMe", 1);
+
+ target.remove();
+ graph.tx().commit();
+ assertVertexCount(graph, "v.name:Target", 0);
+ assertVertexCount(graph, "v.name:KeepMe", 1);
+ System.out.println("[OK] Insert → query → update → query → delete → query consistency");
+ }
+
+ // -------------------------------------------------------------------------
+ // Helpers
+ // -------------------------------------------------------------------------
+
+ private static void assertVertexCount(JanusGraph graph, String query, long expected) {
+ Long actual = graph.indexQuery(MIXED_INDEX_NAME, query).vertexTotals();
+ if (actual == null || actual != expected) {
+ throw new IllegalStateException("Query [" + query + "] expected " + expected + " vertices but got " + actual);
+ }
+ }
+
+ private static final class Schema {
+ final PropertyKey name;
+ final PropertyKey textField;
+ final PropertyKey textStringField;
+ final PropertyKey age;
+ final PropertyKey longField;
+ final PropertyKey active;
+ final PropertyKey birthDate;
+ final PropertyKey created;
+ final PropertyKey batchGroup;
+ final PropertyKey listTags;
+ final PropertyKey setTags;
+
+ Schema(PropertyKey name, PropertyKey textField, PropertyKey textStringField, PropertyKey age,
+ PropertyKey longField, PropertyKey active, PropertyKey birthDate, PropertyKey created,
+ PropertyKey batchGroup, PropertyKey listTags, PropertyKey setTags) {
+ this.name = name;
+ this.textField = textField;
+ this.textStringField = textStringField;
+ this.age = age;
+ this.longField = longField;
+ this.active = active;
+ this.birthDate = birthDate;
+ this.created = created;
+ this.batchGroup = batchGroup;
+ this.listTags = listTags;
+ this.setTags = setTags;
+ }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexMutationQueryIT.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexMutationQueryIT.java
new file mode 100644
index 000000000..bb8f6f4b9
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchMixedIndexMutationQueryIT.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch.smoke;
+
+import org.apache.atlas.runner.OpenSearchITBase;
+import org.testng.annotations.Test;
+
+/**
+ * Integration test: mixed-index mutations and JanusGraph queries via Testcontainers OpenSearch.
+ */
+public class OpenSearchMixedIndexMutationQueryIT extends OpenSearchITBase {
+
+ @Test
+ public void mixedIndexMutationsAndQueries() throws Exception {
+ OpenSearchMixedIndexMutationQueryDriver.execute();
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchSmokeSupport.java b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchSmokeSupport.java
new file mode 100644
index 000000000..28662d90f
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/test/java/org/janusgraph/diskstorage/opensearch/smoke/OpenSearchSmokeSupport.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch.smoke;
+
+import com.google.common.collect.ImmutableMap;
+import org.janusgraph.diskstorage.StandardIndexProvider;
+import org.janusgraph.diskstorage.configuration.ReadConfiguration;
+import org.janusgraph.diskstorage.configuration.backend.CommonsConfiguration;
+import sun.misc.Unsafe;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+final class OpenSearchSmokeSupport {
+
+ private OpenSearchSmokeSupport() {
+ }
+
+ static String getOpenSearchHost() {
+ return System.getProperty("opensearch.host", "localhost");
+ }
+
+ static int getOpenSearchPort() {
+ return Integer.getInteger("opensearch.port", 9200);
+ }
+
+ static void verifyOpenSearchReachable() throws IOException {
+ URL url = new URL("http://" + getOpenSearchHost() + ":" + getOpenSearchPort() + "/");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(5000);
+ connection.setRequestMethod("GET");
+ if (connection.getResponseCode() != 200) {
+ throw new IOException("OpenSearch GET / returned HTTP " + connection.getResponseCode());
+ }
+ System.out.println("[OK] OpenSearch reachable at " + getOpenSearchHost() + ":" + getOpenSearchPort());
+ }
+
+ static void deletePhysicalIndex(String physicalIndex) throws IOException {
+ URL url = new URL("http://" + getOpenSearchHost() + ":" + getOpenSearchPort() + "/" + physicalIndex);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(5000);
+ connection.setRequestMethod("DELETE");
+ connection.getResponseCode();
+ System.out.println("[INFO] Cleared physical index (if present): " + physicalIndex);
+ }
+
+ static String httpGet(String path) throws IOException {
+ URL url = new URL("http://" + getOpenSearchHost() + ":" + getOpenSearchPort() + path);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(10000);
+ connection.setRequestMethod("GET");
+ if (connection.getResponseCode() != 200) {
+ throw new IOException("GET " + path + " returned HTTP " + connection.getResponseCode());
+ }
+ try (java.io.InputStream in = connection.getInputStream()) {
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ static void registerOpenSearchBackend() throws Exception {
+ Class.forName(StandardIndexProvider.class.getName(), true, StandardIndexProvider.class.getClassLoader());
+ Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES");
+ Unsafe unsafe = obtainUnsafe();
+ Object base = unsafe.staticFieldBase(field);
+ long offset = unsafe.staticFieldOffset(field);
+ @SuppressWarnings("unchecked")
+ Map<String, String> current = (Map<String, String>) unsafe.getObject(base, offset);
+ if (current == null) {
+ throw new IllegalStateException("StandardIndexProvider.ALL_MANAGER_CLASSES is null after class init");
+ }
+ Map<String, String> updated = new HashMap<>(current);
+ updated.put("opensearch", "org.janusgraph.diskstorage.opensearch.OpenSearchIndex");
+ unsafe.putObject(base, offset, ImmutableMap.copyOf(updated));
+ @SuppressWarnings("unchecked")
+ Map<String, String> verified = (Map<String, String>) unsafe.getObject(base, offset);
+ if (!"org.janusgraph.diskstorage.opensearch.OpenSearchIndex".equals(verified.get("opensearch"))) {
+ System.out.println("[WARN] opensearch shorthand registration did not stick; using fully-qualified backend class in config");
+ }
+ }
+
+ static ReadConfiguration buildConfiguration(
+ String graphIndexName, String backingIndexName, int maxResultSetSize) {
+ Properties properties = new Properties();
+ properties.setProperty("storage.backend", "inmemory");
+ properties.setProperty("index." + backingIndexName + ".backend",
+ "org.janusgraph.diskstorage.opensearch.OpenSearchIndex");
+ properties.setProperty("index." + backingIndexName + ".hostname", getOpenSearchHost());
+ properties.setProperty("index." + backingIndexName + ".port", String.valueOf(getOpenSearchPort()));
+ properties.setProperty("index." + backingIndexName + ".index-name", graphIndexName);
+ properties.setProperty("index." + backingIndexName + ".opensearch.setup-max-open-scroll-contexts", "false");
+ properties.setProperty("index." + backingIndexName + ".opensearch.bulk-refresh", "wait_for");
+ properties.setProperty("index." + backingIndexName + ".opensearch.retry_on_conflict", "3");
+ properties.setProperty("index." + backingIndexName + ".max-result-set-size",
+ String.valueOf(maxResultSetSize));
+
+ org.apache.commons.configuration2.Configuration commons =
+ org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties);
+ return new CommonsConfiguration(commons);
+ }
+
+ private static Unsafe obtainUnsafe() throws Exception {
+ Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe");
+ unsafeField.setAccessible(true);
+ return (Unsafe) unsafeField.get(null);
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchAggregationsIT.java b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchAggregationsIT.java
new file mode 100644
index 000000000..504166f26
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchAggregationsIT.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery.smoke;
+
+import org.apache.atlas.runner.OpenSearchITBase;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertTrue;
+
+/**
+ * Integration test: Atlas aggregations via Testcontainers OpenSearch.
+ */
+public class OpenSearchAggregationsIT extends OpenSearchITBase {
+
+ @Test
+ public void aggregationsValidation() throws Exception {
+ assertTrue(OpenSearchAggregationsValidationDriver.execute(), "Aggregations validation failed");
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchAggregationsValidationDriver.java b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchAggregationsValidationDriver.java
new file mode 100644
index 000000000..bfb12d89b
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchAggregationsValidationDriver.java
@@ -0,0 +1,599 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery.smoke;
+
+import org.apache.atlas.discovery.FreeTextSearchProcessor;
+import org.apache.atlas.discovery.SearchAggregatorImpl;
+import org.apache.atlas.discovery.SearchContext;
+import org.apache.atlas.model.discovery.QuickSearchParameters;
+import org.apache.atlas.discovery.EntityDiscoveryService;
+import org.apache.atlas.model.discovery.AtlasAggregationEntry;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.Operator;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AggregationContext;
+import org.apache.atlas.repository.graphdb.QuickSearchContext;
+import org.apache.atlas.repository.graphdb.QuickSearchResult;
+import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
+import org.apache.atlas.repository.graphdb.AtlasCardinality;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.lang3.StringUtils;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+/**
+ * Validates Atlas aggregations through {@link AtlasGraphIndexClient#getAggregatedMetrics}
+ * and {@link SearchAggregatorImpl} against a real OpenSearch server. The server version is configurable (system
+ * property {@code opensearch.docker.version} / {@code opensearch.docker.image}) — the dedicated
+ * {@code opensearch-it} CI job validates OpenSearch 3.7 and 3.8. OpenSearch 2.x is NOT currently part of the CI
+ * matrix; this driver has not been verified against 2.x in CI.
+ *
+ * <pre>
+ * cd repository && mvn test-compile exec:java \
+ * -Dexec.classpathScope=test \
+ * -Dexec.mainClass=org.apache.atlas.discovery.smoke.OpenSearchAggregationsValidationDriver \
+ * -Drat.skip=true -Dcheckstyle.skip=true -Dsortpom.skip=true -DskipCheck=true
+ * </pre>
+ */
+public final class OpenSearchAggregationsValidationDriver {
+
+ private static final String TYPE_DATASET = "c54_dataset";
+ private static final String TYPE_TABLE = "c54_table";
+ private static final String TYPE_ASSET = "c54_asset";
+ private static final String SERVICE_A = "c54_service_a";
+ private static final String SERVICE_B = "c54_service_b";
+
+ private static final Map<String, String> RESULTS = new LinkedHashMap<>();
+
+ private OpenSearchAggregationsValidationDriver() {
+ }
+
+ public static boolean execute() throws Exception {
+ RESULTS.clear();
+
+ OpenSearchQuickSearchSmokeSupport.bootstrapApplicationProperties();
+ OpenSearchQuickSearchSmokeSupport.verifyOpenSearchReachable();
+ OpenSearchQuickSearchSmokeSupport.registerAtlasOpenSearchIndex();
+ OpenSearchQuickSearchSmokeSupport.deletePhysicalIndexIfPresent();
+
+ JanusGraph janusGraph = JanusGraphFactory.open(OpenSearchQuickSearchSmokeSupport.buildJanusGraphConfiguration());
+ AtlasTypeRegistry typeRegistry = buildTypeRegistry(new IndexFieldNames());
+ IndexFieldNames indexFields = createSchema(janusGraph, typeRegistry);
+ wireTypeRegistry(typeRegistry, indexFields);
+
+ AtlasGraph graph = new AtlasJanusGraph(janusGraph);
+ insertTestEntities(graph, typeRegistry);
+ graph.commit();
+
+ Thread.sleep(1500);
+
+ AtlasGraphIndexClient indexClient = graph.getGraphIndexClient();
+
+ runValidations(indexClient, graph, typeRegistry, indexFields);
+
+ graph.shutdown();
+
+ return RESULTS.values().stream().allMatch("PASS"::equals);
+ }
+
+ public static void main(String[] args) throws Exception {
+ System.out.println("Atlas Aggregations Validation");
+ boolean allPassed = execute();
+ for (Map.Entry<String, String> entry : RESULTS.entrySet()) {
+ System.out.printf("%-32s %s%n", entry.getKey(), entry.getValue());
+ }
+ System.out.println(allPassed ? "RESULT: PASS" : "RESULT: FAIL");
+ if (!allPassed) {
+ System.exit(1);
+ }
+ }
+
+ private static void runValidations(AtlasGraphIndexClient indexClient, AtlasGraph graph,
+ AtlasTypeRegistry typeRegistry, IndexFieldNames indexFields) throws Exception {
+ Set<AtlasEntityType> allTypes = entityTypes(typeRegistry, TYPE_DATASET, TYPE_TABLE);
+ Map<String, String> indexFieldNameCache = buildIndexFieldNameCache(typeRegistry, indexFields);
+
+ record("Single aggregation field", () -> {
+ AggregationContext ctx = aggContext("", null, allTypes, indexFieldNameCache,
+ setOf(Constants.ENTITY_TYPE_PROPERTY_KEY), Collections.emptySet(), true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ assertTrue(metrics.containsKey(Constants.ENTITY_TYPE_PROPERTY_KEY),
+ "expected typeName aggregation");
+ assertTrue(!metrics.get(Constants.ENTITY_TYPE_PROPERTY_KEY).isEmpty(),
+ "expected non-empty typeName buckets");
+ });
+
+ record("Multiple aggregation fields", () -> {
+ Set<String> fields = linkedSet(Constants.ENTITY_TYPE_PROPERTY_KEY, Constants.STATE_PROPERTY_KEY);
+ AggregationContext ctx = aggContext("", null, allTypes, indexFieldNameCache,
+ fields, Collections.emptySet(), true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ assertTrue(metrics.containsKey(Constants.ENTITY_TYPE_PROPERTY_KEY), "missing typeName");
+ assertTrue(metrics.containsKey(Constants.STATE_PROPERTY_KEY), "missing __state");
+ assertTrue(metrics.get(Constants.STATE_PROPERTY_KEY).stream()
+ .anyMatch(e -> AtlasEntity.Status.ACTIVE.name().equals(e.getName())),
+ "expected ACTIVE state bucket");
+ });
+
+ record("Entity/type filtering", () -> {
+ Set<AtlasEntityType> datasetOnly = entityTypes(typeRegistry, TYPE_DATASET);
+ AggregationContext ctx = aggContext("", null, datasetOnly, indexFieldNameCache,
+ setOf(Constants.ENTITY_TYPE_PROPERTY_KEY), Collections.emptySet(), true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ List<String> typeNames = metrics.get(Constants.ENTITY_TYPE_PROPERTY_KEY).stream()
+ .map(AtlasAggregationEntry::getName).collect(Collectors.toList());
+ assertTrue(typeNames.contains(TYPE_DATASET), "expected dataset type");
+ assertTrue(!typeNames.contains(TYPE_TABLE), "table type should be filtered out, got " + typeNames);
+ });
+
+ record("Attribute filtering", () -> {
+ FilterCriteria filter = leafFilter("owner", Operator.EQ, "team-alpha");
+ Set<AtlasAttribute> attrs = ownerAttribute(typeRegistry);
+ AggregationContext ctx = aggContext("", filter, allTypes, indexFieldNameCache,
+ Collections.emptySet(), attrs, true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ String ownerKey = ownerAttribute(typeRegistry).iterator().next().getQualifiedName();
+ List<AtlasAggregationEntry> ownerBuckets = metrics.get(ownerKey);
+ assertTrue(ownerBuckets != null && ownerBuckets.size() == 1,
+ "expected single owner bucket after filter, got " + ownerBuckets);
+ assertTrue("team-alpha".equals(ownerBuckets.get(0).getName()), "expected team-alpha bucket");
+ assertEquals(ownerBuckets.get(0).getCount(), 4L, "team-alpha count");
+ });
+
+ record("Aggregation matches quick-search filter scope", () -> {
+ FilterCriteria filter = leafFilter("owner", Operator.EQ, "team-alpha");
+ QuickSearchParameters quickParams = quickSearchParams("atlas", TYPE_DATASET, filter);
+ SearchParameters searchParameters = EntityDiscoveryService.createSearchParameters(quickParams);
+ SearchContext searchContext = new SearchContext(searchParameters, typeRegistry, graph, Collections.emptySet());
+
+ QuickSearchResult quickResult = runQuickSearchIndexQuery(searchContext, indexFields);
+ SearchAggregatorImpl aggregator = new SearchAggregatorImpl(searchContext);
+ Map<String, List<AtlasAggregationEntry>> metrics = aggregator.getAggregatedMetrics(
+ setOf(Constants.ENTITY_TYPE_PROPERTY_KEY), Collections.emptySet());
+
+ long typeNameBucketTotal = metrics.get(Constants.ENTITY_TYPE_PROPERTY_KEY).stream()
+ .mapToLong(AtlasAggregationEntry::getCount).sum();
+ assertEquals(typeNameBucketTotal, quickResult.getTotalCount(),
+ "aggregation buckets should match quick-search total under same filters");
+ });
+
+ record("Wildcard query aggregation parity", () -> {
+ QuickSearchParameters quickParams = quickSearchParams("custo*", TYPE_DATASET, null);
+ SearchParameters searchParameters = EntityDiscoveryService.createSearchParameters(quickParams);
+ SearchContext searchContext = new SearchContext(searchParameters, typeRegistry, graph, Collections.emptySet());
+
+ QuickSearchResult quickResult = runQuickSearchIndexQuery(searchContext, indexFields);
+ assertTrue(quickResult.getTotalCount() >= 1, "wildcard quick-search should match customer entity");
+
+ SearchAggregatorImpl aggregator = new SearchAggregatorImpl(searchContext);
+ Map<String, List<AtlasAggregationEntry>> metrics = aggregator.getAggregatedMetrics(
+ setOf(Constants.ENTITY_TYPE_PROPERTY_KEY), Collections.emptySet());
+
+ long typeNameBucketTotal = metrics.get(Constants.ENTITY_TYPE_PROPERTY_KEY).stream()
+ .mapToLong(AtlasAggregationEntry::getCount).sum();
+ assertEquals(typeNameBucketTotal, quickResult.getTotalCount(),
+ "wildcard aggregation total should match quick-search total");
+ });
+
+ record("Empty result", () -> {
+ AggregationContext ctx = aggContext("zzz-no-match-xyz", null, allTypes, indexFieldNameCache,
+ setOf(Constants.ENTITY_TYPE_PROPERTY_KEY), Collections.emptySet(), true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ List<AtlasAggregationEntry> buckets = metrics.get(Constants.ENTITY_TYPE_PROPERTY_KEY);
+ assertTrue(buckets == null || buckets.isEmpty(),
+ "expected empty typeName buckets for non-matching query, got " + buckets);
+ });
+
+ record("Bucket counts", () -> {
+ AggregationContext ctx = aggContext("", null, allTypes, indexFieldNameCache,
+ Collections.emptySet(), ownerAttribute(typeRegistry), true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ String ownerKey = ownerAttribute(typeRegistry).iterator().next().getQualifiedName();
+ long teamAlphaCount = metrics.get(ownerKey).stream()
+ .filter(e -> "team-alpha".equals(e.getName()))
+ .mapToLong(AtlasAggregationEntry::getCount)
+ .findFirst().orElse(-1);
+ assertEquals(teamAlphaCount, 4L, "team-alpha document count");
+ });
+
+ record("Multiple entity types", () -> {
+ AggregationContext ctx = aggContext("", null, allTypes, indexFieldNameCache,
+ setOf(Constants.ENTITY_TYPE_PROPERTY_KEY), Collections.emptySet(), true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ Set<String> types = metrics.get(Constants.ENTITY_TYPE_PROPERTY_KEY).stream()
+ .map(AtlasAggregationEntry::getName).collect(Collectors.toSet());
+ assertTrue(types.contains(TYPE_DATASET) && types.contains(TYPE_TABLE),
+ "expected both entity types in aggregation, got " + types);
+ });
+
+ record("Pagination independence", () -> {
+ // Aggregations always use size=0; verify counts are full-set not truncated.
+ AggregationContext ctx = aggContext("", null, allTypes, indexFieldNameCache,
+ setOf(Constants.ENTITY_TYPE_PROPERTY_KEY), Collections.emptySet(), true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ long totalFromAggs = metrics.get(Constants.ENTITY_TYPE_PROPERTY_KEY).stream()
+ .mapToLong(AtlasAggregationEntry::getCount).sum();
+ assertEquals(totalFromAggs, 13L, "aggregation total doc count (13 entities)");
+ });
+
+ record("Search query + aggregation", () -> {
+ AggregationContext ctx = aggContext("Atlas", null, allTypes, indexFieldNameCache,
+ Collections.emptySet(), ownerAttribute(typeRegistry), true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ String ownerKey = ownerAttribute(typeRegistry).iterator().next().getQualifiedName();
+ long totalOwners = metrics.get(ownerKey).stream().mapToLong(AtlasAggregationEntry::getCount).sum();
+ assertTrue(totalOwners >= 5 && totalOwners <= 10,
+ "Atlas query should match subset of docs, total owner agg=" + totalOwners);
+ });
+
+ record("Atlas response shape", () -> {
+ AggregationContext ctx = aggContext("", null, allTypes, indexFieldNameCache,
+ setOf(Constants.STATE_PROPERTY_KEY), Collections.emptySet(), true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ AtlasAggregationEntry entry = metrics.get(Constants.STATE_PROPERTY_KEY).get(0);
+ assertTrue(entry.getName() != null && entry.getCount() > 0,
+ "AtlasAggregationEntry should expose name and positive count");
+ });
+
+ record("ServiceType post-processing", () -> {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(TYPE_DATASET + "," + TYPE_TABLE);
+ params.setExcludeDeletedEntities(true);
+ params.setIncludeSubTypes(true);
+
+ SearchContext searchContext = new SearchContext(params, typeRegistry, graph, Collections.emptySet());
+ SearchAggregatorImpl aggregator = new SearchAggregatorImpl(searchContext);
+
+ // Match EntityDiscoveryService: typeName and __state both populate indexFieldNameCache.
+ Set<String> fields = linkedSet(Constants.ENTITY_TYPE_PROPERTY_KEY, Constants.STATE_PROPERTY_KEY);
+ Map<String, List<AtlasAggregationEntry>> metrics =
+ aggregator.getAggregatedMetrics(fields, Collections.emptySet());
+
+ assertTrue(metrics.containsKey("ServiceType"), "ServiceType metric should be added");
+ assertTrue(metrics.get("ServiceType").stream().anyMatch(e -> SERVICE_A.equals(e.getName()) && e.getCount() > 0),
+ "service A should have non-zero count");
+ assertTrue(metrics.get("ServiceType").stream().noneMatch(e -> e.getCount() == 0),
+ "zero-count service entries should be removed");
+ });
+
+ record("__state ACTIVE/DELETED counts", () -> {
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Deleted One", "team-alpha",
+ AtlasEntity.Status.DELETED);
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Deleted Two", "team-beta",
+ AtlasEntity.Status.DELETED);
+ graph.commit();
+ Thread.sleep(1500);
+
+ AggregationContext ctx = aggContext("", null, allTypes, indexFieldNameCache,
+ setOf(Constants.STATE_PROPERTY_KEY), Collections.emptySet(), false, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = indexClient.getAggregatedMetrics(ctx);
+
+ Map<String, Long> stateCounts = metrics.get(Constants.STATE_PROPERTY_KEY).stream()
+ .collect(Collectors.toMap(AtlasAggregationEntry::getName, AtlasAggregationEntry::getCount));
+ assertEquals(stateCounts.get(AtlasEntity.Status.ACTIVE.name()).longValue(), 13L,
+ "ACTIVE entity count");
+ assertEquals(stateCounts.get(AtlasEntity.Status.DELETED.name()).longValue(), 2L,
+ "DELETED entity count");
+ });
+
+ record("Existing index", () -> {
+ AggregationContext ctx = aggContext("", null, allTypes, indexFieldNameCache,
+ setOf(Constants.ENTITY_TYPE_PROPERTY_KEY), Collections.emptySet(), true, true);
+ Map<String, List<AtlasAggregationEntry>> first = indexClient.getAggregatedMetrics(ctx);
+ Map<String, List<AtlasAggregationEntry>> second = indexClient.getAggregatedMetrics(ctx);
+
+ assertEquals(first.size(), second.size(), "repeat agg on existing index");
+ assertTrue(!second.get(Constants.ENTITY_TYPE_PROPERTY_KEY).isEmpty(),
+ "existing index should still return buckets");
+ });
+ }
+
+ // -------------------------------------------------------------------------
+ // Fixture setup (AtlasJanusGraphManagement path; __state and __typeName use isStringField=false)
+ // -------------------------------------------------------------------------
+
+ private static IndexFieldNames createSchema(JanusGraph janusGraph, AtlasTypeRegistry typeRegistry) throws Exception {
+ AtlasEntityType assetType = typeRegistry.getEntityTypeByName(TYPE_ASSET);
+ String nameProperty = assetType.getAttribute("name").getVertexPropertyName();
+ String ownerProperty = assetType.getAttribute("owner").getVertexPropertyName();
+
+ AtlasJanusGraph graph = new AtlasJanusGraph(janusGraph);
+
+ IndexFieldNames fields = new IndexFieldNames();
+
+ try (AtlasGraphManagement mgmt = graph.getManagementSystem()) {
+ mgmt.createVertexMixedIndex(OpenSearchQuickSearchSmokeSupport.VERTEX_INDEX,
+ OpenSearchQuickSearchSmokeSupport.BACKING_INDEX_NAME, Collections.emptyList());
+
+ AtlasPropertyKey guidKey = mgmt.makePropertyKey(Constants.GUID_PROPERTY_KEY, String.class, AtlasCardinality.SINGLE);
+ AtlasPropertyKey typeKey = mgmt.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class, AtlasCardinality.SINGLE);
+ AtlasPropertyKey stateKey = mgmt.makePropertyKey(Constants.STATE_PROPERTY_KEY, String.class, AtlasCardinality.SINGLE);
+ AtlasPropertyKey nameKey = mgmt.makePropertyKey(nameProperty, String.class, AtlasCardinality.SINGLE);
+ AtlasPropertyKey ownerKey = mgmt.makePropertyKey(ownerProperty, String.class, AtlasCardinality.SINGLE);
+
+ fields.guidIndexField = mgmt.addMixedIndex(OpenSearchQuickSearchSmokeSupport.VERTEX_INDEX, guidKey, false);
+ fields.typeIndexField = mgmt.addMixedIndex(OpenSearchQuickSearchSmokeSupport.VERTEX_INDEX, typeKey, false, true);
+ fields.stateIndexField = mgmt.addMixedIndex(OpenSearchQuickSearchSmokeSupport.VERTEX_INDEX, stateKey, false, true);
+ fields.nameIndexField = mgmt.addMixedIndex(OpenSearchQuickSearchSmokeSupport.VERTEX_INDEX, nameKey, true);
+ fields.ownerIndexField = mgmt.addMixedIndex(OpenSearchQuickSearchSmokeSupport.VERTEX_INDEX, ownerKey, true);
+
+ mgmt.updateSchemaStatus();
+ mgmt.setIsSuccess(true);
+ }
+ graph.commit();
+
+ return fields;
+ }
+
+ private static AtlasTypeRegistry buildTypeRegistry(IndexFieldNames indexFields) throws Exception {
+ AtlasTypeRegistry registry = new AtlasTypeRegistry();
+
+ AtlasEntityDef assetDef = entityDef(TYPE_ASSET, Collections.emptySet(), SERVICE_A, true);
+ AtlasEntityDef datasetDef = entityDef(TYPE_DATASET, Collections.singleton(TYPE_ASSET), SERVICE_A, false);
+ AtlasEntityDef tableDef = entityDef(TYPE_TABLE, Collections.singleton(TYPE_ASSET), SERVICE_B, false);
+
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+ typesDef.getEntityDefs().add(assetDef);
+ typesDef.getEntityDefs().add(datasetDef);
+ typesDef.getEntityDefs().add(tableDef);
+ registry.updateTypes(typesDef);
+
+ return registry;
+ }
+
+ private static void wireTypeRegistry(AtlasTypeRegistry registry, IndexFieldNames indexFields) {
+ wireType(registry.getEntityTypeByName(TYPE_ASSET), indexFields);
+ wireType(registry.getEntityTypeByName(TYPE_DATASET), indexFields);
+ wireType(registry.getEntityTypeByName(TYPE_TABLE), indexFields);
+
+ registry.addIndexFieldName(Constants.ENTITY_TYPE_PROPERTY_KEY, indexFields.typeIndexField);
+ registry.addIndexFieldName(Constants.STATE_PROPERTY_KEY, indexFields.stateIndexField);
+ registry.addIndexFieldName("owner", indexFields.ownerIndexField);
+ registry.addIndexFieldName("name", indexFields.nameIndexField);
+ }
+
+ private static void wireType(AtlasEntityType type, IndexFieldNames indexFields) {
+ type.getAttribute("name").setIndexFieldName(indexFields.nameIndexField);
+ type.getAttribute("owner").setIndexFieldName(indexFields.ownerIndexField);
+ }
+
+ private static AtlasEntityDef entityDef(String typeName, Set<String> superTypes, String serviceType,
+ boolean includeIndexedAttrs) {
+ AtlasEntityDef def = new AtlasEntityDef();
+ def.setName(typeName);
+ def.setSuperTypes(superTypes);
+ def.setServiceType(serviceType);
+
+ if (includeIndexedAttrs) {
+ List<AtlasAttributeDef> attrs = new ArrayList<>();
+ attrs.add(indexedAttr("name"));
+ attrs.add(indexedAttr("owner"));
+ def.setAttributeDefs(attrs);
+ }
+ return def;
+ }
+
+ private static AtlasAttributeDef indexedAttr(String name) {
+ AtlasAttributeDef attr = new AtlasAttributeDef(name, "string");
+ attr.setIndexType(AtlasAttributeDef.IndexType.STRING);
+ return attr;
+ }
+
+ private static void insertTestEntities(AtlasGraph graph, AtlasTypeRegistry typeRegistry) throws Exception {
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Alice Atlas World", "team-alpha");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Bob Atlas Platform", "team-beta");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Charlie Atlas Marketing", "team-alpha");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Delta Atlas Ops", "team-alpha");
+ insertEntity(graph, typeRegistry, TYPE_TABLE, "Atlas Report Summary", "team-gamma");
+ insertEntity(graph, typeRegistry, TYPE_TABLE, "Sales Atlas Quarterly", "team-delta");
+ insertEntity(graph, typeRegistry, TYPE_TABLE, "Team-Omega Special", "Team-Omega");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Customer Atlas Data", "team-alpha");
+
+ for (int i = 1; i <= 5; i++) {
+ insertEntity(graph, typeRegistry, TYPE_DATASET,
+ "Rank Entity " + i, String.format("rank-%02d", i));
+ }
+ }
+
+ private static void insertEntity(AtlasGraph graph, AtlasTypeRegistry typeRegistry, String typeName,
+ String name, String owner) throws Exception {
+ insertEntity(graph, typeRegistry, typeName, name, owner, AtlasEntity.Status.ACTIVE);
+ }
+
+ private static void insertEntity(AtlasGraph graph, AtlasTypeRegistry typeRegistry, String typeName,
+ String name, String owner, AtlasEntity.Status state) throws Exception {
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
+ AtlasVertex vertex = graph.addVertex();
+ vertex.setProperty(Constants.GUID_PROPERTY_KEY, UUID.randomUUID().toString());
+ vertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
+ vertex.setProperty(Constants.STATE_PROPERTY_KEY, state.name());
+ vertex.setProperty(entityType.getAttribute("name").getVertexPropertyName(), name);
+ vertex.setProperty(entityType.getAttribute("owner").getVertexPropertyName(), owner);
+ }
+
+ // -------------------------------------------------------------------------
+ // Aggregation helpers
+ // -------------------------------------------------------------------------
+
+ private static QuickSearchParameters quickSearchParams(String query, String typeName, FilterCriteria filter) {
+ QuickSearchParameters params = new QuickSearchParameters();
+ params.setQuery(query);
+ params.setTypeName(typeName);
+ params.setEntityFilters(filter);
+ params.setLimit(50);
+ params.setOffset(0);
+ params.setExcludeDeletedEntities(true);
+ params.setIncludeSubTypes(true);
+ return params;
+ }
+
+ private static QuickSearchResult runQuickSearchIndexQuery(SearchContext searchContext,
+ IndexFieldNames indexFields) throws Exception {
+ SearchParameters searchParameters = searchContext.getSearchParameters();
+ String query = searchParameters.getQuery();
+
+ if (StringUtils.isNotEmpty(query) && !org.apache.atlas.type.AtlasStructType.AtlasAttribute.hastokenizeChar(query)) {
+ query = query + "*";
+ }
+
+ searchParameters.setQuery(query);
+
+ Map<String, String> indexFieldNameCache = FreeTextSearchProcessor.buildOpenSearchIndexFieldNameCache(
+ searchContext.getTypeRegistry());
+ indexFieldNameCache.put(Constants.ENTITY_TYPE_PROPERTY_KEY, indexFields.typeIndexField);
+ indexFieldNameCache.put(Constants.STATE_PROPERTY_KEY, indexFields.stateIndexField);
+
+ QuickSearchContext quickSearchContext = new QuickSearchContext(query, searchParameters.getEntityFilters(),
+ FreeTextSearchProcessor.resolveOpenSearchEntityTypes(searchContext),
+ FreeTextSearchProcessor.resolveOpenSearchClassificationTypeNames(searchContext),
+ indexFieldNameCache, searchParameters.getExcludeDeletedEntities(),
+ searchParameters.getIncludeSubTypes(), 0, 0);
+
+ AtlasGraphIndexClient indexClient = searchContext.getGraph().getGraphIndexClient();
+ return indexClient.quickSearch(quickSearchContext);
+ }
+
+ private static AggregationContext aggContext(String query, FilterCriteria filter,
+ Set<AtlasEntityType> entityTypes,
+ Map<String, String> indexFieldNameCache,
+ Set<String> commonFields, Set<AtlasAttribute> attributes,
+ boolean excludeDeleted, boolean includeSubTypes) {
+ return new AggregationContext(query, filter, entityTypes, commonFields, attributes,
+ indexFieldNameCache, excludeDeleted, includeSubTypes);
+ }
+
+ private static Map<String, String> buildIndexFieldNameCache(AtlasTypeRegistry typeRegistry,
+ IndexFieldNames indexFields) {
+ Map<String, String> cache = new HashMap<>();
+ cache.put(Constants.ENTITY_TYPE_PROPERTY_KEY, indexFields.typeIndexField);
+ cache.put(Constants.STATE_PROPERTY_KEY, indexFields.stateIndexField);
+ cache.put("owner", indexFields.ownerIndexField);
+ cache.put("name", indexFields.nameIndexField);
+ return cache;
+ }
+
+ private static Set<AtlasEntityType> entityTypes(AtlasTypeRegistry registry, String... names) {
+ Set<AtlasEntityType> ret = new LinkedHashSet<>();
+ for (String name : names) {
+ ret.add(registry.getEntityTypeByName(name));
+ }
+ return ret;
+ }
+
+ private static Set<AtlasAttribute> ownerAttribute(AtlasTypeRegistry registry) {
+ AtlasAttribute owner = registry.getEntityTypeByName(TYPE_ASSET).getAttribute("owner");
+ return Collections.singleton(owner);
+ }
+
+ private static FilterCriteria leafFilter(String attributeName, Operator operator, String value) {
+ FilterCriteria criteria = new FilterCriteria();
+ criteria.setAttributeName(attributeName);
+ criteria.setOperator(operator);
+ criteria.setAttributeValue(value);
+ return criteria;
+ }
+
+ @SafeVarargs
+ private static <T> Set<T> setOf(T... items) {
+ Set<T> ret = new LinkedHashSet<>();
+ Collections.addAll(ret, items);
+ return ret;
+ }
+
+ @SafeVarargs
+ private static <T> Set<T> linkedSet(T... items) {
+ return setOf(items);
+ }
+
+ // -------------------------------------------------------------------------
+ // Test harness helpers
+ // -------------------------------------------------------------------------
+
+ private interface ValidationStep {
+ void run() throws Exception;
+ }
+
+ private static void record(String name, ValidationStep step) {
+ try {
+ step.run();
+ RESULTS.put(name, "PASS");
+ } catch (AssertionError | Exception e) {
+ RESULTS.put(name, "FAIL (" + e.getMessage() + ")");
+ }
+ }
+
+ private static void assertTrue(boolean condition, String message) {
+ if (!condition) {
+ throw new AssertionError(message);
+ }
+ }
+
+ private static void assertEquals(long actual, long expected, String label) {
+ if (actual != expected) {
+ throw new AssertionError(label + ": expected " + expected + " but was " + actual);
+ }
+ }
+
+ private static void assertEquals(int actual, int expected, String label) {
+ if (actual != expected) {
+ throw new AssertionError(label + ": expected " + expected + " but was " + actual);
+ }
+ }
+
+ private static final class IndexFieldNames {
+ private String guidIndexField;
+ private String typeIndexField;
+ private String stateIndexField;
+ private String nameIndexField;
+ private String ownerIndexField;
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchQuickSearchIT.java b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchQuickSearchIT.java
new file mode 100644
index 000000000..97a59f70b
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchQuickSearchIT.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery.smoke;
+
+import org.apache.atlas.runner.OpenSearchITBase;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertTrue;
+
+/**
+ * Integration test: Atlas quick search via Testcontainers OpenSearch.
+ */
+public class OpenSearchQuickSearchIT extends OpenSearchITBase {
+
+ @Test
+ public void quickSearchValidation() throws Exception {
+ assertTrue(OpenSearchQuickSearchValidationDriver.execute(), "Quick search validation failed");
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchQuickSearchSmokeSupport.java b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchQuickSearchSmokeSupport.java
new file mode 100644
index 000000000..f59c74db5
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchQuickSearchSmokeSupport.java
@@ -0,0 +1,227 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery.smoke;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.repository.graphdb.janus.AtlasOpenSearchIndexClient;
+import org.janusgraph.diskstorage.configuration.ReadConfiguration;
+import org.janusgraph.diskstorage.configuration.backend.CommonsConfiguration;
+import java.io.IOException;
+import java.util.Properties;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.io.OutputStream;
+import java.lang.reflect.Field;
+import org.janusgraph.diskstorage.StandardIndexProvider;
+import org.janusgraph.diskstorage.opensearch.OpenSearchMajorVersion;
+import sun.misc.Unsafe;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Shared helpers for Atlas quick-search validation against a live OpenSearch cluster.
+ */
+final class OpenSearchQuickSearchSmokeSupport {
+
+ static final String GRAPH_INDEX_NAME = "c5janus";
+ static final String BACKING_INDEX_NAME = "search";
+ static final String VERTEX_INDEX = "vertex_index";
+ static final String PHYSICAL_INDEX = GRAPH_INDEX_NAME + "_" + VERTEX_INDEX;
+
+ /** Must match {@link org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase} registration. */
+ static final String OPENSEARCH_INDEX_PROVIDER =
+ "org.janusgraph.diskstorage.opensearch.AtlasOpenSearchIndex";
+
+ private OpenSearchQuickSearchSmokeSupport() {
+ }
+
+ static String getOpenSearchHost() {
+ return System.getProperty("opensearch.host", "localhost");
+ }
+
+ static int getOpenSearchPort() {
+ return Integer.getInteger("opensearch.port", 9200);
+ }
+
+ static void bootstrapApplicationProperties() throws Exception {
+ System.setProperty(ApplicationProperties.ATLAS_CONFIGURATION_DIRECTORY_PROPERTY, "src/test/resources");
+ System.setProperty(ApplicationProperties.ATLAS_PROPERTIES_FILENAME_SYSTEM_CONF, "atlas-opensearch-c5-application.properties");
+ System.setProperty("atlas.graph.index.search.hostname", getOpenSearchHost());
+ System.setProperty("atlas.graph.index.search.port", String.valueOf(getOpenSearchPort()));
+ ApplicationProperties.forceReload();
+ }
+
+ static void verifyOpenSearchReachable() throws IOException {
+ URL url = new URL("http://" + getOpenSearchHost() + ":" + getOpenSearchPort() + "/");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(5000);
+ connection.setRequestMethod("GET");
+ if (connection.getResponseCode() != 200) {
+ throw new IOException("OpenSearch GET / returned HTTP " + connection.getResponseCode());
+ }
+ }
+
+ static String readOpenSearchVersion() throws IOException {
+ URL url = new URL("http://" + getOpenSearchHost() + ":" + getOpenSearchPort() + "/");
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(5000);
+ connection.setRequestMethod("GET");
+ try (java.io.InputStream in = connection.getInputStream()) {
+ String body = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ int numberStart = body.indexOf("\"number\" : \"") + 12;
+ int numberEnd = body.indexOf('"', numberStart);
+ return body.substring(numberStart, numberEnd);
+ }
+ }
+
+ static OpenSearchMajorVersion readOpenSearchMajorVersion() throws IOException {
+ return OpenSearchMajorVersion.parse(readOpenSearchVersion());
+ }
+
+ /**
+ * JanusGraph configuration aligned with {@code atlas-opensearch-c5-application.properties}.
+ * Avoids {@link org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase} static init
+ * so the driver can run on JDK versions where that init requires extra {@code --add-opens} flags.
+ */
+ static ReadConfiguration buildJanusGraphConfiguration() throws IOException {
+ java.io.File dataDir = java.io.File.createTempFile("c5-quicksearch-", "");
+ if (!dataDir.delete() || !dataDir.mkdir()) {
+ throw new IOException("Failed to create temp data directory for JanusGraph");
+ }
+ dataDir.deleteOnExit();
+
+ Properties properties = new Properties();
+ properties.setProperty("storage.backend", "berkeleyje");
+ properties.setProperty("storage.directory", new java.io.File(dataDir, "berkeley").getAbsolutePath());
+ properties.setProperty("index.search.backend", "opensearch");
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".backend", OPENSEARCH_INDEX_PROVIDER);
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".hostname", getOpenSearchHost());
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".port", String.valueOf(getOpenSearchPort()));
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".index-name", GRAPH_INDEX_NAME);
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".opensearch.setup-max-open-scroll-contexts", "false");
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".opensearch.bulk-refresh", "wait_for");
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".opensearch.retry_on_conflict", "3");
+ properties.setProperty("index." + BACKING_INDEX_NAME + ".max-result-set-size", "500");
+
+ org.apache.commons.configuration2.Configuration commons =
+ org.apache.commons.configuration2.ConfigurationConverter.getConfiguration(properties);
+ return new CommonsConfiguration(commons);
+ }
+
+ static void registerAtlasOpenSearchIndex() throws Exception {
+ // AtlasOpenSearchIndexClient keeps its keyword-subfield/suggestion/search-weight registrations in
+ // process-static fields (correct for a real Atlas server, which has exactly one schema for its whole
+ // lifetime). The discovery smoke drivers each build their OWN independent, differently-shaped schema, and when
+ // more than one driver runs in the same JVM/fork (the normal case under the opensearch-it profile, since
+ // Surefire bundles all matching *OpenSearch*IT classes into one suite), JanusGraph's deterministic
+ // property-key-to-field-name encoding means one driver's registered field names (e.g. from
+ // OpenSearchAggregationsValidationDriver's AtlasJanusGraphManagement#addMixedIndex calls) can collide with
+ // and incorrectly apply to another driver's differently-mapped fields of the same encoded name — silently
+ // turning EQ/type-filter queries into a filter on a non-existent ".keyword" field (zero results). Reset
+ // before each driver builds its own schema so no state leaks across driver classes.
+ AtlasOpenSearchIndexClient.setKeywordSubfieldIndexFields(java.util.Collections.emptySet());
+ AtlasOpenSearchIndexClient.applySearchWeight(java.util.Collections.emptyMap());
+ AtlasOpenSearchIndexClient.applySuggestionFields(java.util.Collections.emptyList());
+
+ Class.forName(StandardIndexProvider.class.getName(), true, StandardIndexProvider.class.getClassLoader());
+ Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES");
+ Unsafe unsafe = obtainUnsafe();
+ Object base = unsafe.staticFieldBase(field);
+ long offset = unsafe.staticFieldOffset(field);
+ @SuppressWarnings("unchecked")
+ Map<String, String> current = (Map<String, String>) unsafe.getObject(base, offset);
+ if (current == null) {
+ throw new IllegalStateException("StandardIndexProvider.ALL_MANAGER_CLASSES is null");
+ }
+ Map<String, String> updated = new HashMap<>(current);
+ updated.put("opensearch", OPENSEARCH_INDEX_PROVIDER);
+ unsafe.putObject(base, offset, ImmutableMap.copyOf(updated));
+ }
+
+ static void deletePhysicalIndexIfPresent() throws IOException {
+ URL url = new URL("http://" + getOpenSearchHost() + ":" + getOpenSearchPort() + "/" + PHYSICAL_INDEX);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(5000);
+ connection.setRequestMethod("DELETE");
+ connection.getResponseCode();
+ }
+
+ static long openSearchDocumentCount() throws IOException {
+ String body = httpGet("/" + PHYSICAL_INDEX + "/_count");
+ int valueStart = body.indexOf("\"count\":") + 8;
+ int valueEnd = body.indexOf(',', valueStart);
+ if (valueEnd < 0) {
+ valueEnd = body.indexOf('}', valueStart);
+ }
+ return Long.parseLong(body.substring(valueStart, valueEnd).trim());
+ }
+
+ static void debugOpenSearchSearch(String queryJson) throws IOException {
+ System.out.println("[DEBUG] OpenSearch POST /" + PHYSICAL_INDEX + "/_search");
+ System.out.println("[DEBUG] request body: " + queryJson);
+ String response = httpPost("/" + PHYSICAL_INDEX + "/_search", queryJson);
+ System.out.println("[DEBUG] response (truncated): "
+ + (response.length() > 500 ? response.substring(0, 500) + "..." : response));
+ }
+
+ static String httpGet(String path) throws IOException {
+ URL url = new URL("http://" + getOpenSearchHost() + ":" + getOpenSearchPort() + path);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(15000);
+ connection.setRequestMethod("GET");
+ if (connection.getResponseCode() != 200) {
+ throw new IOException("GET " + path + " returned HTTP " + connection.getResponseCode());
+ }
+ try (java.io.InputStream in = connection.getInputStream()) {
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ static String httpPost(String path, String jsonBody) throws IOException {
+ URL url = new URL("http://" + getOpenSearchHost() + ":" + getOpenSearchPort() + path);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setConnectTimeout(5000);
+ connection.setReadTimeout(15000);
+ connection.setRequestMethod("POST");
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Content-Type", "application/json");
+ byte[] bytes = jsonBody.getBytes(StandardCharsets.UTF_8);
+ connection.setRequestProperty("Content-Length", String.valueOf(bytes.length));
+ try (OutputStream out = connection.getOutputStream()) {
+ out.write(bytes);
+ }
+ if (connection.getResponseCode() != 200) {
+ throw new IOException("POST " + path + " returned HTTP " + connection.getResponseCode());
+ }
+ try (java.io.InputStream in = connection.getInputStream()) {
+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ }
+ }
+
+ private static Unsafe obtainUnsafe() throws Exception {
+ Field unsafeField = Unsafe.class.getDeclaredField("theUnsafe");
+ unsafeField.setAccessible(true);
+ return (Unsafe) unsafeField.get(null);
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchQuickSearchValidationDriver.java b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchQuickSearchValidationDriver.java
new file mode 100644
index 000000000..47308f466
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchQuickSearchValidationDriver.java
@@ -0,0 +1,475 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery.smoke;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.discovery.EntityDiscoveryService;
+import org.apache.atlas.discovery.FreeTextSearchProcessor;
+import org.apache.atlas.discovery.SearchContext;
+import org.apache.atlas.discovery.SearchProcessor;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.discovery.QuickSearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.Operator;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph;
+import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.util.AtlasRepositoryConfiguration;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.Mapping;
+import org.janusgraph.diskstorage.opensearch.OpenSearchMajorVersion;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * Validates Atlas discovery quick-search through the existing stack
+ * (EntityDiscoveryService query shaping → SearchContext → FreeTextSearchProcessor → graph.indexQuery)
+ * against a real OpenSearch server. No new discovery logic; validation only. The server version is configurable
+ * (system property {@code opensearch.docker.version} / {@code opensearch.docker.image}) — the dedicated
+ * {@code opensearch-it} CI job validates OpenSearch 3.7 and 3.8. OpenSearch 2.x is NOT currently part of the CI
+ * matrix; this driver has not been verified against 2.x in CI.
+ *
+ * <pre>
+ * mvn -pl repository -am test-compile exec:java \
+ * -Dexec.classpathScope=test \
+ * -Dexec.mainClass=org.apache.atlas.discovery.smoke.OpenSearchQuickSearchValidationDriver \
+ * -Drat.skip=true -Dcheckstyle.skip=true -Dsortpom.skip=true
+ * </pre>
+ */
+public final class OpenSearchQuickSearchValidationDriver {
+
+ private static final String TYPE_DATASET = "c5_dataset";
+ private static final String TYPE_TABLE = "c5_table";
+ private static final String TYPE_ASSET = "c5_asset";
+
+ private static final Map<String, String> RESULTS = new LinkedHashMap<>();
+
+ private OpenSearchQuickSearchValidationDriver() {
+ }
+
+ public static boolean execute() throws Exception {
+ RESULTS.clear();
+
+ OpenSearchQuickSearchSmokeSupport.bootstrapApplicationProperties();
+ OpenSearchQuickSearchSmokeSupport.verifyOpenSearchReachable();
+
+ OpenSearchQuickSearchSmokeSupport.registerAtlasOpenSearchIndex();
+ if (!AtlasRepositoryConfiguration.isFreeTextSearchEnabled()) {
+ throw new IllegalStateException("atlas.search.freetext.enable must be true for quick search validation");
+ }
+
+ OpenSearchQuickSearchSmokeSupport.deletePhysicalIndexIfPresent();
+
+ JanusGraph janusGraph = JanusGraphFactory.open(OpenSearchQuickSearchSmokeSupport.buildJanusGraphConfiguration());
+ AtlasTypeRegistry typeRegistry = buildTypeRegistry(new IndexFieldNames());
+ IndexFieldNames indexFields = createSchema(janusGraph, typeRegistry);
+ wireIndexFieldNames(typeRegistry, indexFields);
+ AtlasGraph graph = new AtlasJanusGraph(janusGraph);
+ insertTestEntities(graph, typeRegistry);
+
+ graph.commit();
+
+ Thread.sleep(1500);
+
+ runValidations(graph, typeRegistry, indexFields);
+
+ graph.shutdown();
+
+ return RESULTS.values().stream().allMatch("PASS"::equals);
+ }
+
+ public static void main(String[] args) throws Exception {
+ String osVersion = OpenSearchQuickSearchSmokeSupport.readOpenSearchVersion();
+ OpenSearchMajorVersion major = OpenSearchQuickSearchSmokeSupport.readOpenSearchMajorVersion();
+
+ System.out.println();
+ System.out.println("Atlas Quick Search Validation");
+ System.out.println("----------------------------------");
+ System.out.println("Backend: OpenSearch");
+ System.out.println("OpenSearch: " + osVersion + " (major=" + major + ")");
+ System.out.println("JanusGraph index prefix: " + AtlasGraphUtilsV2.getIndexSearchPrefix());
+ System.out.println("Physical index: " + OpenSearchQuickSearchSmokeSupport.PHYSICAL_INDEX);
+ System.out.println();
+
+ boolean allPassed = execute();
+
+ System.out.println();
+ for (Map.Entry<String, String> entry : RESULTS.entrySet()) {
+ System.out.printf("%-24s %s%n", entry.getKey(), entry.getValue());
+ }
+ System.out.println();
+ System.out.println(allPassed ? "RESULT: PASS" : "RESULT: FAIL");
+
+ if (!allPassed) {
+ System.exit(1);
+ }
+ }
+
+ // -------------------------------------------------------------------------
+ // Validations (Atlas discovery path)
+ // -------------------------------------------------------------------------
+
+ private static void runValidations(AtlasGraph graph, AtlasTypeRegistry typeRegistry, IndexFieldNames indexFields)
+ throws Exception {
+ record("Basic keyword", () -> {
+ QuickSearchOutcome out = quickSearch(graph, typeRegistry, indexFields, params("atlas", TYPE_DATASET, 10, 0, null, null));
+ assertTrue(out.resultSize >= 2, "expected at least 2 atlas matches, got " + out.resultSize);
+ assertTrue(out.approximateCount >= 2, "count=" + out.approximateCount);
+ debugTrace(out, "{\"query_string\":{\"query\":\"atlas*\",\"fields\":[\"*\"]}}");
+ });
+
+ record("Multiple keywords", () -> {
+ QuickSearchOutcome out = quickSearch(graph, typeRegistry, indexFields, params("alice atlas", TYPE_DATASET, 10, 0, null, null));
+ assertTrue(out.resultSize >= 1, "expected alice atlas match");
+ });
+
+ record("Type filter", () -> {
+ QuickSearchOutcome dataset = quickSearch(graph, typeRegistry, indexFields, params("atlas", TYPE_DATASET, 10, 0, null, null));
+ QuickSearchOutcome table = quickSearch(graph, typeRegistry, indexFields, params("atlas", TYPE_TABLE, 10, 0, null, null));
+ assertTrue(dataset.resultSize >= 2, "dataset hits=" + dataset.resultSize);
+ assertTrue(table.resultSize >= 1, "table hits=" + table.resultSize);
+ assertTrue(dataset.resultSize != table.resultSize || table.resultSize == 1,
+ "type filter should differentiate results");
+ });
+
+ record("Property filter", () -> {
+ FilterCriteria filter = new FilterCriteria();
+ filter.setAttributeName("owner");
+ filter.setAttributeValue("team-alpha");
+ filter.setOperator(Operator.EQ);
+ QuickSearchOutcome out = quickSearch(graph, typeRegistry, indexFields,
+ params("atlas", TYPE_DATASET, 10, 0, filter, null));
+ assertTrue(out.resultSize >= 1, "owner filter expected hits, got " + out.resultSize);
+ });
+
+ record("Pagination", () -> {
+ QuickSearchOutcome page1 = quickSearch(graph, typeRegistry, indexFields, params("report", null, 2, 0, null, null));
+ QuickSearchOutcome page2 = quickSearch(graph, typeRegistry, indexFields, params("report", null, 2, 2, null, null));
+ assertEquals(page1.resultSize, 2, "page1 size");
+ assertTrue(page2.resultSize >= 1, "page2 size");
+ assertDifferentGuids(page1, page2);
+ });
+
+ record("Result count", () -> {
+ QuickSearchOutcome out = quickSearch(graph, typeRegistry, indexFields, params("atlas", null, 50, 0, null, null));
+ assertTrue(out.approximateCount >= 3, "approximateCount=" + out.approximateCount);
+ assertTrue(out.resultSize >= 3, "resultSize=" + out.resultSize);
+ });
+
+ record("No-result query", () -> {
+ QuickSearchOutcome out = quickSearch(graph, typeRegistry, indexFields,
+ params("zzznomatch999", TYPE_DATASET, 10, 0, null, null));
+ assertEquals(out.resultSize, 0, "resultSize");
+ assertEquals(out.approximateCount, 0L, "approximateCount");
+ });
+
+ record("Case/partial search", () -> {
+ QuickSearchOutcome lower = quickSearch(graph, typeRegistry, indexFields, params("alice", TYPE_DATASET, 10, 0, null, null));
+ QuickSearchOutcome upper = quickSearch(graph, typeRegistry, indexFields, params("ALICE", TYPE_DATASET, 10, 0, null, null));
+ assertTrue(lower.resultSize >= 1, "lowercase partial");
+ assertTrue(upper.resultSize >= 1, "uppercase partial (Atlas adds trailing *)");
+ });
+
+ record("Multiple entity types", () -> {
+ QuickSearchOutcome out = quickSearch(graph, typeRegistry, indexFields, params("report", null, 20, 0, null, null));
+ assertTrue(out.resultSize >= 2, "hits across types=" + out.resultSize);
+ assertTrue(out.typesSeen.contains(TYPE_DATASET) && out.typesSeen.contains(TYPE_TABLE),
+ "types seen: " + out.typesSeen);
+ });
+
+ record("Existing index", () -> {
+ long count = OpenSearchQuickSearchSmokeSupport.openSearchDocumentCount();
+ assertTrue(count >= 5, "index should already contain seeded docs, count=" + count);
+ QuickSearchOutcome out = quickSearch(graph, typeRegistry, indexFields, params("charlie", TYPE_DATASET, 5, 0, null, null));
+ assertTrue(out.resultSize >= 1, "search on existing index");
+ });
+
+ record("Prefix wildcard query", () -> {
+ QuickSearchOutcome out = quickSearch(graph, typeRegistry, indexFields, params("custo*", TYPE_DATASET, 10, 0, null, null));
+ assertTrue(out.resultSize >= 1, "custo* should match Customer Atlas Data, got " + out.resultSize);
+ assertTrue(out.approximateCount >= 1, "approximateCount=" + out.approximateCount);
+ });
+ }
+
+ /**
+ * Mirrors {@link EntityDiscoveryService#quickSearch} search half (no aggregations).
+ */
+ private static QuickSearchOutcome quickSearch(AtlasGraph graph, AtlasTypeRegistry typeRegistry,
+ IndexFieldNames indexFields,
+ QuickSearchParameters quickSearchParameters) throws AtlasBaseException {
+ String query = quickSearchParameters.getQuery();
+ if (StringUtils.isNotEmpty(query) && !AtlasAttribute.hastokenizeChar(query)) {
+ query = query + "*";
+ }
+ quickSearchParameters.setQuery(query);
+
+ SearchParameters searchParameters = EntityDiscoveryService.createSearchParameters(quickSearchParameters);
+ SearchContext searchContext = new SearchContext(searchParameters, typeRegistry, graph,
+ buildIndexedKeys(typeRegistry, indexFields));
+
+ SearchProcessor processor = searchContext.getSearchProcessor();
+ if (!(processor instanceof FreeTextSearchProcessor) && searchContext.getSearchParameters().getQuery() != null) {
+ // SearchContext may chain processors; ensure we exercise the full processor chain as quickSearch does.
+ }
+
+ List<AtlasVertex> vertices = processor.execute();
+ long count = processor.getResultCount();
+
+ QuickSearchOutcome outcome = new QuickSearchOutcome();
+ outcome.atlasQuery = searchParameters.getQuery();
+ outcome.approximateCount = count;
+ outcome.resultSize = vertices.size();
+ outcome.vertices = vertices;
+ outcome.typesSeen = new ArrayList<>();
+
+ for (AtlasVertex vertex : vertices) {
+ String typeName = vertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
+ if (typeName != null && !outcome.typesSeen.contains(typeName)) {
+ outcome.typesSeen.add(typeName);
+ }
+ }
+
+ return outcome;
+ }
+
+ private static QuickSearchParameters params(String query, String typeName, int limit, int offset,
+ FilterCriteria entityFilter, String sortBy) {
+ QuickSearchParameters p = new QuickSearchParameters();
+ p.setQuery(query);
+ p.setTypeName(typeName);
+ p.setLimit(limit);
+ p.setOffset(offset);
+ p.setExcludeDeletedEntities(true);
+ p.setIncludeSubTypes(true);
+ p.setEntityFilters(entityFilter);
+ p.setSortBy(sortBy);
+ return p;
+ }
+
+ private static void debugTrace(QuickSearchOutcome outcome, String exampleOsQuery) {
+ try {
+ System.out.println("[DEBUG] Atlas query (post suffix): " + outcome.atlasQuery);
+ System.out.println("[DEBUG] Processor approximate count: " + outcome.approximateCount);
+ OpenSearchQuickSearchSmokeSupport.debugOpenSearchSearch(exampleOsQuery);
+ } catch (Exception e) {
+ System.out.println("[DEBUG] OpenSearch debug trace skipped: " + e.getMessage());
+ }
+ }
+
+ // -------------------------------------------------------------------------
+ // Fixture setup
+ // -------------------------------------------------------------------------
+
+ private static IndexFieldNames createSchema(JanusGraph graph, AtlasTypeRegistry typeRegistry)
+ throws org.apache.atlas.exception.AtlasBaseException {
+ AtlasEntityType assetType = typeRegistry.getEntityTypeByName(TYPE_ASSET);
+ String nameProperty = assetType.getAttribute("name").getVertexPropertyName();
+ String ownerProperty = assetType.getAttribute("owner").getVertexPropertyName();
+
+ JanusGraphManagement mgmt = graph.openManagement();
+
+ PropertyKey guidKey = mgmt.makePropertyKey(Constants.GUID_PROPERTY_KEY).dataType(String.class).cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey typeKey = mgmt.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY).dataType(String.class).cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey stateKey = mgmt.makePropertyKey(Constants.STATE_PROPERTY_KEY).dataType(String.class).cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey nameKey = mgmt.makePropertyKey(nameProperty).dataType(String.class).cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey ownerKey = mgmt.makePropertyKey(ownerProperty).dataType(String.class).cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+
+ mgmt.buildIndex(OpenSearchQuickSearchSmokeSupport.VERTEX_INDEX, Vertex.class)
+ .addKey(guidKey, Mapping.STRING.asParameter())
+ .addKey(typeKey, Mapping.STRING.asParameter())
+ .addKey(stateKey, Mapping.STRING.asParameter())
+ .addKey(nameKey, Mapping.TEXT.asParameter())
+ .addKey(ownerKey, Mapping.STRING.asParameter())
+ .buildMixedIndex(OpenSearchQuickSearchSmokeSupport.BACKING_INDEX_NAME);
+
+ IndexFieldNames fields = new IndexFieldNames();
+ fields.guidIndexField = guidKey.name();
+ fields.typeIndexField = typeKey.name();
+ fields.stateIndexField = stateKey.name();
+ fields.nameIndexField = nameKey.name();
+ fields.ownerIndexField = ownerKey.name();
+
+ mgmt.commit();
+ graph.tx().commit();
+ return fields;
+ }
+
+ private static AtlasTypeRegistry buildTypeRegistry(IndexFieldNames indexFields) throws org.apache.atlas.exception.AtlasBaseException {
+ AtlasTypeRegistry registry = new AtlasTypeRegistry();
+
+ AtlasEntityDef assetDef = entityDef(TYPE_ASSET, Collections.emptySet(), indexFields, true);
+ AtlasEntityDef datasetDef = entityDef(TYPE_DATASET, Collections.singleton(TYPE_ASSET), indexFields, false);
+ AtlasEntityDef tableDef = entityDef(TYPE_TABLE, Collections.singleton(TYPE_ASSET), indexFields, false);
+
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+ typesDef.getEntityDefs().add(assetDef);
+ typesDef.getEntityDefs().add(datasetDef);
+ typesDef.getEntityDefs().add(tableDef);
+ registry.updateTypes(typesDef);
+
+ return registry;
+ }
+
+ private static void wireIndexFieldNames(AtlasTypeRegistry registry, IndexFieldNames indexFields) {
+ wireIndexFieldNames(registry.getEntityTypeByName(TYPE_ASSET), indexFields);
+ wireIndexFieldNames(registry.getEntityTypeByName(TYPE_DATASET), indexFields);
+ wireIndexFieldNames(registry.getEntityTypeByName(TYPE_TABLE), indexFields);
+ registry.addIndexFieldName(Constants.ENTITY_TYPE_PROPERTY_KEY, indexFields.typeIndexField);
+ registry.addIndexFieldName(Constants.STATE_PROPERTY_KEY, indexFields.stateIndexField);
+ }
+
+ private static AtlasEntityDef entityDef(String typeName, java.util.Set<String> superTypes,
+ IndexFieldNames indexFields, boolean includeIndexedAttrs) {
+ AtlasEntityDef def = new AtlasEntityDef();
+ def.setName(typeName);
+ def.setSuperTypes(superTypes);
+
+ if (includeIndexedAttrs) {
+ List<AtlasAttributeDef> attrs = new ArrayList<>();
+ attrs.add(indexedAttr("name", indexFields.nameIndexField));
+ attrs.add(indexedAttr("owner", indexFields.ownerIndexField));
+ def.setAttributeDefs(attrs);
+ }
+ return def;
+ }
+
+ private static AtlasAttributeDef indexedAttr(String name, String indexFieldName) {
+ AtlasAttributeDef attr = new AtlasAttributeDef(name, "string");
+ attr.setIndexType(AtlasAttributeDef.IndexType.STRING);
+ return attr;
+ }
+
+ private static void wireIndexFieldNames(AtlasEntityType type, IndexFieldNames indexFields) {
+ type.getAttribute("name").setIndexFieldName(indexFields.nameIndexField);
+ type.getAttribute("owner").setIndexFieldName(indexFields.ownerIndexField);
+ }
+
+ private static Set<String> buildIndexedKeys(AtlasTypeRegistry typeRegistry, IndexFieldNames indexFields)
+ throws org.apache.atlas.exception.AtlasBaseException {
+ Set<String> keys = new HashSet<>();
+ AtlasEntityType assetType = typeRegistry.getEntityTypeByName(TYPE_ASSET);
+ keys.add(assetType.getAttribute("name").getVertexPropertyName());
+ keys.add(assetType.getAttribute("owner").getVertexPropertyName());
+ keys.add(indexFields.guidIndexField);
+ keys.add(indexFields.typeIndexField);
+ keys.add(indexFields.stateIndexField);
+ return keys;
+ }
+
+ private static void insertTestEntities(AtlasGraph graph, AtlasTypeRegistry typeRegistry)
+ throws org.apache.atlas.exception.AtlasBaseException {
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Alice Atlas World", "team-alpha");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Bob Atlas Platform", "team-beta");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Charlie Marketing Report", "team-alpha");
+ insertEntity(graph, typeRegistry, TYPE_TABLE, "Atlas Report Summary", "team-gamma");
+ insertEntity(graph, typeRegistry, TYPE_TABLE, "Sales Report Quarterly", "team-delta");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Customer Atlas Data", "team-alpha");
+ }
+
+ private static void insertEntity(AtlasGraph graph, AtlasTypeRegistry typeRegistry, String typeName,
+ String name, String owner) throws org.apache.atlas.exception.AtlasBaseException {
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
+ AtlasVertex vertex = graph.addVertex();
+ vertex.setProperty(Constants.GUID_PROPERTY_KEY, UUID.randomUUID().toString());
+ vertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
+ vertex.setProperty(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
+ vertex.setProperty(entityType.getAttribute("name").getVertexPropertyName(), name);
+ vertex.setProperty(entityType.getAttribute("owner").getVertexPropertyName(), owner);
+ }
+
+ // -------------------------------------------------------------------------
+ // Test harness helpers
+ // -------------------------------------------------------------------------
+
+ private interface ValidationStep {
+ void run() throws Exception;
+ }
+
+ private static void record(String name, ValidationStep step) {
+ try {
+ step.run();
+ RESULTS.put(name, "PASS");
+ } catch (AssertionError | Exception e) {
+ RESULTS.put(name, "FAIL (" + e.getMessage() + ")");
+ }
+ }
+
+ private static void assertTrue(boolean condition, String message) {
+ if (!condition) {
+ throw new AssertionError(message);
+ }
+ }
+
+ private static void assertEquals(long actual, long expected, String label) {
+ if (actual != expected) {
+ throw new AssertionError(label + ": expected " + expected + " but was " + actual);
+ }
+ }
+
+ private static void assertEquals(int actual, int expected, String label) {
+ if (actual != expected) {
+ throw new AssertionError(label + ": expected " + expected + " but was " + actual);
+ }
+ }
+
+ private static void assertDifferentGuids(QuickSearchOutcome a, QuickSearchOutcome b) {
+ String guidA = a.vertices.isEmpty() ? null : a.vertices.get(0).getProperty(Constants.GUID_PROPERTY_KEY, String.class);
+ String guidB = b.vertices.isEmpty() ? null : b.vertices.get(0).getProperty(Constants.GUID_PROPERTY_KEY, String.class);
+ if (guidA != null && guidA.equals(guidB)) {
+ throw new AssertionError("pagination returned duplicate first GUID");
+ }
+ }
+
+ private static final class IndexFieldNames {
+ private String guidIndexField;
+ private String typeIndexField;
+ private String stateIndexField;
+ private String nameIndexField;
+ private String ownerIndexField;
+ }
+
+ private static final class QuickSearchOutcome {
+ private String atlasQuery;
+ private long approximateCount;
+ private int resultSize;
+ private List<AtlasVertex> vertices = Collections.emptyList();
+ private List<String> typesSeen = Collections.emptyList();
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSearchWeightBaselineDriver.java b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSearchWeightBaselineDriver.java
new file mode 100644
index 000000000..e4107498e
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSearchWeightBaselineDriver.java
@@ -0,0 +1,720 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery.smoke;
+
+import ch.qos.logback.classic.Level;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.atlas.discovery.EntityDiscoveryService;
+import org.apache.atlas.discovery.FreeTextSearchProcessor;
+import org.apache.atlas.discovery.SearchContext;
+import org.apache.atlas.discovery.SearchProcessor;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.AtlasAggregationEntry;
+import org.apache.atlas.model.discovery.QuickSearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.Operator;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AggregationContext;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph;
+import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.util.AtlasRepositoryConfiguration;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.Mapping;
+import org.janusgraph.diskstorage.opensearch.OpenSearchMajorVersion;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+/**
+ * Read-only baseline: controlled ranking dataset and live payload capture.
+ * Compares Atlas/JanusGraph quick-search ordering vs raw OpenSearch {@code query_string}
+ * vs proposed {@code multi_match} field boosts. No production code changes.
+ *
+ * <pre>
+ * cd repository && mvn test-compile exec:java \
+ * -Dexec.classpathScope=test \
+ * -Dexec.mainClass=org.apache.atlas.discovery.smoke.OpenSearchSearchWeightBaselineDriver \
+ * -Drat.skip=true -Dcheckstyle.skip=true -Dsortpom.skip=true -DskipCheck=true
+ * </pre>
+ */
+public final class OpenSearchSearchWeightBaselineDriver {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private static final String TYPE_ASSET = "c55_asset";
+ private static final String TYPE_DATASET = "c55_dataset";
+
+ /** Conceptual Solr weights for documentation / multi_match probe. */
+ private static final int WEIGHT_NAME = 10;
+ private static final int WEIGHT_OWNER = 3;
+
+ private static final String LABEL_A = "A";
+ private static final String LABEL_B = "B";
+ private static final String LABEL_C = "C";
+ private static final String LABEL_D = "D";
+
+ private OpenSearchSearchWeightBaselineDriver() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ enableOpenSearchIndexDebugLogging();
+
+ OpenSearchQuickSearchSmokeSupport.bootstrapApplicationProperties();
+ OpenSearchQuickSearchSmokeSupport.verifyOpenSearchReachable();
+ String osVersion = OpenSearchQuickSearchSmokeSupport.readOpenSearchVersion();
+ OpenSearchMajorVersion major = OpenSearchQuickSearchSmokeSupport.readOpenSearchMajorVersion();
+
+ if (!AtlasRepositoryConfiguration.isFreeTextSearchEnabled()) {
+ throw new IllegalStateException("atlas.search.freetext.enable must be true");
+ }
+
+ OpenSearchQuickSearchSmokeSupport.registerAtlasOpenSearchIndex();
+ OpenSearchQuickSearchSmokeSupport.deletePhysicalIndexIfPresent();
+
+ JanusGraph janusGraph = JanusGraphFactory.open(OpenSearchQuickSearchSmokeSupport.buildJanusGraphConfiguration());
+ AtlasTypeRegistry typeRegistry = buildTypeRegistry();
+ IndexFieldNames indexFields = createSchema(janusGraph, typeRegistry);
+ wireTypeRegistry(typeRegistry, indexFields);
+
+ Map<String, String> guidByLabel = insertBaselineEntities(janusGraph, typeRegistry);
+ AtlasGraph graph = new AtlasJanusGraph(janusGraph);
+ graph.commit();
+ Thread.sleep(1500);
+
+ Map<String, String> osFieldNames = fetchOpenSearchFieldNames();
+
+ System.out.println();
+ System.out.println("Search-Weight Ranking Baseline (read-only)");
+ System.out.println("================================================");
+ System.out.println("OpenSearch: " + osVersion + " (major=" + major + ")");
+ System.out.println("Physical index: " + OpenSearchQuickSearchSmokeSupport.PHYSICAL_INDEX);
+ System.out.println("JanusGraph index prefix: " + AtlasGraphUtilsV2.getIndexSearchPrefix());
+ System.out.println("OpenSearch doc count: " + OpenSearchQuickSearchSmokeSupport.openSearchDocumentCount());
+ System.out.println("Mapped name field: " + osFieldNames.get("name"));
+ System.out.println("Mapped owner field: " + osFieldNames.get("owner"));
+ System.out.println();
+
+ printSolrAvailability();
+ printControlledDataset();
+
+ String atlasQueryTerm = "customer";
+ String janusQueryString = buildJanusGraphQueryString(atlasQueryTerm, typeRegistry);
+
+ System.out.println("--- Atlas query string (JanusGraph RawQuery input) ---");
+ System.out.println(janusQueryString);
+ System.out.println();
+
+ String queryStringPayload = buildQueryStringSearchPayload(janusQueryString, 20);
+ String multiMatchPayload = buildMultiMatchPayload(atlasQueryTerm, osFieldNames, 20);
+
+ System.out.println("--- Live OpenSearch _search payload: query_string (JanusGraph equivalent) ---");
+ System.out.println(prettyJson(queryStringPayload));
+ System.out.println();
+
+ System.out.println("--- Proposed _search payload: bool filter + multi_match field boosts ---");
+ System.out.println(prettyJson(multiMatchPayload));
+ System.out.println();
+
+ List<RankedHit> atlasRanking = runAtlasQuickSearch(graph, typeRegistry, indexFields, atlasQueryTerm, guidByLabel);
+
+ System.out.println("--- Atlas ordering (observation 1) ---");
+ printHitList(atlasRanking);
+
+ List<RankedHit> queryStringRanking = runRawSearch(queryStringPayload, guidByLabel, "query_string (observation 2)");
+ List<RankedHit> multiMatchRanking = runRawSearch(multiMatchPayload, guidByLabel, "multi_match name^10 owner^3 (observation 3)");
+
+ printRankingMatrix(atlasRanking, queryStringRanking, multiMatchRanking);
+
+ String invertedPayload = buildInvertedWeightPayload(atlasQueryTerm, osFieldNames, 20);
+ List<RankedHit> invertedRanking = runRawSearch(invertedPayload, guidByLabel, "multi_match INVERTED name^3 owner^10 (observation 4)");
+ System.out.println("--- Weight inversion (observation 4) ---");
+ System.out.println("name^10/owner^3: " + formatOrder(multiMatchRanking));
+ System.out.println("name^3/owner^10: " + formatOrder(invertedRanking));
+ System.out.println("Ranking changed when weights inverted: " + !sameLabelOrder(multiMatchRanking, invertedRanking));
+ System.out.println();
+
+ runExtendedProbes(graph, typeRegistry, indexFields, guidByLabel, osFieldNames);
+ runAggregationIndependenceProbe(graph, typeRegistry, indexFields);
+
+ graph.shutdown();
+
+ System.out.println();
+ System.out.println("BASELINE COMPLETE (read-only — no production changes)");
+ }
+
+ // -------------------------------------------------------------------------
+ // Core ranking experiment
+ // -------------------------------------------------------------------------
+
+ private static void printControlledDataset() {
+ System.out.println("--- Controlled dataset (conceptual weights: name^10, owner^3) ---");
+ System.out.println(" A: name=customer owner=alice (name match, high weight)");
+ System.out.println(" B: name=sales owner=customer (owner match, low weight)");
+ System.out.println(" C: name=customer sales owner=bob (name token match, high weight)");
+ System.out.println(" D: name=other owner=other labels=customer (labels^10 probe)");
+ System.out.println();
+ System.out.println("Expected Solr/edismax ordering (name^10 > owner^3): A, C, B (D via labels separate probe)");
+ System.out.println();
+ }
+
+ private static void printSolrAvailability() {
+ System.out.println("--- Solr baseline ---");
+ try {
+ java.net.URL url = new java.net.URL("http://localhost:8983/solr/");
+ java.net.HttpURLConnection c = (java.net.HttpURLConnection) url.openConnection();
+ c.setConnectTimeout(2000);
+ c.setReadTimeout(2000);
+ c.setRequestMethod("GET");
+ if (c.getResponseCode() == 200) {
+ System.out.println("Solr reachable at localhost:8983 — manual /freetext comparison not automated in this driver.");
+ } else {
+ System.out.println("Solr not available (HTTP " + c.getResponseCode() + ") — Solr ordering marked N/A.");
+ }
+ } catch (Exception e) {
+ System.out.println("Solr not available — Solr ordering marked N/A.");
+ }
+ System.out.println();
+ }
+
+ private static String buildJanusGraphQueryString(String queryTerm, AtlasTypeRegistry typeRegistry)
+ throws AtlasBaseException {
+ String query = queryTerm;
+ if (!AtlasAttribute.hastokenizeChar(query)) {
+ query = query + "*";
+ }
+
+ AtlasEntityType datasetType = typeRegistry.getEntityTypeByName(TYPE_DATASET);
+ String typeFilter = AtlasAttribute.escapeIndexQueryValue(
+ Collections.singleton(datasetType.getTypeName()), true);
+
+ String prefix = AtlasGraphUtilsV2.getIndexSearchPrefix();
+ return query + SearchProcessor.AND_STR + typeFilter + SearchProcessor.AND_STR
+ + "(" + prefix + "\"" + Constants.STATE_PROPERTY_KEY + "\":ACTIVE)";
+ }
+
+ private static List<RankedHit> runAtlasQuickSearch(AtlasGraph graph, AtlasTypeRegistry typeRegistry,
+ IndexFieldNames indexFields, String queryTerm,
+ Map<String, String> guidByLabel) throws Exception {
+ QuickSearchParameters qp = new QuickSearchParameters();
+ qp.setQuery(queryTerm);
+ qp.setTypeName(TYPE_DATASET);
+ qp.setExcludeDeletedEntities(true);
+ qp.setIncludeSubTypes(true);
+ qp.setLimit(20);
+ qp.setOffset(0);
+
+ SearchParameters sp = EntityDiscoveryService.createSearchParameters(qp);
+ String query = sp.getQuery();
+ if (query != null && !AtlasAttribute.hastokenizeChar(query)) {
+ query = query + "*";
+ }
+ sp.setQuery(query);
+
+ SearchContext ctx = new SearchContext(sp, typeRegistry, graph, buildIndexedKeys(typeRegistry, indexFields));
+ SearchProcessor processor = ctx.getSearchProcessor();
+
+ System.out.println("--- Atlas quick-search path ---");
+ System.out.println("Processor: " + processor.getClass().getSimpleName());
+ if (!(processor instanceof FreeTextSearchProcessor)) {
+ System.out.println("WARN: expected FreeTextSearchProcessor");
+ }
+
+ List<AtlasVertex> vertices = processor.execute();
+
+ List<RankedHit> hits = new ArrayList<>();
+ int rank = 1;
+ for (AtlasVertex v : vertices) {
+ String guid = v.getProperty(Constants.GUID_PROPERTY_KEY, String.class);
+ String label = labelForGuid(guid, guidByLabel);
+ hits.add(new RankedHit(rank++, label, guid, Double.NaN, "Atlas/JanusGraph"));
+ }
+ return hits;
+ }
+
+ private static List<RankedHit> runRawSearch(String payload, Map<String, String> guidByLabel, String probeLabel) throws Exception {
+ System.out.println("--- Raw OpenSearch: " + probeLabel + " ---");
+ String response;
+ try {
+ response = OpenSearchQuickSearchSmokeSupport.httpPost(
+ "/" + OpenSearchQuickSearchSmokeSupport.PHYSICAL_INDEX + "/_search", payload);
+ } catch (IOException e) {
+ System.out.println("WARN: " + e.getMessage());
+ System.out.println("(Atlas/JanusGraph query_string syntax is not always directly POSTable to OpenSearch)");
+ return Collections.emptyList();
+ }
+ JsonNode root = MAPPER.readTree(response);
+ JsonNode hitsNode = root.path("hits").path("hits");
+
+ List<RankedHit> hits = new ArrayList<>();
+ int rank = 1;
+ for (JsonNode hit : hitsNode) {
+ String id = hit.path("_id").asText();
+ double score = hit.path("_score").asDouble();
+ JsonNode source = hit.path("_source");
+ String guid = source.isObject() && source.has("__guid") ? source.get("__guid").asText() : id;
+ String entityLabel = labelForGuid(guid, guidByLabel);
+ if (entityLabel == null) {
+ entityLabel = guid.length() > 8 ? guid.substring(0, 8) + "…" : guid;
+ }
+ hits.add(new RankedHit(rank++, entityLabel, guid, score, null));
+ }
+ return hits;
+ }
+
+ private static void printHitList(List<RankedHit> hits) {
+ if (hits.isEmpty()) {
+ System.out.println(" (no hits)");
+ } else {
+ for (RankedHit h : hits) {
+ System.out.printf(" %d. %s%n", h.rank, h.label);
+ }
+ }
+ System.out.println();
+ }
+
+ private static String formatOrder(List<RankedHit> hits) {
+ return hits.stream().map(h -> h.label).collect(Collectors.joining(" → "));
+ }
+
+ private static void printRankingMatrix(List<RankedHit> atlas, List<RankedHit> queryString, List<RankedHit> multiMatch) {
+ System.out.println("--- Ranking comparison (query: customer*) ---");
+ System.out.printf("%-28s %-22s %-22s%n", "Atlas/JanusGraph order", "OS query_string order", "OS multi_match order");
+ int rows = Math.max(atlas.size(), Math.max(queryString.size(), multiMatch.size()));
+ for (int i = 0; i < rows; i++) {
+ System.out.printf("%-28s %-22s %-22s%n",
+ formatHit(atlas, i),
+ formatHit(queryString, i),
+ formatHit(multiMatch, i));
+ }
+ System.out.println();
+ System.out.println("Solr /freetext ordering: N/A (Solr not in baseline environment)");
+ System.out.println();
+
+ boolean atlasMatchesMultiMatch = sameLabelOrder(atlas, multiMatch);
+ boolean atlasMatchesQueryString = sameLabelOrder(atlas, queryString);
+ boolean queryStringMatchesMultiMatch = sameLabelOrder(queryString, multiMatch);
+
+ System.out.println("--- Baseline conclusions ---");
+ System.out.println("Atlas order == query_string order: " + atlasMatchesQueryString
+ + " (expected true — same underlying JanusGraph path)");
+ System.out.println("Atlas order == multi_match order: " + atlasMatchesMultiMatch
+ + " (false implies weighted search path would change ranking)");
+ System.out.println("query_string == multi_match: " + queryStringMatchesMultiMatch);
+ if (!atlasMatchesMultiMatch) {
+ System.out.println("→ Ranking parity should target Atlas-level multi_match weighting.");
+ }
+ System.out.println();
+ }
+
+ // -------------------------------------------------------------------------
+ // Extended probes (read-only)
+ // -------------------------------------------------------------------------
+
+ private static void runExtendedProbes(AtlasGraph graph, AtlasTypeRegistry typeRegistry,
+ IndexFieldNames indexFields, Map<String, String> guidByLabel,
+ Map<String, String> osFieldNames) throws Exception {
+ System.out.println("--- Extended probes ---");
+
+ // labels^10 probe via raw multi_match
+ String labelsPayload = buildMultiMatchPayloadWithExtraField("customer", osFieldNames,
+ osFieldNames.get("labels"), 10, 10);
+ List<RankedHit> labelsHits = runRawSearch(labelsPayload, guidByLabel, "labels^10 probe");
+ System.out.println("labels^10 multi_match order: " + labelsHits.stream()
+ .map(h -> h.label).collect(Collectors.joining(" > ")));
+
+ // owner-only weight dominance: owner^10 name^3 inverted
+ String invertedPayload = buildInvertedWeightPayload("customer", osFieldNames, 10);
+ List<RankedHit> invertedHits = runRawSearch(invertedPayload, guidByLabel, "extended inverted weights");
+ System.out.println("inverted owner^10/name^3 order: " + invertedHits.stream()
+ .map(h -> h.label + "(" + String.format("%.2f", h.score) + ")")
+ .collect(Collectors.joining(" > ")));
+
+ // Type filter + quick search
+ QuickSearchParameters qp = new QuickSearchParameters();
+ qp.setQuery("customer");
+ qp.setTypeName(TYPE_DATASET);
+ qp.setExcludeDeletedEntities(true);
+ qp.setIncludeSubTypes(true);
+ qp.setLimit(10);
+ SearchParameters sp = EntityDiscoveryService.createSearchParameters(qp);
+ sp.setQuery(sp.getQuery() + "*");
+ SearchContext ctx = new SearchContext(sp, typeRegistry, graph, buildIndexedKeys(typeRegistry, indexFields));
+ int typeFilteredCount = ctx.getSearchProcessor().execute().size();
+ System.out.println("Type-filtered Atlas hits (c55_dataset): " + typeFilteredCount);
+
+ // Property filter
+ FilterCriteria filter = new FilterCriteria();
+ filter.setAttributeName("owner");
+ filter.setOperator(Operator.EQ);
+ filter.setAttributeValue("customer");
+ qp.setEntityFilters(filter);
+ sp = EntityDiscoveryService.createSearchParameters(qp);
+ sp.setQuery("customer*");
+ ctx = new SearchContext(sp, typeRegistry, graph, buildIndexedKeys(typeRegistry, indexFields));
+ int propFilteredCount = ctx.getSearchProcessor().execute().size();
+ System.out.println("Owner=customer filter + customer* query hits: " + propFilteredCount + " (expect B only)");
+
+ // Pagination
+ qp.setEntityFilters(null);
+ qp.setLimit(2);
+ qp.setOffset(0);
+ sp = EntityDiscoveryService.createSearchParameters(qp);
+ sp.setQuery("customer*");
+ ctx = new SearchContext(sp, typeRegistry, graph, buildIndexedKeys(typeRegistry, indexFields));
+ long total = ctx.getSearchProcessor().getResultCount();
+ int page1 = ctx.getSearchProcessor().execute().size();
+ qp.setOffset(2);
+ sp = EntityDiscoveryService.createSearchParameters(qp);
+ sp.setQuery("customer*");
+ ctx = new SearchContext(sp, typeRegistry, graph, buildIndexedKeys(typeRegistry, indexFields));
+ int page2 = ctx.getSearchProcessor().execute().size();
+ System.out.println("Pagination: total=" + total + " page1=" + page1 + " page2=" + page2);
+
+ System.out.println();
+ }
+
+ private static void runAggregationIndependenceProbe(AtlasGraph graph, AtlasTypeRegistry typeRegistry,
+ IndexFieldNames indexFields) throws Exception {
+ AtlasGraphIndexClient client = graph.getGraphIndexClient();
+ Set<AtlasEntityType> types = Collections.singleton(typeRegistry.getEntityTypeByName(TYPE_DATASET));
+ Map<String, String> cache = new HashMap<>();
+ cache.put(Constants.ENTITY_TYPE_PROPERTY_KEY, indexFields.typeIndexField);
+ cache.put(Constants.STATE_PROPERTY_KEY, indexFields.stateIndexField);
+
+ AggregationContext ctx = new AggregationContext("customer", null, types,
+ Collections.singleton(Constants.ENTITY_TYPE_PROPERTY_KEY), Collections.emptySet(),
+ cache, true, true);
+
+ Map<String, List<AtlasAggregationEntry>> metrics = client.getAggregatedMetrics(ctx);
+ long typeCount = metrics.getOrDefault(Constants.ENTITY_TYPE_PROPERTY_KEY, Collections.emptyList())
+ .stream().mapToLong(AtlasAggregationEntry::getCount).sum();
+
+ System.out.println("--- Aggregation independence (unaffected by search weights) ---");
+ System.out.println("typeName agg total with query=customer: " + typeCount + " (expect 4 baseline entities)");
+ System.out.println();
+ }
+
+ // -------------------------------------------------------------------------
+ // Payload builders
+ // -------------------------------------------------------------------------
+
+ private static String buildQueryStringSearchPayload(String query, int size) throws Exception {
+ Map<String, Object> body = new LinkedHashMap<>();
+ body.put("query", Collections.singletonMap("query_string",
+ Collections.singletonMap("query", query)));
+ body.put("size", size);
+ body.put("_source", Collections.singletonList("__guid"));
+ return MAPPER.writeValueAsString(body);
+ }
+
+ private static String buildMultiMatchPayload(String queryTerm, Map<String, String> osFields, int size)
+ throws Exception {
+ List<String> fields = new ArrayList<>();
+ fields.add(osFields.get("name") + "^" + WEIGHT_NAME);
+ fields.add(osFields.get("owner") + "^" + WEIGHT_OWNER);
+
+ Map<String, Object> multiMatch = new LinkedHashMap<>();
+ multiMatch.put("query", queryTerm);
+ multiMatch.put("fields", fields);
+ multiMatch.put("type", "best_fields");
+
+ Map<String, Object> bool = new LinkedHashMap<>();
+ bool.put("must", Collections.singletonList(Collections.singletonMap("multi_match", multiMatch)));
+ bool.put("filter", Collections.singletonList(
+ Collections.singletonMap("term", Collections.singletonMap("__state", "ACTIVE"))));
+
+ Map<String, Object> body = new LinkedHashMap<>();
+ body.put("query", Collections.singletonMap("bool", bool));
+ body.put("size", size);
+ body.put("_source", Collections.singletonList("__guid"));
+ return MAPPER.writeValueAsString(body);
+ }
+
+ private static String buildMultiMatchPayloadWithExtraField(String queryTerm, Map<String, String> osFields,
+ String extraField, int extraWeight, int size)
+ throws Exception {
+ List<String> fields = new ArrayList<>();
+ fields.add(osFields.get("name") + "^" + WEIGHT_NAME);
+ fields.add(osFields.get("owner") + "^" + WEIGHT_OWNER);
+ if (extraField != null) {
+ fields.add(extraField + "^" + extraWeight);
+ }
+ Map<String, Object> multiMatch = new LinkedHashMap<>();
+ multiMatch.put("query", queryTerm);
+ multiMatch.put("fields", fields);
+ Map<String, Object> body = new LinkedHashMap<>();
+ body.put("query", Collections.singletonMap("multi_match", multiMatch));
+ body.put("size", size);
+ body.put("_source", Collections.singletonList("__guid"));
+ return MAPPER.writeValueAsString(body);
+ }
+
+ private static String buildInvertedWeightPayload(String queryTerm, Map<String, String> osFields, int size)
+ throws Exception {
+ List<String> fields = new ArrayList<>();
+ fields.add(osFields.get("name") + "^3");
+ fields.add(osFields.get("owner") + "^10");
+ Map<String, Object> multiMatch = new LinkedHashMap<>();
+ multiMatch.put("query", queryTerm);
+ multiMatch.put("fields", fields);
+ Map<String, Object> body = new LinkedHashMap<>();
+ body.put("query", Collections.singletonMap("multi_match", multiMatch));
+ body.put("size", size);
+ body.put("_source", Collections.singletonList("__guid"));
+ return MAPPER.writeValueAsString(body);
+ }
+
+ // -------------------------------------------------------------------------
+ // Fixture
+ // -------------------------------------------------------------------------
+
+ private static Map<String, String> insertBaselineEntities(JanusGraph graph, AtlasTypeRegistry typeRegistry)
+ throws Exception {
+ Map<String, String> guidByLabel = new LinkedHashMap<>();
+ guidByLabel.put(LABEL_A, insertEntity(graph, typeRegistry, "customer", "alice", null));
+ guidByLabel.put(LABEL_B, insertEntity(graph, typeRegistry, "sales", "customer", null));
+ guidByLabel.put(LABEL_C, insertEntity(graph, typeRegistry, "customer sales", "bob", null));
+ guidByLabel.put(LABEL_D, insertEntity(graph, typeRegistry, "other", "other", "customer"));
+ return guidByLabel;
+ }
+
+ private static String insertEntity(JanusGraph graph, AtlasTypeRegistry typeRegistry,
+ String name, String owner, String labels) throws Exception {
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(TYPE_DATASET);
+ String guid = UUID.randomUUID().toString();
+ org.apache.tinkerpop.gremlin.structure.Vertex v = graph.addVertex();
+ v.property(Constants.GUID_PROPERTY_KEY, guid);
+ v.property(Constants.ENTITY_TYPE_PROPERTY_KEY, TYPE_DATASET);
+ v.property(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
+ v.property(entityType.getAttribute("name").getVertexPropertyName(), name);
+ v.property(entityType.getAttribute("owner").getVertexPropertyName(), owner);
+ if (labels != null) {
+ v.property(entityType.getAttribute("labels").getVertexPropertyName(), labels);
+ }
+ return guid;
+ }
+
+ private static IndexFieldNames createSchema(JanusGraph graph, AtlasTypeRegistry typeRegistry) throws Exception {
+ AtlasEntityType assetType = typeRegistry.getEntityTypeByName(TYPE_ASSET);
+ String nameProperty = assetType.getAttribute("name").getVertexPropertyName();
+ String ownerProperty = assetType.getAttribute("owner").getVertexPropertyName();
+ String labelsProperty = assetType.getAttribute("labels").getVertexPropertyName();
+
+ JanusGraphManagement mgmt = graph.openManagement();
+ PropertyKey guidKey = mgmt.makePropertyKey(Constants.GUID_PROPERTY_KEY).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey typeKey = mgmt.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey stateKey = mgmt.makePropertyKey(Constants.STATE_PROPERTY_KEY).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey nameKey = mgmt.makePropertyKey(nameProperty).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey ownerKey = mgmt.makePropertyKey(ownerProperty).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey labelsKey = mgmt.makePropertyKey(labelsProperty).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+
+ mgmt.buildIndex(OpenSearchQuickSearchSmokeSupport.VERTEX_INDEX, Vertex.class)
+ .addKey(guidKey, Mapping.STRING.asParameter())
+ .addKey(typeKey, Mapping.STRING.asParameter())
+ .addKey(stateKey, Mapping.STRING.asParameter())
+ .addKey(nameKey, Mapping.TEXT.asParameter())
+ .addKey(ownerKey, Mapping.STRING.asParameter())
+ .addKey(labelsKey, Mapping.STRING.asParameter())
+ .buildMixedIndex(OpenSearchQuickSearchSmokeSupport.BACKING_INDEX_NAME);
+
+ IndexFieldNames fields = new IndexFieldNames();
+ fields.guidIndexField = guidKey.name();
+ fields.typeIndexField = typeKey.name();
+ fields.stateIndexField = stateKey.name();
+ fields.nameIndexField = nameKey.name();
+ fields.ownerIndexField = ownerKey.name();
+ fields.labelsIndexField = labelsKey.name();
+ mgmt.commit();
+ graph.tx().commit();
+ return fields;
+ }
+
+ private static AtlasTypeRegistry buildTypeRegistry() throws Exception {
+ AtlasTypeRegistry registry = new AtlasTypeRegistry();
+ AtlasEntityDef assetDef = new AtlasEntityDef();
+ assetDef.setName(TYPE_ASSET);
+ assetDef.setAttributeDefs(new ArrayList<>());
+ assetDef.getAttributeDefs().add(attr("name", WEIGHT_NAME));
+ assetDef.getAttributeDefs().add(attr("owner", WEIGHT_OWNER));
+ assetDef.getAttributeDefs().add(attr("labels", 10));
+
+ AtlasEntityDef datasetDef = new AtlasEntityDef();
+ datasetDef.setName(TYPE_DATASET);
+ datasetDef.setSuperTypes(Collections.singleton(TYPE_ASSET));
+
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+ typesDef.getEntityDefs().add(assetDef);
+ typesDef.getEntityDefs().add(datasetDef);
+ registry.updateTypes(typesDef);
+ return registry;
+ }
+
+ private static AtlasAttributeDef attr(String name, int searchWeight) {
+ AtlasAttributeDef a = new AtlasAttributeDef(name, "string");
+ a.setIndexType(AtlasAttributeDef.IndexType.STRING);
+ a.setSearchWeight(searchWeight);
+ return a;
+ }
+
+ private static void wireTypeRegistry(AtlasTypeRegistry registry, IndexFieldNames fields) {
+ wire(registry.getEntityTypeByName(TYPE_ASSET), fields);
+ wire(registry.getEntityTypeByName(TYPE_DATASET), fields);
+ registry.addIndexFieldName(Constants.ENTITY_TYPE_PROPERTY_KEY, fields.typeIndexField);
+ registry.addIndexFieldName(Constants.STATE_PROPERTY_KEY, fields.stateIndexField);
+ }
+
+ private static void wire(AtlasEntityType type, IndexFieldNames fields) {
+ type.getAttribute("name").setIndexFieldName(fields.nameIndexField);
+ type.getAttribute("owner").setIndexFieldName(fields.ownerIndexField);
+ type.getAttribute("labels").setIndexFieldName(fields.labelsIndexField);
+ }
+
+ private static Set<String> buildIndexedKeys(AtlasTypeRegistry registry, IndexFieldNames fields) {
+ Set<String> keys = new LinkedHashSet<>();
+ keys.add(fields.nameIndexField);
+ keys.add(fields.ownerIndexField);
+ keys.add(fields.labelsIndexField);
+ keys.add(fields.typeIndexField);
+ keys.add(fields.stateIndexField);
+ return keys;
+ }
+
+ private static Map<String, String> fetchOpenSearchFieldNames() throws Exception {
+ String mapping = OpenSearchQuickSearchSmokeSupport.httpGet(
+ "/" + OpenSearchQuickSearchSmokeSupport.PHYSICAL_INDEX + "/_mapping");
+ JsonNode props = MAPPER.readTree(mapping)
+ .path(OpenSearchQuickSearchSmokeSupport.PHYSICAL_INDEX)
+ .path("mappings").path("properties");
+ Map<String, String> ret = new HashMap<>();
+ List<String> names = new ArrayList<>();
+ props.fieldNames().forEachRemaining(names::add);
+ for (String key : names) {
+ if (key.contains("name") && !key.contains("type") && !key.contains("owner")) {
+ ret.put("name", key);
+ } else if (key.contains("owner")) {
+ ret.put("owner", key);
+ } else if (key.contains("labels") || key.contains("label")) {
+ ret.put("labels", key);
+ }
+ }
+ return ret;
+ }
+
+ // -------------------------------------------------------------------------
+ // Helpers
+ // -------------------------------------------------------------------------
+
+ private static void enableOpenSearchIndexDebugLogging() {
+ ch.qos.logback.classic.Logger logger = (ch.qos.logback.classic.Logger)
+ LoggerFactory.getLogger("org.janusgraph.diskstorage.opensearch.OpenSearchIndex");
+ logger.setLevel(Level.DEBUG);
+ }
+
+ private static String prettyJson(String json) {
+ try {
+ return MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(MAPPER.readTree(json));
+ } catch (Exception e) {
+ return json;
+ }
+ }
+
+ private static String labelForGuid(String guid, Map<String, String> guidByLabel) {
+ for (Map.Entry<String, String> e : guidByLabel.entrySet()) {
+ if (e.getValue().equals(guid)) {
+ return e.getKey();
+ }
+ }
+ return null;
+ }
+
+ private static String formatHit(List<RankedHit> hits, int index) {
+ if (index >= hits.size()) {
+ return "-";
+ }
+ RankedHit h = hits.get(index);
+ if (Double.isNaN(h.score)) {
+ return h.label;
+ }
+ return h.label + " (" + String.format("%.2f", h.score) + ")";
+ }
+
+ private static boolean sameLabelOrder(List<RankedHit> a, List<RankedHit> b) {
+ List<String> la = a.stream().map(h -> h.label).collect(Collectors.toList());
+ List<String> lb = b.stream().map(h -> h.label).collect(Collectors.toList());
+ return la.equals(lb);
+ }
+
+ private static final class RankedHit {
+ final int rank;
+ final String label;
+ final String id;
+ final double score;
+ final String source;
+
+ RankedHit(int rank, String label, String id, double score, String source) {
+ this.rank = rank;
+ this.label = label;
+ this.id = id;
+ this.score = score;
+ this.source = source;
+ }
+ }
+
+ private static final class IndexFieldNames {
+ private String guidIndexField;
+ private String typeIndexField;
+ private String stateIndexField;
+ private String nameIndexField;
+ private String ownerIndexField;
+ private String labelsIndexField;
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSearchWeightIT.java b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSearchWeightIT.java
new file mode 100644
index 000000000..7f163d39e
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSearchWeightIT.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery.smoke;
+
+import org.apache.atlas.runner.OpenSearchITBase;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertTrue;
+
+/**
+ * Integration test: Atlas search-weight parity via Testcontainers OpenSearch.
+ */
+public class OpenSearchSearchWeightIT extends OpenSearchITBase {
+
+ @Test
+ public void searchWeightValidation() throws Exception {
+ assertTrue(OpenSearchSearchWeightValidationDriver.execute(), "Search-weight validation failed");
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSearchWeightValidationDriver.java b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSearchWeightValidationDriver.java
new file mode 100644
index 000000000..ec55c0112
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSearchWeightValidationDriver.java
@@ -0,0 +1,466 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery.smoke;
+
+import org.apache.atlas.discovery.EntityDiscoveryService;
+import org.apache.atlas.discovery.FreeTextSearchProcessor;
+import org.apache.atlas.discovery.SearchContext;
+import org.apache.atlas.discovery.SearchProcessor;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.AtlasAggregationEntry;
+import org.apache.atlas.model.discovery.QuickSearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.Operator;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AggregationContext;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph;
+import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.util.AtlasRepositoryConfiguration;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.Mapping;
+import org.janusgraph.diskstorage.opensearch.OpenSearchMajorVersion;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+/**
+ * Validates weighted OpenSearch quick-search through the production path
+ * (SolrIndexHelper weight map → applySearchWeight → FreeTextSearchProcessor → quickSearch).
+ *
+ * <pre>
+ * cd repository && mvn test-compile exec:java \
+ * -Dexec.classpathScope=test \
+ * -Dexec.mainClass=org.apache.atlas.discovery.smoke.OpenSearchSearchWeightValidationDriver \
+ * -Drat.skip=true -Dcheckstyle.skip=true -Dsortpom.skip=true -DskipCheck=true
+ * </pre>
+ */
+public final class OpenSearchSearchWeightValidationDriver {
+
+ private static final String TYPE_ASSET = "c554_asset";
+ private static final String TYPE_DATASET = "c554_dataset";
+
+ private static final int WEIGHT_NAME = 10;
+ private static final int WEIGHT_OWNER = 3;
+
+ private static final String LABEL_A = "A";
+ private static final String LABEL_B = "B";
+ private static final String LABEL_C = "C";
+ private static final String LABEL_D = "D";
+
+ private static final Map<String, String> RESULTS = new LinkedHashMap<>();
+ private static Map<String, String> guidByLabel = Collections.emptyMap();
+
+ private OpenSearchSearchWeightValidationDriver() {
+ }
+
+ public static boolean execute() throws Exception {
+ RESULTS.clear();
+
+ OpenSearchQuickSearchSmokeSupport.bootstrapApplicationProperties();
+ OpenSearchQuickSearchSmokeSupport.verifyOpenSearchReachable();
+
+ if (!AtlasRepositoryConfiguration.isFreeTextSearchEnabled()) {
+ throw new IllegalStateException("atlas.search.freetext.enable must be true");
+ }
+
+ OpenSearchQuickSearchSmokeSupport.registerAtlasOpenSearchIndex();
+ OpenSearchQuickSearchSmokeSupport.deletePhysicalIndexIfPresent();
+
+ JanusGraph janusGraph = JanusGraphFactory.open(OpenSearchQuickSearchSmokeSupport.buildJanusGraphConfiguration());
+ AtlasTypeRegistry typeRegistry = buildTypeRegistry();
+ IndexFieldNames indexFields = createSchema(janusGraph, typeRegistry);
+ wireTypeRegistry(typeRegistry, indexFields);
+
+ Map<String, String> entityGuids = insertBaselineEntities(janusGraph, typeRegistry);
+ guidByLabel = entityGuids;
+ AtlasGraph graph = new AtlasJanusGraph(janusGraph);
+ graph.commit();
+ Thread.sleep(1500);
+
+ applySearchWeights(graph, indexFields);
+
+ runValidations(graph, typeRegistry, indexFields, guidByLabel);
+
+ graph.shutdown();
+
+ return RESULTS.values().stream().allMatch("PASS"::equals);
+ }
+
+ public static void main(String[] args) throws Exception {
+ System.out.println("Search-Weight Validation (production path)");
+ boolean allPassed = execute();
+ for (Map.Entry<String, String> entry : RESULTS.entrySet()) {
+ System.out.printf("%-32s %s%n", entry.getKey(), entry.getValue());
+ }
+ System.out.println(allPassed ? "RESULT: PASS" : "RESULT: FAIL");
+ if (!allPassed) {
+ System.exit(1);
+ }
+ }
+
+ private static void runValidations(AtlasGraph graph, AtlasTypeRegistry typeRegistry, IndexFieldNames indexFields,
+ Map<String, String> guidByLabel) throws Exception {
+ record("Weighted ranking name^10 owner^3", () -> {
+ List<String> order = searchOrder(graph, typeRegistry, indexFields, "customer", TYPE_DATASET, 20, 0);
+ assertTrue(order.size() >= 3, "expected at least 3 hits, got " + order);
+ assertTrue(order.indexOf(LABEL_A) < order.indexOf(LABEL_B),
+ "name match (A) should rank above owner match (B): " + order);
+ assertTrue(order.indexOf(LABEL_A) < order.indexOf(LABEL_C),
+ "A should rank above C: " + order);
+ });
+
+ record("Inverted weights change ranking", () -> {
+ List<String> baseline = searchOrder(graph, typeRegistry, indexFields, "customer", TYPE_DATASET, 20, 0);
+ applyInvertedWeights(graph, indexFields);
+ List<String> inverted = searchOrder(graph, typeRegistry, indexFields, "customer", TYPE_DATASET, 20, 0);
+ applySearchWeights(graph, indexFields);
+ assertTrue(!baseline.equals(inverted), "inverting weights should change order: " + baseline + " vs " + inverted);
+ assertTrue(inverted.indexOf(LABEL_B) < inverted.indexOf(LABEL_A),
+ "owner^10 should put B above A: " + inverted);
+ });
+
+ record("Type filter", () -> {
+ List<String> order = searchOrder(graph, typeRegistry, indexFields, "customer", TYPE_DATASET, 10, 0);
+ assertTrue(order.contains(LABEL_A), "type-filtered search should include A");
+ assertFalse(order.contains("unknown"), "unexpected label");
+ });
+
+ record("Property filter owner=customer", () -> {
+ FilterCriteria filter = new FilterCriteria();
+ filter.setAttributeName("owner");
+ filter.setOperator(Operator.EQ);
+ filter.setAttributeValue("customer");
+ List<String> order = searchWithFilter(graph, typeRegistry, indexFields, "customer", TYPE_DATASET, filter, 10, 0);
+ assertTrue(order.size() == 1 && LABEL_B.equals(order.get(0)),
+ "owner=customer should return B only, got " + order);
+ });
+
+ record("Pagination + count", () -> {
+ QuickSearchOutcome page1 = quickSearch(graph, typeRegistry, indexFields,
+ params("customer", TYPE_DATASET, 2, 0, null));
+ QuickSearchOutcome page2 = quickSearch(graph, typeRegistry, indexFields,
+ params("customer", TYPE_DATASET, 2, 2, null));
+ assertTrue(page1.approximateCount >= 3, "total count=" + page1.approximateCount);
+ assertTrue(page1.resultSize <= 2, "page1 size=" + page1.resultSize);
+ assertTrue(page2.resultSize >= 1, "page2 size=" + page2.resultSize);
+ });
+
+ record("Empty weight map fallback", () -> {
+ graph.getGraphIndexClient().applySearchWeight(Constants.VERTEX_INDEX, Collections.emptyMap());
+ List<String> order = searchOrder(graph, typeRegistry, indexFields, "customer", TYPE_DATASET, 20, 0);
+ applySearchWeights(graph, indexFields);
+ assertTrue(order.size() >= 3, "fallback should still return hits, got " + order);
+ });
+
+ record("Aggregation independence", () -> {
+ AtlasGraphIndexClient client = graph.getGraphIndexClient();
+ Map<String, String> cache = new HashMap<>();
+ cache.put(Constants.ENTITY_TYPE_PROPERTY_KEY, indexFields.typeIndexField);
+ cache.put(Constants.STATE_PROPERTY_KEY, indexFields.stateIndexField);
+ AggregationContext ctx = new AggregationContext("customer", null,
+ Collections.singleton(typeRegistry.getEntityTypeByName(TYPE_DATASET)),
+ Collections.singleton(Constants.ENTITY_TYPE_PROPERTY_KEY),
+ Collections.emptySet(), cache, true, true);
+ Map<String, List<AtlasAggregationEntry>> metrics = client.getAggregatedMetrics(ctx);
+ long typeCount = metrics.getOrDefault(Constants.ENTITY_TYPE_PROPERTY_KEY, Collections.emptyList())
+ .stream().mapToLong(AtlasAggregationEntry::getCount).sum();
+ assertTrue(typeCount >= 4, "aggregation count=" + typeCount);
+ });
+
+ record("Uses FreeTextSearchProcessor", () -> {
+ SearchContext ctx = buildSearchContext(graph, typeRegistry, indexFields,
+ params("customer", TYPE_DATASET, 10, 0, null));
+ assertTrue(ctx.getSearchProcessor() instanceof FreeTextSearchProcessor,
+ "processor=" + ctx.getSearchProcessor().getClass().getSimpleName());
+ });
+ }
+
+ private static List<String> searchOrder(AtlasGraph graph, AtlasTypeRegistry typeRegistry,
+ IndexFieldNames indexFields, String query, String typeName,
+ int limit, int offset) throws AtlasBaseException {
+ QuickSearchOutcome out = quickSearch(graph, typeRegistry, indexFields, params(query, typeName, limit, offset, null));
+ List<String> labels = new ArrayList<>();
+ for (AtlasVertex v : out.vertices) {
+ String guid = v.getProperty(Constants.GUID_PROPERTY_KEY, String.class);
+ labels.add(labelForGuid(guid));
+ }
+ return labels;
+ }
+
+ private static List<String> searchWithFilter(AtlasGraph graph, AtlasTypeRegistry typeRegistry,
+ IndexFieldNames indexFields, String query, String typeName,
+ FilterCriteria filter, int limit, int offset) throws AtlasBaseException {
+ QuickSearchOutcome out = quickSearch(graph, typeRegistry, indexFields,
+ params(query, typeName, limit, offset, filter));
+ List<String> labels = new ArrayList<>();
+ for (AtlasVertex v : out.vertices) {
+ String guid = v.getProperty(Constants.GUID_PROPERTY_KEY, String.class);
+ labels.add(labelForGuid(guid));
+ }
+ return labels;
+ }
+
+ private static QuickSearchOutcome quickSearch(AtlasGraph graph, AtlasTypeRegistry typeRegistry,
+ IndexFieldNames indexFields,
+ QuickSearchParameters quickSearchParameters) throws AtlasBaseException {
+ String query = quickSearchParameters.getQuery();
+ if (org.apache.commons.lang3.StringUtils.isNotEmpty(query)
+ && !org.apache.atlas.type.AtlasStructType.AtlasAttribute.hastokenizeChar(query)) {
+ query = query + "*";
+ }
+ quickSearchParameters.setQuery(query);
+
+ SearchParameters searchParameters = EntityDiscoveryService.createSearchParameters(quickSearchParameters);
+ SearchContext searchContext = buildSearchContext(graph, typeRegistry, indexFields, quickSearchParameters);
+
+ SearchProcessor processor = searchContext.getSearchProcessor();
+ List<AtlasVertex> vertices = processor.execute();
+ long count = processor.getResultCount();
+
+ QuickSearchOutcome outcome = new QuickSearchOutcome();
+ outcome.approximateCount = count;
+ outcome.resultSize = vertices.size();
+ outcome.vertices = vertices;
+ return outcome;
+ }
+
+ private static SearchContext buildSearchContext(AtlasGraph graph, AtlasTypeRegistry typeRegistry,
+ IndexFieldNames indexFields,
+ QuickSearchParameters quickSearchParameters) throws AtlasBaseException {
+ SearchParameters searchParameters = EntityDiscoveryService.createSearchParameters(quickSearchParameters);
+ return new SearchContext(searchParameters, typeRegistry, graph, buildIndexedKeys(typeRegistry, indexFields));
+ }
+
+ private static QuickSearchParameters params(String query, String typeName, int limit, int offset,
+ FilterCriteria entityFilter) {
+ QuickSearchParameters p = new QuickSearchParameters();
+ p.setQuery(query);
+ p.setTypeName(typeName);
+ p.setLimit(limit);
+ p.setOffset(offset);
+ p.setExcludeDeletedEntities(true);
+ p.setIncludeSubTypes(true);
+ p.setEntityFilters(entityFilter);
+ return p;
+ }
+
+ private static void applySearchWeights(AtlasGraph graph, IndexFieldNames indexFields) throws org.apache.atlas.AtlasException {
+ Map<String, Integer> weights = new LinkedHashMap<>();
+ weights.put(indexFields.nameIndexField, WEIGHT_NAME);
+ weights.put(indexFields.ownerIndexField, WEIGHT_OWNER);
+ weights.put(indexFields.labelsIndexField, 10);
+ graph.getGraphIndexClient().applySearchWeight(Constants.VERTEX_INDEX, weights);
+ }
+
+ private static void applyInvertedWeights(AtlasGraph graph, IndexFieldNames indexFields) throws org.apache.atlas.AtlasException {
+ Map<String, Integer> weights = new LinkedHashMap<>();
+ weights.put(indexFields.nameIndexField, WEIGHT_OWNER);
+ weights.put(indexFields.ownerIndexField, WEIGHT_NAME);
+ weights.put(indexFields.labelsIndexField, 10);
+ graph.getGraphIndexClient().applySearchWeight(Constants.VERTEX_INDEX, weights);
+ }
+
+ private static Map<String, String> insertBaselineEntities(JanusGraph graph, AtlasTypeRegistry typeRegistry)
+ throws Exception {
+ Map<String, String> guidByLabel = new LinkedHashMap<>();
+ guidByLabel.put(LABEL_A, insertEntity(graph, typeRegistry, "customer", "alice", null));
+ guidByLabel.put(LABEL_B, insertEntity(graph, typeRegistry, "sales", "customer", null));
+ guidByLabel.put(LABEL_C, insertEntity(graph, typeRegistry, "customer sales", "bob", null));
+ guidByLabel.put(LABEL_D, insertEntity(graph, typeRegistry, "other", "other", "customer"));
+ return guidByLabel;
+ }
+
+ private static String insertEntity(JanusGraph graph, AtlasTypeRegistry typeRegistry,
+ String name, String owner, String labels) throws Exception {
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(TYPE_DATASET);
+ String guid = UUID.randomUUID().toString();
+ Vertex v = graph.addVertex();
+ v.property(Constants.GUID_PROPERTY_KEY, guid);
+ v.property(Constants.ENTITY_TYPE_PROPERTY_KEY, TYPE_DATASET);
+ v.property(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
+ v.property(entityType.getAttribute("name").getVertexPropertyName(), name);
+ v.property(entityType.getAttribute("owner").getVertexPropertyName(), owner);
+ if (labels != null) {
+ v.property(entityType.getAttribute("labels").getVertexPropertyName(), labels);
+ }
+ return guid;
+ }
+
+ private static IndexFieldNames createSchema(JanusGraph graph, AtlasTypeRegistry typeRegistry) throws Exception {
+ AtlasEntityType assetType = typeRegistry.getEntityTypeByName(TYPE_ASSET);
+ String nameProperty = assetType.getAttribute("name").getVertexPropertyName();
+ String ownerProperty = assetType.getAttribute("owner").getVertexPropertyName();
+ String labelsProperty = assetType.getAttribute("labels").getVertexPropertyName();
+
+ JanusGraphManagement mgmt = graph.openManagement();
+ PropertyKey guidKey = mgmt.makePropertyKey(Constants.GUID_PROPERTY_KEY).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey typeKey = mgmt.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey stateKey = mgmt.makePropertyKey(Constants.STATE_PROPERTY_KEY).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey nameKey = mgmt.makePropertyKey(nameProperty).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey ownerKey = mgmt.makePropertyKey(ownerProperty).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey labelsKey = mgmt.makePropertyKey(labelsProperty).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+
+ mgmt.buildIndex(OpenSearchQuickSearchSmokeSupport.VERTEX_INDEX, Vertex.class)
+ .addKey(guidKey, Mapping.STRING.asParameter())
+ .addKey(typeKey, Mapping.STRING.asParameter())
+ .addKey(stateKey, Mapping.STRING.asParameter())
+ .addKey(nameKey, Mapping.TEXT.asParameter())
+ .addKey(ownerKey, Mapping.STRING.asParameter())
+ .addKey(labelsKey, Mapping.STRING.asParameter())
+ .buildMixedIndex(OpenSearchQuickSearchSmokeSupport.BACKING_INDEX_NAME);
+
+ IndexFieldNames fields = new IndexFieldNames();
+ fields.guidIndexField = guidKey.name();
+ fields.typeIndexField = typeKey.name();
+ fields.stateIndexField = stateKey.name();
+ fields.nameIndexField = nameKey.name();
+ fields.ownerIndexField = ownerKey.name();
+ fields.labelsIndexField = labelsKey.name();
+ mgmt.commit();
+ graph.tx().commit();
+ return fields;
+ }
+
+ private static AtlasTypeRegistry buildTypeRegistry() throws Exception {
+ AtlasTypeRegistry registry = new AtlasTypeRegistry();
+ AtlasEntityDef assetDef = new AtlasEntityDef();
+ assetDef.setName(TYPE_ASSET);
+ assetDef.setAttributeDefs(new ArrayList<>());
+ assetDef.getAttributeDefs().add(attr("name", WEIGHT_NAME));
+ assetDef.getAttributeDefs().add(attr("owner", WEIGHT_OWNER));
+ assetDef.getAttributeDefs().add(attr("labels", 10));
+
+ AtlasEntityDef datasetDef = new AtlasEntityDef();
+ datasetDef.setName(TYPE_DATASET);
+ datasetDef.setSuperTypes(Collections.singleton(TYPE_ASSET));
+
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+ typesDef.getEntityDefs().add(assetDef);
+ typesDef.getEntityDefs().add(datasetDef);
+ registry.updateTypes(typesDef);
+ return registry;
+ }
+
+ private static AtlasAttributeDef attr(String name, int searchWeight) {
+ AtlasAttributeDef a = new AtlasAttributeDef(name, "string");
+ a.setIndexType(AtlasAttributeDef.IndexType.STRING);
+ a.setSearchWeight(searchWeight);
+ return a;
+ }
+
+ private static void wireTypeRegistry(AtlasTypeRegistry registry, IndexFieldNames fields) {
+ wire(registry.getEntityTypeByName(TYPE_ASSET), fields);
+ wire(registry.getEntityTypeByName(TYPE_DATASET), fields);
+ registry.addIndexFieldName(Constants.ENTITY_TYPE_PROPERTY_KEY, fields.typeIndexField);
+ registry.addIndexFieldName(Constants.STATE_PROPERTY_KEY, fields.stateIndexField);
+ }
+
+ private static void wire(AtlasEntityType type, IndexFieldNames fields) {
+ type.getAttribute("name").setIndexFieldName(fields.nameIndexField);
+ type.getAttribute("owner").setIndexFieldName(fields.ownerIndexField);
+ type.getAttribute("labels").setIndexFieldName(fields.labelsIndexField);
+ }
+
+ private static Set<String> buildIndexedKeys(AtlasTypeRegistry registry, IndexFieldNames fields) {
+ Set<String> keys = new LinkedHashSet<>();
+ keys.add(fields.nameIndexField);
+ keys.add(fields.ownerIndexField);
+ keys.add(fields.labelsIndexField);
+ keys.add(fields.typeIndexField);
+ keys.add(fields.stateIndexField);
+ return keys;
+ }
+
+ private static String labelForGuid(String guid) {
+ for (Map.Entry<String, String> e : guidByLabel.entrySet()) {
+ if (e.getValue().equals(guid)) {
+ return e.getKey();
+ }
+ }
+ return guid != null && guid.length() > 8 ? guid.substring(0, 8) + "…" : guid;
+ }
+
+ private static void record(String name, ValidationRunnable runnable) {
+ try {
+ runnable.run();
+ RESULTS.put(name, "PASS");
+ } catch (AssertionError | Exception e) {
+ RESULTS.put(name, "FAIL: " + e.getMessage());
+ }
+ }
+
+ private static void assertTrue(boolean condition, String message) {
+ if (!condition) {
+ throw new AssertionError(message);
+ }
+ }
+
+ private static void assertFalse(boolean condition, String message) {
+ if (condition) {
+ throw new AssertionError(message);
+ }
+ }
+
+ @FunctionalInterface
+ private interface ValidationRunnable {
+ void run() throws Exception;
+ }
+
+ private static final class QuickSearchOutcome {
+ long approximateCount;
+ int resultSize;
+ List<AtlasVertex> vertices;
+ }
+
+ private static final class IndexFieldNames {
+ String guidIndexField;
+ String typeIndexField;
+ String stateIndexField;
+ String nameIndexField;
+ String ownerIndexField;
+ String labelsIndexField;
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSuggestionsIT.java b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSuggestionsIT.java
new file mode 100644
index 000000000..d77f97f84
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSuggestionsIT.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery.smoke;
+
+import org.apache.atlas.runner.OpenSearchITBase;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertTrue;
+
+/**
+ * Integration test: Atlas suggestions via Testcontainers OpenSearch.
+ */
+public class OpenSearchSuggestionsIT extends OpenSearchITBase {
+
+ @Test
+ public void suggestionsValidation() throws Exception {
+ assertTrue(OpenSearchSuggestionsValidationDriver.execute(), "Suggestions validation failed");
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSuggestionsValidationDriver.java b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSuggestionsValidationDriver.java
new file mode 100644
index 000000000..d5de2f922
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/smoke/OpenSearchSuggestionsValidationDriver.java
@@ -0,0 +1,416 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery.smoke;
+
+import org.apache.atlas.discovery.SuggestionsProviderImpl;
+import org.apache.atlas.model.discovery.AtlasSuggestionsResult;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.Mapping;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Validates Atlas suggestions through {@link SuggestionsProviderImpl} and
+ * {@link AtlasJanusGraphIndexClient#getSuggestions(String, String)} against a real OpenSearch server. The server
+ * version is configurable (system property {@code opensearch.docker.version} / {@code opensearch.docker.image})
+ * — the dedicated {@code opensearch-it} CI job validates OpenSearch 3.7 and 3.8. OpenSearch 2.x is NOT currently
+ * part of the CI matrix; this driver has not been verified against 2.x in CI.
+ *
+ * <pre>
+ * cd repository && mvn test-compile exec:java \
+ * -Dexec.classpathScope=test \
+ * -Dexec.mainClass=org.apache.atlas.discovery.smoke.OpenSearchSuggestionsValidationDriver \
+ * -Drat.skip=true -Dcheckstyle.skip=true -Dsortpom.skip=true -DskipCheck=true
+ * </pre>
+ */
+public final class OpenSearchSuggestionsValidationDriver {
+
+ private static final String TYPE_DATASET = "c53_dataset";
+ private static final String TYPE_TABLE = "c53_table";
+ private static final String TYPE_ASSET = "c53_asset";
+
+ private static final Map<String, String> RESULTS = new LinkedHashMap<>();
+
+ private static final int ATLAS_SUGGESTION_LIMIT = 5;
+
+ private OpenSearchSuggestionsValidationDriver() {
+ }
+
+ public static boolean execute() throws Exception {
+ RESULTS.clear();
+
+ OpenSearchQuickSearchSmokeSupport.bootstrapApplicationProperties();
+ OpenSearchQuickSearchSmokeSupport.verifyOpenSearchReachable();
+ OpenSearchQuickSearchSmokeSupport.registerAtlasOpenSearchIndex();
+ OpenSearchQuickSearchSmokeSupport.deletePhysicalIndexIfPresent();
+
+ JanusGraph janusGraph = JanusGraphFactory.open(OpenSearchQuickSearchSmokeSupport.buildJanusGraphConfiguration());
+ AtlasTypeRegistry typeRegistry = buildTypeRegistry(new IndexFieldNames());
+ IndexFieldNames indexFields = createSchema(janusGraph, typeRegistry);
+ wireTypeRegistry(typeRegistry, indexFields);
+
+ AtlasGraph graph = new AtlasJanusGraph(janusGraph);
+ insertTestEntities(graph, typeRegistry);
+ graph.commit();
+
+ Thread.sleep(1500);
+
+ AtlasGraphIndexClient indexClient = graph.getGraphIndexClient();
+ SuggestionsProviderImpl suggestionsProvider = new SuggestionsProviderImpl(graph, typeRegistry);
+
+ indexClient.applySuggestionFields(Constants.VERTEX_INDEX,
+ Arrays.asList(indexFields.ownerIndexField, indexFields.nameIndexField));
+
+ runValidations(indexClient, suggestionsProvider, indexFields);
+
+ graph.shutdown();
+
+ return RESULTS.values().stream().allMatch("PASS"::equals);
+ }
+
+ public static void main(String[] args) throws Exception {
+ System.out.println("Atlas Suggestions Validation");
+ boolean allPassed = execute();
+ for (Map.Entry<String, String> entry : RESULTS.entrySet()) {
+ System.out.printf("%-28s %s%n", entry.getKey(), entry.getValue());
+ }
+ System.out.println(allPassed ? "RESULT: PASS" : "RESULT: FAIL");
+ if (!allPassed) {
+ System.exit(1);
+ }
+ }
+
+ private static void runValidations(AtlasGraphIndexClient indexClient, SuggestionsProviderImpl suggestionsProvider,
+ IndexFieldNames indexFields) throws Exception {
+ record("Basic prefix", () -> {
+ List<String> suggestions = indexClient.getSuggestions("team", indexFields.ownerIndexField);
+ assertTrue(!suggestions.isEmpty(), "expected suggestions for prefix team");
+ assertTrue(suggestions.stream().anyMatch(s -> s.startsWith("team-")), "expected team-* terms");
+ });
+
+ record("No matching prefix", () -> {
+ List<String> suggestions = indexClient.getSuggestions("zzz-no-match", indexFields.ownerIndexField);
+ assertTrue(suggestions.isEmpty(), "expected empty list, got " + suggestions);
+ });
+
+ record("Multiple matching terms", () -> {
+ List<String> suggestions = indexClient.getSuggestions("team", indexFields.ownerIndexField);
+ assertTrue(suggestions.size() >= 3, "expected at least 3 team-* terms, got " + suggestions.size());
+ });
+
+ record("Frequency ordering", () -> {
+ List<String> suggestions = indexClient.getSuggestions("team", indexFields.ownerIndexField);
+ assertTrue("team-alpha".equals(suggestions.get(0)),
+ "team-alpha (3 docs) should rank first, got " + suggestions);
+ });
+
+ record("Maximum results", () -> {
+ List<String> suggestions = indexClient.getSuggestions("rank", indexFields.ownerIndexField);
+ assertEquals(suggestions.size(), ATLAS_SUGGESTION_LIMIT,
+ "expected max " + ATLAS_SUGGESTION_LIMIT + " suggestions");
+ });
+
+ record("Suggestion field", () -> {
+ List<String> ownerSuggestions = indexClient.getSuggestions("team", indexFields.ownerIndexField);
+ List<String> nameSuggestions = indexClient.getSuggestions("Alice", indexFields.nameIndexField);
+ assertTrue(!ownerSuggestions.isEmpty(), "owner field should return suggestions");
+ assertTrue(!nameSuggestions.isEmpty(), "name field should return suggestions");
+ assertTrue(ownerSuggestions.stream().noneMatch(s -> s.startsWith("Alice")),
+ "owner suggestions should not contain name values");
+ });
+
+ record("Multiple entity types", () -> {
+ List<String> suggestions = indexClient.getSuggestions("team", indexFields.ownerIndexField);
+ assertTrue(suggestions.size() >= 3,
+ "team-* owners should appear across dataset and table entities, got " + suggestions);
+ });
+
+ record("Case behavior", () -> {
+ List<String> lower = indexClient.getSuggestions("team", indexFields.ownerIndexField);
+ List<String> upper = indexClient.getSuggestions("Team", indexFields.ownerIndexField);
+ assertTrue(!lower.isEmpty(), "lowercase prefix should match keyword owner values");
+ // Keyword fields are case-sensitive; Team-* owner exists but team-* prefix should not match it.
+ assertTrue(upper.stream().noneMatch("team-alpha"::equals),
+ "uppercase prefix should not return lowercase-only values on keyword field");
+ });
+
+ record("High-weight suggestion field", () -> {
+ List<String> suggestions = indexClient.getSuggestions("team", null);
+ assertTrue(!suggestions.isEmpty(), "configured suggestion fields should return results without explicit field");
+ assertTrue(suggestions.stream().anyMatch(s -> s.startsWith("team-")),
+ "should include owner-field team-* terms, got " + suggestions);
+ });
+
+ record("Explicit field only", () -> {
+ List<String> ownerOnly = indexClient.getSuggestions("Alice", indexFields.ownerIndexField);
+ List<String> nameOnly = indexClient.getSuggestions("Alice", indexFields.nameIndexField);
+ assertTrue(ownerOnly.isEmpty(), "owner field should not match Alice name values");
+ assertTrue(!nameOnly.isEmpty(), "name field should match Alice values");
+ });
+
+ record("Cross-field frequency merge", () -> {
+ List<String> suggestions = indexClient.getSuggestions("customer", null);
+ assertTrue("customer".equals(suggestions.get(0)),
+ "merged customer frequency should rank first, got " + suggestions);
+ assertTrue(suggestions.contains("customer_data"),
+ "customer_data should appear in merged suggestions");
+ });
+
+ record("Deduplication", () -> {
+ List<String> suggestions = indexClient.getSuggestions("customer", null);
+ long customerCount = suggestions.stream().filter("customer"::equals).count();
+ assertEquals((int) customerCount, 1, "customer should appear once after merge");
+ });
+
+ record("Special character prefixes", () -> {
+ for (String prefix : Arrays.asList("cust-", "cust_", "cust.", "cust+", "cust*", "cust?", "cust(", "cust[", "cust\\")) {
+ List<String> suggestions = indexClient.getSuggestions(prefix, indexFields.ownerIndexField);
+ assertTrue(suggestions != null, "prefix " + prefix + " should not cause errors");
+ }
+ List<String> dashMatch = indexClient.getSuggestions("cust-", indexFields.ownerIndexField);
+ assertTrue(dashMatch.stream().anyMatch(s -> s.equals("cust-special")),
+ "cust- prefix should match cust-special, got " + dashMatch);
+ });
+
+ record("Exclude deleted entities", () -> {
+ List<String> suggestions = indexClient.getSuggestions("deleted-team", indexFields.ownerIndexField);
+ assertTrue(suggestions.isEmpty(),
+ "DELETED entity terms should be excluded, got " + suggestions);
+ });
+
+ record("Single request latency", () -> {
+ long start = System.nanoTime();
+ List<String> suggestions = indexClient.getSuggestions("team", null);
+ long elapsedMs = (System.nanoTime() - start) / 1_000_000L;
+ assertTrue(!suggestions.isEmpty(), "multi-field suggestions should return results");
+ System.out.printf(" [perf] multi-field suggestions (2 fields, 1 request): %d ms, %d results%n",
+ elapsedMs, suggestions.size());
+ });
+
+ record("Low-weight field", () -> {
+ List<String> suggestions = indexClient.getSuggestions("c53", null);
+ assertTrue(suggestions.isEmpty(),
+ "typeName values should be excluded from in-memory suggestion fields, got " + suggestions);
+ });
+
+ record("Existing index", () -> {
+ AtlasSuggestionsResult result = suggestionsProvider.getSuggestions("team", "owner");
+ assertTrue(result.getSuggestions() != null && !result.getSuggestions().isEmpty(),
+ "SuggestionsProvider path should work on existing index");
+ });
+
+ record("Provider API shape", () -> {
+ AtlasSuggestionsResult result = suggestionsProvider.getSuggestions("team", "owner");
+ assertTrue("team".equals(result.getPrefixString()), "prefixString preserved");
+ assertTrue("owner".equals(result.getFieldName()), "fieldName preserved");
+ assertTrue(result.getSuggestions().size() <= ATLAS_SUGGESTION_LIMIT,
+ "response respects Atlas suggestion limit");
+ });
+ }
+
+ // -------------------------------------------------------------------------
+ // Fixture setup (STRING mapping on name for Solr parity)
+ // -------------------------------------------------------------------------
+
+ private static IndexFieldNames createSchema(JanusGraph graph, AtlasTypeRegistry typeRegistry) throws Exception {
+ AtlasEntityType assetType = typeRegistry.getEntityTypeByName(TYPE_ASSET);
+ String nameProperty = assetType.getAttribute("name").getVertexPropertyName();
+ String ownerProperty = assetType.getAttribute("owner").getVertexPropertyName();
+
+ JanusGraphManagement mgmt = graph.openManagement();
+
+ PropertyKey guidKey = mgmt.makePropertyKey(Constants.GUID_PROPERTY_KEY).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey typeKey = mgmt.makePropertyKey(Constants.ENTITY_TYPE_PROPERTY_KEY).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey stateKey = mgmt.makePropertyKey(Constants.STATE_PROPERTY_KEY).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey nameKey = mgmt.makePropertyKey(nameProperty).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+ PropertyKey ownerKey = mgmt.makePropertyKey(ownerProperty).dataType(String.class)
+ .cardinality(org.janusgraph.core.Cardinality.SINGLE).make();
+
+ mgmt.buildIndex(OpenSearchQuickSearchSmokeSupport.VERTEX_INDEX, Vertex.class)
+ .addKey(guidKey, Mapping.STRING.asParameter())
+ .addKey(typeKey, Mapping.STRING.asParameter())
+ .addKey(stateKey, Mapping.STRING.asParameter())
+ .addKey(nameKey, Mapping.STRING.asParameter())
+ .addKey(ownerKey, Mapping.STRING.asParameter())
+ .buildMixedIndex(OpenSearchQuickSearchSmokeSupport.BACKING_INDEX_NAME);
+
+ IndexFieldNames fields = new IndexFieldNames();
+ fields.guidIndexField = guidKey.name();
+ fields.typeIndexField = typeKey.name();
+ fields.stateIndexField = stateKey.name();
+ fields.nameIndexField = nameKey.name();
+ fields.ownerIndexField = ownerKey.name();
+
+ mgmt.commit();
+ graph.tx().commit();
+ return fields;
+ }
+
+ private static AtlasTypeRegistry buildTypeRegistry(IndexFieldNames indexFields) throws Exception {
+ AtlasTypeRegistry registry = new AtlasTypeRegistry();
+
+ AtlasEntityDef assetDef = entityDef(TYPE_ASSET, Collections.emptySet(), true);
+ AtlasEntityDef datasetDef = entityDef(TYPE_DATASET, Collections.singleton(TYPE_ASSET), false);
+ AtlasEntityDef tableDef = entityDef(TYPE_TABLE, Collections.singleton(TYPE_ASSET), false);
+
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+ typesDef.getEntityDefs().add(assetDef);
+ typesDef.getEntityDefs().add(datasetDef);
+ typesDef.getEntityDefs().add(tableDef);
+ registry.updateTypes(typesDef);
+
+ return registry;
+ }
+
+ private static void wireTypeRegistry(AtlasTypeRegistry registry, IndexFieldNames indexFields) {
+ wireType(registry.getEntityTypeByName(TYPE_ASSET), indexFields);
+ wireType(registry.getEntityTypeByName(TYPE_DATASET), indexFields);
+ wireType(registry.getEntityTypeByName(TYPE_TABLE), indexFields);
+
+ registry.addIndexFieldName("owner", indexFields.ownerIndexField);
+ registry.addIndexFieldName("name", indexFields.nameIndexField);
+ }
+
+ private static void wireType(AtlasEntityType type, IndexFieldNames indexFields) {
+ type.getAttribute("name").setIndexFieldName(indexFields.nameIndexField);
+ type.getAttribute("owner").setIndexFieldName(indexFields.ownerIndexField);
+ }
+
+ private static AtlasEntityDef entityDef(String typeName, java.util.Set<String> superTypes, boolean includeIndexedAttrs) {
+ AtlasEntityDef def = new AtlasEntityDef();
+ def.setName(typeName);
+ def.setSuperTypes(superTypes);
+
+ if (includeIndexedAttrs) {
+ List<AtlasAttributeDef> attrs = new ArrayList<>();
+ attrs.add(indexedAttr("name"));
+ attrs.add(indexedAttr("owner"));
+ def.setAttributeDefs(attrs);
+ }
+ return def;
+ }
+
+ private static AtlasAttributeDef indexedAttr(String name) {
+ AtlasAttributeDef attr = new AtlasAttributeDef(name, "string");
+ attr.setIndexType(AtlasAttributeDef.IndexType.STRING);
+ return attr;
+ }
+
+ private static void insertTestEntities(AtlasGraph graph, AtlasTypeRegistry typeRegistry) throws Exception {
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Alice Atlas World", "team-alpha");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Bob Atlas Platform", "team-beta");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Charlie Atlas Marketing", "team-alpha");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "Delta Atlas Ops", "team-alpha");
+ insertEntity(graph, typeRegistry, TYPE_TABLE, "Atlas Report Summary", "team-gamma");
+ insertEntity(graph, typeRegistry, TYPE_TABLE, "Sales Atlas Quarterly", "team-delta");
+ insertEntity(graph, typeRegistry, TYPE_TABLE, "Team-Omega Special", "Team-Omega");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "merge-foo-1", "customer");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "merge-foo-2", "customer");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "merge-foo-3", "customer");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "customer", "merge-owner-1");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "customer", "merge-owner-2");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "customer_data report", "customer_data");
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "special chars", "cust-special");
+
+ insertEntity(graph, typeRegistry, TYPE_DATASET, "deleted entity", "deleted-team-x",
+ AtlasEntity.Status.DELETED);
+
+ for (int i = 1; i <= 9; i++) {
+ insertEntity(graph, typeRegistry, TYPE_DATASET,
+ "Rank Entity " + i, String.format("rank-%02d", i));
+ }
+ }
+
+ private static void insertEntity(AtlasGraph graph, AtlasTypeRegistry typeRegistry, String typeName,
+ String name, String owner) throws Exception {
+ insertEntity(graph, typeRegistry, typeName, name, owner, AtlasEntity.Status.ACTIVE);
+ }
+
+ private static void insertEntity(AtlasGraph graph, AtlasTypeRegistry typeRegistry, String typeName,
+ String name, String owner, AtlasEntity.Status status) throws Exception {
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
+ AtlasVertex vertex = graph.addVertex();
+ vertex.setProperty(Constants.GUID_PROPERTY_KEY, UUID.randomUUID().toString());
+ vertex.setProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
+ vertex.setProperty(Constants.STATE_PROPERTY_KEY, status.name());
+ vertex.setProperty(entityType.getAttribute("name").getVertexPropertyName(), name);
+ vertex.setProperty(entityType.getAttribute("owner").getVertexPropertyName(), owner);
+ }
+
+ // -------------------------------------------------------------------------
+ // Test harness helpers
+ // -------------------------------------------------------------------------
+
+ private interface ValidationStep {
+ void run() throws Exception;
+ }
+
+ private static void record(String name, ValidationStep step) {
+ try {
+ step.run();
+ RESULTS.put(name, "PASS");
+ } catch (AssertionError | Exception e) {
+ RESULTS.put(name, "FAIL (" + e.getMessage() + ")");
+ }
+ }
+
+ private static void assertTrue(boolean condition, String message) {
+ if (!condition) {
+ throw new AssertionError(message);
+ }
+ }
+
+ private static void assertEquals(int actual, int expected, String label) {
+ if (actual != expected) {
+ throw new AssertionError(label + ": expected " + expected + " but was " + actual);
+ }
+ }
+
+ private static final class IndexFieldNames {
+ private String guidIndexField;
+ private String typeIndexField;
+ private String stateIndexField;
+ private String nameIndexField;
+ private String ownerIndexField;
+ }
+}
diff --git a/repository/src/test/resources/atlas-opensearch-c5-application.properties b/repository/src/test/resources/atlas-opensearch-c5-application.properties
new file mode 100644
index 000000000..052c3d250
--- /dev/null
+++ b/repository/src/test/resources/atlas-opensearch-c5-application.properties
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+atlas.graph.storage.backend=inmemory
+atlas.graph.index.search.backend=opensearch
+atlas.graph.index.search.hostname=localhost
+atlas.graph.index.search.port=9200
+atlas.graph.index.search.index-name=c5janus
+atlas.graph.index.search.max-result-set-size=500
+atlas.graph.index.recovery.enable=false
+atlas.search.freetext.enable=true
+atlas.search.fulltext.enable=false
diff --git a/test-tools/src/main/java/org/apache/atlas/runner/OpenSearchITBase.java b/test-tools/src/main/java/org/apache/atlas/runner/OpenSearchITBase.java
new file mode 100644
index 000000000..a860df522
--- /dev/null
+++ b/test-tools/src/main/java/org/apache/atlas/runner/OpenSearchITBase.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.runner;
+
+import org.testng.SkipException;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+
+/**
+ * TestNG base class that provisions OpenSearch via Testcontainers before each test class.
+ */
+public abstract class OpenSearchITBase {
+ @BeforeClass(alwaysRun = true)
+ public void startOpenSearchContainer() {
+ if (!OpenSearchTestContainerRunner.isDockerAvailable()) {
+ throw new SkipException("Docker is not available; skipping OpenSearch integration test");
+ }
+ OpenSearchTestContainerRunner.start();
+ }
+
+ @AfterClass(alwaysRun = true)
+ public void stopOpenSearchContainer() {
+ OpenSearchTestContainerRunner.stop();
+ }
+}
diff --git a/test-tools/src/main/java/org/apache/atlas/runner/OpenSearchTestContainerRunner.java b/test-tools/src/main/java/org/apache/atlas/runner/OpenSearchTestContainerRunner.java
new file mode 100644
index 000000000..f7c7ddece
--- /dev/null
+++ b/test-tools/src/main/java/org/apache/atlas/runner/OpenSearchTestContainerRunner.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.runner;
+
+import org.testcontainers.DockerClientFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.DockerImageName;
+
+import java.time.Duration;
+
+/**
+ * Starts a single-node OpenSearch cluster via Testcontainers for integration tests.
+ * <p>
+ * The container image is configurable so the same suite can be validated against multiple OpenSearch server
+ * versions (co-primary targets 3.7/3.8 and the secondary 2.x compatibility target) without code changes:
+ * <ul>
+ * <li>{@code -Dopensearch.docker.image=opensearchproject/opensearch:3.8.0} — full image override, or</li>
+ * <li>{@code -Dopensearch.docker.version=3.8.0} — version tag override for the default image repository.</li>
+ * </ul>
+ * The default remains {@value #DEFAULT_VERSION} so existing 3.7 coverage is preserved when nothing is set.
+ */
+public final class OpenSearchTestContainerRunner {
+ public static final String IMAGE_PROPERTY = "opensearch.docker.image";
+ public static final String VERSION_PROPERTY = "opensearch.docker.version";
+ public static final String DEFAULT_REPO = "opensearchproject/opensearch";
+ public static final String DEFAULT_VERSION = "3.7.0";
+
+ private static final DockerImageName OPENSEARCH_IMAGE = DockerImageName.parse(resolveImageName());
+
+ private static GenericContainer<?> container;
+
+ private OpenSearchTestContainerRunner() {
+ }
+
+ /**
+ * Resolves the OpenSearch container image from system properties, falling back to
+ * {@code opensearchproject/opensearch:3.7.0}. Package-visible for unit testing the resolution rules.
+ */
+ static String resolveImageName() {
+ String image = System.getProperty(IMAGE_PROPERTY);
+
+ if (image != null && !image.trim().isEmpty()) {
+ return image.trim();
+ }
+
+ String version = System.getProperty(VERSION_PROPERTY);
+
+ if (version == null || version.trim().isEmpty()) {
+ version = DEFAULT_VERSION;
+ }
+
+ return DEFAULT_REPO + ":" + version.trim();
+ }
+
+ public static String getImageName() {
+ return OPENSEARCH_IMAGE.asCanonicalNameString();
+ }
+
+ public static boolean isDockerAvailable() {
+ try {
+ return DockerClientFactory.instance().isDockerAvailable();
+ } catch (Throwable t) {
+ return false;
+ }
+ }
+
+ @SuppressWarnings("resource")
+ public static synchronized void start() {
+ if (container != null && container.isRunning()) {
+ applySystemProperties();
+ return;
+ }
+
+ container = new GenericContainer<>(OPENSEARCH_IMAGE)
+ .withExposedPorts(9200)
+ .withEnv("discovery.type", "single-node")
+ .withEnv("plugins.security.disabled", "true")
+ .withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true")
+ .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m")
+ .waitingFor(Wait.forHttp("/")
+ .forPort(9200)
+ .forStatusCode(200)
+ .withStartupTimeout(Duration.ofMinutes(3)));
+ container.start();
+ applySystemProperties();
+ }
+
+ public static synchronized void stop() {
+ if (container != null) {
+ container.stop();
+ container = null;
+ }
+ }
+
+ public static String getHost() {
+ ensureRunning();
+ return container.getHost();
+ }
+
+ public static int getPort() {
+ ensureRunning();
+ return container.getMappedPort(9200);
+ }
+
+ private static void ensureRunning() {
+ if (container == null || !container.isRunning()) {
+ throw new IllegalStateException("OpenSearch test container is not running");
+ }
+ }
+
+ private static void applySystemProperties() {
+ String host = container.getHost();
+ int port = container.getMappedPort(9200);
+ System.setProperty("opensearch.host", host);
+ System.setProperty("opensearch.port", String.valueOf(port));
+ System.setProperty("opensearch.test.host", host);
+ System.setProperty("opensearch.test.port", String.valueOf(port));
+ }
+}
diff --git a/test-tools/src/test/java/org/apache/atlas/runner/OpenSearchTestContainerRunnerTest.java b/test-tools/src/test/java/org/apache/atlas/runner/OpenSearchTestContainerRunnerTest.java
new file mode 100644
index 000000000..127c69736
--- /dev/null
+++ b/test-tools/src/test/java/org/apache/atlas/runner/OpenSearchTestContainerRunnerTest.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.runner;
+
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+/**
+ * Unit tests for {@link OpenSearchTestContainerRunner#resolveImageName()}. OpenSearch server version is
+ * configurable via system properties without code changes to validate 3.7 (default), 3.8, and 2.x
+ * compatibility targets. Docker-free — exercises only the package-private string-resolution rules; this test class
+ * lives in the same package specifically so it can call {@code resolveImageName()} directly (no reflection).
+ */
+public class OpenSearchTestContainerRunnerTest {
+
+ @AfterMethod
+ public void clearSystemProperties() {
+ System.clearProperty(OpenSearchTestContainerRunner.IMAGE_PROPERTY);
+ System.clearProperty(OpenSearchTestContainerRunner.VERSION_PROPERTY);
+ }
+
+ @Test
+ public void defaultsTo37WhenNoPropertiesSet() {
+ System.clearProperty(OpenSearchTestContainerRunner.IMAGE_PROPERTY);
+ System.clearProperty(OpenSearchTestContainerRunner.VERSION_PROPERTY);
+
+ assertEquals(OpenSearchTestContainerRunner.resolveImageName(), "opensearchproject/opensearch:3.7.0",
+ "default image must remain 3.7.0 so existing coverage is preserved when nothing is overridden");
+ }
+
+ @Test
+ public void versionPropertySelects38() {
+ System.setProperty(OpenSearchTestContainerRunner.VERSION_PROPERTY, "3.8.0");
+
+ assertEquals(OpenSearchTestContainerRunner.resolveImageName(), "opensearchproject/opensearch:3.8.0",
+ "version override must select the 3.8 image so the CI matrix can validate 3.8 support");
+ }
+
+ @Test
+ public void versionPropertySelects2xCompatibilityTarget() {
+ // The secondary/compatibility target: the resolution mechanism must support arbitrary version tags,
+ // including 2.x, without any code change — even though the CI matrix does not currently exercise it.
+ System.setProperty(OpenSearchTestContainerRunner.VERSION_PROPERTY, "2.11.0");
+
+ assertEquals(OpenSearchTestContainerRunner.resolveImageName(), "opensearchproject/opensearch:2.11.0",
+ "version override must select a 2.x image tag for the 2.x compatibility target");
+ }
+
+ @Test
+ public void fullImageOverrideTakesPrecedenceOverVersionProperty() {
+ System.setProperty(OpenSearchTestContainerRunner.VERSION_PROPERTY, "3.8.0");
+ System.setProperty(OpenSearchTestContainerRunner.IMAGE_PROPERTY, "myrepo/custom-opensearch:9.9.9");
+
+ assertEquals(OpenSearchTestContainerRunner.resolveImageName(), "myrepo/custom-opensearch:9.9.9",
+ "a full image override must win over a version-only override");
+ }
+
+ @Test
+ public void blankImagePropertyFallsBackToVersionResolution() {
+ System.setProperty(OpenSearchTestContainerRunner.IMAGE_PROPERTY, " ");
+ System.setProperty(OpenSearchTestContainerRunner.VERSION_PROPERTY, "3.8.0");
+
+ assertEquals(OpenSearchTestContainerRunner.resolveImageName(), "opensearchproject/opensearch:3.8.0",
+ "a blank image override must not be treated as set; version resolution must still apply");
+ }
+
+ @Test
+ public void blankVersionPropertyFallsBackToDefaultVersion() {
+ System.setProperty(OpenSearchTestContainerRunner.VERSION_PROPERTY, " ");
+
+ assertEquals(OpenSearchTestContainerRunner.resolveImageName(), "opensearchproject/opensearch:3.7.0",
+ "a blank version override must not be treated as set; default 3.7.0 must be used");
+ }
+
+ @Test
+ public void imageAndVersionPropertiesAreTrimmed() {
+ System.setProperty(OpenSearchTestContainerRunner.VERSION_PROPERTY, " 3.8.0 ");
+
+ assertEquals(OpenSearchTestContainerRunner.resolveImageName(), "opensearchproject/opensearch:3.8.0",
+ "surrounding whitespace in the version property must be trimmed");
+
+ System.clearProperty(OpenSearchTestContainerRunner.VERSION_PROPERTY);
+ System.setProperty(OpenSearchTestContainerRunner.IMAGE_PROPERTY, " myrepo/custom-opensearch:1.2.3 ");
+
+ assertEquals(OpenSearchTestContainerRunner.resolveImageName(), "myrepo/custom-opensearch:1.2.3",
+ "surrounding whitespace in the image property must be trimmed");
+ }
+}
(atlas) 01/02: ATLAS-4796: Support OpenSearch as mixed index backend
Posted by pi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pinal-shah pushed a commit to branch ATLAS-4796_u
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit ec7d9204de1041f1e0b8b71468dad5b02e11a322
Author: Pinal Shah <pi...@apache.org>
AuthorDate: Tue Sep 8 21:19:26 2026 +0530
ATLAS-4796: Support OpenSearch as mixed index backend
Add JanusGraph OpenSearch index provider, Atlas integration layer,
repository search support, Docker setup, and build configuration.
---
dev-support/atlas-docker/.env | 1 +
.../atlas-docker/Dockerfile.atlas-opensearch | 27 +
dev-support/atlas-docker/README.md | 83 +-
.../hbase/opensearch/atlas-application.properties | 89 ++
.../atlas/hbase/solr/atlas-application.properties | 91 ++
.../opensearch/atlas-application.properties | 99 ++
.../postgres/solr/atlas-application.properties | 101 ++
.../atlas-docker/docker-compose.atlas-common.yml | 45 +
.../atlas-docker/docker-compose.atlas-index.yml | 10 +
dev-support/atlas-docker/docker-compose.atlas.yml | 27 +-
.../repository/graphdb/AtlasGraphIndexClient.java | 24 +
.../repository/graphdb/AtlasGraphManagement.java | 24 +-
.../repository/graphdb/QuickSearchContext.java | 89 ++
.../repository/graphdb/QuickSearchResult.java | 42 +
graphdb/janus/pom.xml | 77 +
.../graphdb/janus/AtlasJanusGraphDatabase.java | 44 +-
.../graphdb/janus/AtlasJanusGraphIndexClient.java | 65 +-
.../graphdb/janus/AtlasJanusGraphManagement.java | 68 +-
.../graphdb/janus/AtlasOpenSearchIndexClient.java | 672 +++++++++
.../graphdb/janus/AtlasOpenSearchQueryBuilder.java | 727 +++++++++
.../opensearch/AtlasOpenSearchIndex.java | 100 ++
graphdb/janusgraph-opensearch/pom.xml | 162 ++
.../diskstorage/opensearch/OpenSearchClient.java | 77 +
.../opensearch/OpenSearchConstants.java | 31 +
.../diskstorage/opensearch/OpenSearchIndex.java | 1566 ++++++++++++++++++++
.../opensearch/OpenSearchMajorVersion.java | 56 +
.../diskstorage/opensearch/OpenSearchMutation.java | 90 ++
.../diskstorage/opensearch/OpenSearchRequest.java | 120 ++
.../diskstorage/opensearch/OpenSearchResponse.java | 57 +
.../diskstorage/opensearch/OpenSearchScroll.java | 121 ++
.../diskstorage/opensearch/OpenSearchSetup.java | 78 +
.../opensearch/compat/AbstractOSCompat.java | 235 +++
.../diskstorage/opensearch/compat/OS2Compat.java | 29 +
.../diskstorage/opensearch/compat/OS3Compat.java | 21 +
.../opensearch/compat/OSCompatUtils.java | 33 +
.../opensearch/mapping/IndexMapping.java | 46 +
.../opensearch/mapping/TypedIndexMappings.java | 37 +
.../opensearch/mapping/TypelessIndexMappings.java | 34 +
.../opensearch/rest/RestAggResponse.java | 29 +
.../diskstorage/opensearch/rest/RestAggResult.java | 32 +
.../diskstorage/opensearch/rest/RestAggValue.java | 30 +
.../opensearch/rest/RestBulkResponse.java | 79 +
.../opensearch/rest/RestClientSetup.java | 287 ++++
.../opensearch/rest/RestCountResponse.java | 31 +
.../opensearch/rest/RestIndexSettings.java | 47 +
.../opensearch/rest/RestOpenSearchClient.java | 755 ++++++++++
.../diskstorage/opensearch/rest/RestSearchHit.java | 91 ++
.../opensearch/rest/RestSearchResponse.java | 97 ++
.../opensearch/rest/RestSearchResults.java | 85 ++
.../util/BasicAuthHttpClientConfigCallback.java | 49 +
.../util/ConnectionKeepAliveConfigCallback.java | 38 +
.../opensearch/rest/util/HttpAuthTypes.java | 26 +
.../rest/util/RestClientAuthenticator.java | 42 +
.../rest/util/RestClientAuthenticatorBase.java | 41 +
.../rest/util/SSLConfigurationCallback.java | 190 +++
.../diskstorage/opensearch/script/OSScript.java | 41 +
.../opensearch/script/OSScriptResponse.java | 41 +
graphdb/pom.xml | 1 +
.../org/apache/atlas/ApplicationProperties.java | 6 +
pom.xml | 14 +
repository/pom.xml | 71 +-
.../atlas/discovery/FreeTextSearchProcessor.java | 245 ++-
.../atlas/discovery/SearchAggregatorImpl.java | 8 +
.../repository/graph/GraphBackedSearchIndexer.java | 43 +-
.../atlas/repository/graph/SolrIndexHelper.java | 59 +
test-tools/pom.xml | 13 +
66 files changed, 7731 insertions(+), 58 deletions(-)
diff --git a/dev-support/atlas-docker/.env b/dev-support/atlas-docker/.env
index 2041f68aa..bf410214a 100644
--- a/dev-support/atlas-docker/.env
+++ b/dev-support/atlas-docker/.env
@@ -25,3 +25,4 @@ KAFKA_VERSION=3.9.1
HIVE_VERSION=4.1.0
ATLAS_BACKEND=hbase
+ATLAS_INDEX_BACKEND=solr
diff --git a/dev-support/atlas-docker/Dockerfile.atlas-opensearch b/dev-support/atlas-docker/Dockerfile.atlas-opensearch
new file mode 100644
index 000000000..efd8b6c8a
--- /dev/null
+++ b/dev-support/atlas-docker/Dockerfile.atlas-opensearch
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM opensearchproject/opensearch:3.7.0
+
+ENV discovery.type=single-node \
+ DISABLE_INSTALL_DEMO_CONFIG=true \
+ DISABLE_SECURITY_PLUGIN=true \
+ OPENSEARCH_JAVA_OPTS="-Xms512m -Xmx512m"
+
+EXPOSE 9200 9600
+
+HEALTHCHECK --interval=10s --timeout=10s --retries=12 \
+ CMD curl -sf "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s" || exit 1
diff --git a/dev-support/atlas-docker/README.md b/dev-support/atlas-docker/README.md
index 0be922f07..e6e4f4e9c 100644
--- a/dev-support/atlas-docker/README.md
+++ b/dev-support/atlas-docker/README.md
@@ -21,6 +21,29 @@ under the License.
Docker files in this folder create docker images and run them to build Apache Atlas, deploy Apache Atlas and dependent services in containers.
+## Configuration
+
+Atlas server configuration is mounted from:
+
+```text
+config/atlas/${ATLAS_BACKEND}/${ATLAS_INDEX_BACKEND}/atlas-application.properties
+```
+
+Authentication credentials are mounted from `config/atlas/users-credentials.properties`.
+
+| Variable | Values | Controls |
+|----------|--------|----------|
+| `ATLAS_BACKEND` | `hbase`, `postgres` | Graph storage backend |
+| `ATLAS_INDEX_BACKEND` | `solr`, `opensearch` | Search index backend |
+
+Defaults in `.env`: `ATLAS_BACKEND=hbase`, `ATLAS_INDEX_BACKEND=solr`.
+
+**Important:** Switching `ATLAS_INDEX_BACKEND` requires a fresh Atlas data directory (`rm -rf ./data`) because JanusGraph validates the index backend on first init.
+
+For OpenSearch development, set `BUILD_HOST_SRC=true` when building so the container uses your local source tree (includes OpenSearch backend changes).
+
+See also [OpenSearch session context](../opensearch-docker/OPENSEARCH_SESSION_CONTEXT.md) for C6 REST validation after the stack is up.
+
## Usage
1. Ensure that you have recent version of Docker installed from [docker.io](http://www.docker.io) (as of this writing: Engine 20.10.5, Compose 1.28.5).
@@ -28,7 +51,7 @@ Docker files in this folder create docker images and run them to build Apache At
2. Set this folder as your working directory.
-3. Update environment variables in .env file, if necessary
+3. Update environment variables in `.env`, if necessary
4. Execute following command to download necessary archives to setup Atlas/HDFS/HBase/Kafka services:
@@ -46,9 +69,6 @@ Docker files in this folder create docker images and run them to build Apache At
6. Build and deploy Apache Atlas in containers using docker compose
- Atlas server configuration is mounted from `config/atlas/${ATLAS_BACKEND}/atlas-application.properties`.
- The file authentication credentials are mounted from `config/atlas/users-credentials.properties`.
-
1. Build atlas-base image with the following command:
```shell
@@ -62,25 +82,72 @@ Docker files in this folder create docker images and run them to build Apache At
docker compose -f docker-compose.atlas-build.yml up
```
- Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
+ Time taken to complete the build might vary (upto an hour), depending on status of `${HOME}/.m2` directory cache.
- 3. To install and start Atlas using Postgres as backend store, execute following commands:
+ 3. To install and start Atlas using Postgres as backend store (Solr index):
```shell
export ATLAS_BACKEND=postgres
+ export ATLAS_INDEX_BACKEND=solr
docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-postgres.yml up -d --wait
```
The Postgres overlay runs `config/init_postgres.sh` as a one-shot initialization service before Atlas starts.
This creates the required roles, databases, and Atlas RDBMS schema.
- 4. To install and start Atlas using HBase as backend store, execute following commands:
+ 4. To install and start Atlas using HBase as backend store (Solr index):
+
+ ```shell
+ export ATLAS_BACKEND=hbase
+ export ATLAS_INDEX_BACKEND=solr
+ docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml up -d --wait
+ ```
+
+ 5. To install and start Atlas using **HBase + OpenSearch** (recommended for OpenSearch backend validation):
```shell
export ATLAS_BACKEND=hbase
+ export ATLAS_INDEX_BACKEND=opensearch
+ export COMPOSE_PROFILES=opensearch-index
+ export BUILD_HOST_SRC=true
+ rm -rf ./data
+ docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml build
docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-hadoop.yml up -d --wait
```
- Apache Atlas will be installed at /opt/atlas/, and logs are at /var/log/atlas directory.
+ 6. To install and start Atlas using **Postgres + OpenSearch**:
+
+ ```shell
+ export ATLAS_BACKEND=postgres
+ export ATLAS_INDEX_BACKEND=opensearch
+ export COMPOSE_PROFILES=opensearch-index
+ export BUILD_HOST_SRC=true
+ rm -rf ./data
+ docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-postgres.yml build
+ docker compose -f docker-compose.atlas.yml -f docker-compose.atlas-postgres.yml up -d --wait
+ ```
+
+ **Verify OpenSearch + Atlas:**
+
+ ```shell
+ curl -s http://localhost:9200/
+ curl -s http://localhost:15601/api/status
+ curl -u admin:atlasR0cks! http://localhost:21000/api/atlas/admin/version
+ curl -u admin:atlasR0cks! "http://localhost:21000/api/atlas/v2/search/quick?query=*&limit=1"
+ ```
+
+ OpenSearch Dashboards UI: http://localhost:15601 (no login when security plugin is disabled)
+
+ If Dashboards fails to load in Chrome with `[I18n] A locale must be a non-empty string`, clear site
+ data for the Dashboards URL (Chrome → Settings → Privacy → Site settings → View all → localhost),
+ use a private window, or recreate the container after pulling the updated compose port mapping.
+
+ If Docker Compose hangs on `Pulling` for locally built images, use:
+
+ ```shell
+ export COMPOSE_PULL_POLICY=never
+ ```
+
+ Apache Atlas will be installed at `/opt/atlas/`, and logs are at `/var/log/atlas` directory.
7. Atlas Admin can be accessed at http://localhost:21000 (admin/atlasR0cks!)
diff --git a/dev-support/atlas-docker/config/atlas/hbase/opensearch/atlas-application.properties b/dev-support/atlas-docker/config/atlas/hbase/opensearch/atlas-application.properties
new file mode 100644
index 000000000..f024c6485
--- /dev/null
+++ b/dev-support/atlas-docker/config/atlas/hbase/opensearch/atlas-application.properties
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+######### Graph Database Configs #########
+
+atlas.graph.storage.backend=hbase2
+atlas.graph.storage.hbase.table=apache_atlas_janus
+atlas.graph.storage.hostname=atlas-zk.example.com:2181
+atlas.graph.storage.hbase.regions-per-server=1
+atlas.graph.storage.hbase.compression-algorithm=NONE
+atlas.graph.graph.replace-instance-if-exists=true
+
+atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository
+atlas.audit.hbase.tablename=apache_atlas_entity_audit
+atlas.audit.zookeeper.session.timeout.ms=1000
+atlas.audit.hbase.zookeeper.quorum=atlas-zk.example.com:2181
+
+######### Graph Search Index #########
+
+atlas.graph.index.search.backend=opensearch
+atlas.graph.index.search.hostname=atlas-opensearch.example.com
+atlas.graph.index.search.port=9200
+atlas.graph.index.search.max-result-set-size=150
+atlas.graph.index.search.opensearch.setup-max-open-scroll-contexts=false
+
+######### Notification Configs #########
+
+atlas.notification.embedded=false
+atlas.kafka.data=${sys:atlas.home}/data/kafka
+atlas.kafka.zookeeper.connect=atlas-zk.example.com:2181
+atlas.kafka.bootstrap.servers=atlas-kafka.example.com:9092
+atlas.kafka.zookeeper.session.timeout.ms=400
+atlas.kafka.zookeeper.connection.timeout.ms=200
+atlas.kafka.zookeeper.sync.time.ms=20
+atlas.kafka.auto.commit.interval.ms=1000
+atlas.kafka.hook.group.id=atlas
+atlas.kafka.enable.auto.commit=false
+atlas.kafka.auto.offset.reset=earliest
+atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
+atlas.kafka.poll.timeout.ms=1000
+atlas.notification.create.topics=true
+atlas.notification.replicas=1
+atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES
+atlas.notification.log.failed.messages=true
+atlas.notification.consumer.retry.interval=500
+atlas.notification.hook.retry.interval=1000
+
+######### Security Properties #########
+
+atlas.enableTLS=false
+atlas.authentication.method.kerberos=false
+atlas.authentication.method.file=true
+atlas.authentication.method.ldap.type=none
+atlas.authentication.method.file.filename=${sys:atlas.home}/conf/users-credentials.properties
+
+######### Server Properties #########
+
+atlas.rest.address=http://localhost:21000
+atlas.server.ha.enabled=false
+
+######### Atlas Authorization #########
+
+atlas.authorizer.impl=simple
+atlas.authorizer.simple.authz.policy.file=atlas-simple-authz-policy.json
+
+######### CSRF Configs #########
+
+atlas.rest-csrf.enabled=true
+atlas.rest-csrf.browser-useragents-regex=^Mozilla.*,^Opera.*,^Chrome.*
+atlas.rest-csrf.methods-to-ignore=GET,OPTIONS,HEAD,TRACE
+atlas.rest-csrf.custom-header=X-XSRF-HEADER
+
+######### Atlas Metric/Stats configs #########
+
+atlas.metric.query.cache.ttlInSecs=900
diff --git a/dev-support/atlas-docker/config/atlas/hbase/solr/atlas-application.properties b/dev-support/atlas-docker/config/atlas/hbase/solr/atlas-application.properties
new file mode 100644
index 000000000..a4c11ec45
--- /dev/null
+++ b/dev-support/atlas-docker/config/atlas/hbase/solr/atlas-application.properties
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+######### Graph Database Configs #########
+
+atlas.graph.storage.backend=hbase2
+atlas.graph.storage.hbase.table=apache_atlas_janus
+atlas.graph.storage.hostname=atlas-zk.example.com:2181
+atlas.graph.storage.hbase.regions-per-server=1
+atlas.graph.storage.hbase.compression-algorithm=NONE
+atlas.graph.graph.replace-instance-if-exists=true
+
+atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.HBaseBasedAuditRepository
+atlas.audit.hbase.tablename=apache_atlas_entity_audit
+atlas.audit.zookeeper.session.timeout.ms=1000
+atlas.audit.hbase.zookeeper.quorum=atlas-zk.example.com:2181
+
+######### Graph Search Index #########
+
+atlas.graph.index.search.backend=solr
+atlas.graph.index.search.solr.mode=http
+atlas.graph.index.search.solr.http-urls=http://atlas-solr.example.com:8983/solr
+atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
+atlas.graph.index.search.solr.zookeeper-session-timeout=60000
+atlas.graph.index.search.solr.wait-searcher=false
+atlas.graph.index.search.max-result-set-size=150
+
+######### Notification Configs #########
+
+atlas.notification.embedded=false
+atlas.kafka.data=${sys:atlas.home}/data/kafka
+atlas.kafka.zookeeper.connect=atlas-zk.example.com:2181
+atlas.kafka.bootstrap.servers=atlas-kafka.example.com:9092
+atlas.kafka.zookeeper.session.timeout.ms=400
+atlas.kafka.zookeeper.connection.timeout.ms=200
+atlas.kafka.zookeeper.sync.time.ms=20
+atlas.kafka.auto.commit.interval.ms=1000
+atlas.kafka.hook.group.id=atlas
+atlas.kafka.enable.auto.commit=false
+atlas.kafka.auto.offset.reset=earliest
+atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
+atlas.kafka.poll.timeout.ms=1000
+atlas.notification.create.topics=true
+atlas.notification.replicas=1
+atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES
+atlas.notification.log.failed.messages=true
+atlas.notification.consumer.retry.interval=500
+atlas.notification.hook.retry.interval=1000
+
+######### Security Properties #########
+
+atlas.enableTLS=false
+atlas.authentication.method.kerberos=false
+atlas.authentication.method.file=true
+atlas.authentication.method.ldap.type=none
+atlas.authentication.method.file.filename=${sys:atlas.home}/conf/users-credentials.properties
+
+######### Server Properties #########
+
+atlas.rest.address=http://localhost:21000
+atlas.server.ha.enabled=false
+
+######### Atlas Authorization #########
+
+atlas.authorizer.impl=simple
+atlas.authorizer.simple.authz.policy.file=atlas-simple-authz-policy.json
+
+######### CSRF Configs #########
+
+atlas.rest-csrf.enabled=true
+atlas.rest-csrf.browser-useragents-regex=^Mozilla.*,^Opera.*,^Chrome.*
+atlas.rest-csrf.methods-to-ignore=GET,OPTIONS,HEAD,TRACE
+atlas.rest-csrf.custom-header=X-XSRF-HEADER
+
+######### Atlas Metric/Stats configs #########
+
+atlas.metric.query.cache.ttlInSecs=900
diff --git a/dev-support/atlas-docker/config/atlas/postgres/opensearch/atlas-application.properties b/dev-support/atlas-docker/config/atlas/postgres/opensearch/atlas-application.properties
new file mode 100644
index 000000000..980028884
--- /dev/null
+++ b/dev-support/atlas-docker/config/atlas/postgres/opensearch/atlas-application.properties
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+######### Graph Database Configs #########
+
+atlas.graph.storage.backend=rdbms
+atlas.graph.storage.hbase.table=apache_atlas_janus
+atlas.graph.storage.hbase.compression-algorithm=NONE
+atlas.graph.graph.replace-instance-if-exists=true
+
+atlas.graph.storage.rdbms.jpa.hikari.driverClassName=org.postgresql.Driver
+atlas.graph.storage.rdbms.jpa.hikari.jdbcUrl=jdbc:postgresql://atlas-db/atlas
+atlas.graph.storage.rdbms.jpa.hikari.username=atlas
+atlas.graph.storage.rdbms.jpa.hikari.password=atlasR0cks!
+atlas.graph.storage.rdbms.jpa.hikari.maximumPoolSize=40
+atlas.graph.storage.rdbms.jpa.hikari.minimumIdle=5
+atlas.graph.storage.rdbms.jpa.hikari.idleTimeout=300000
+atlas.graph.storage.rdbms.jpa.hikari.connectionTestQuery=select 1
+atlas.graph.storage.rdbms.jpa.hikari.maxLifetime=1800000
+atlas.graph.storage.rdbms.jpa.hikari.connectionTimeout=30000
+atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.dialect=org.eclipse.persistence.platform.database.PostgreSQLPlatform
+
+atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository
+atlas.audit.hbase.tablename=apache_atlas_entity_audit
+atlas.audit.zookeeper.session.timeout.ms=1000
+atlas.audit.hbase.zookeeper.quorum=atlas-zk.example.com:2181
+
+######### Graph Search Index #########
+
+atlas.graph.index.search.backend=opensearch
+atlas.graph.index.search.hostname=atlas-opensearch.example.com
+atlas.graph.index.search.port=9200
+atlas.graph.index.search.max-result-set-size=150
+atlas.graph.index.search.opensearch.setup-max-open-scroll-contexts=false
+
+######### Notification Configs #########
+
+atlas.notification.embedded=false
+atlas.kafka.data=${sys:atlas.home}/data/kafka
+atlas.kafka.zookeeper.connect=atlas-zk.example.com:2181
+atlas.kafka.bootstrap.servers=atlas-kafka.example.com:9092
+atlas.kafka.zookeeper.session.timeout.ms=400
+atlas.kafka.zookeeper.connection.timeout.ms=200
+atlas.kafka.zookeeper.sync.time.ms=20
+atlas.kafka.auto.commit.interval.ms=1000
+atlas.kafka.hook.group.id=atlas
+atlas.kafka.enable.auto.commit=false
+atlas.kafka.auto.offset.reset=earliest
+atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
+atlas.kafka.poll.timeout.ms=1000
+atlas.notification.create.topics=true
+atlas.notification.replicas=1
+atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES
+atlas.notification.log.failed.messages=true
+atlas.notification.consumer.retry.interval=500
+atlas.notification.hook.retry.interval=1000
+
+######### Security Properties #########
+
+atlas.enableTLS=false
+atlas.authentication.method.kerberos=false
+atlas.authentication.method.file=true
+atlas.authentication.method.ldap.type=none
+atlas.authentication.method.file.filename=${sys:atlas.home}/conf/users-credentials.properties
+
+######### Server Properties #########
+
+atlas.rest.address=http://localhost:21000
+atlas.server.ha.enabled=false
+
+######### Atlas Authorization #########
+
+atlas.authorizer.impl=simple
+atlas.authorizer.simple.authz.policy.file=atlas-simple-authz-policy.json
+
+######### CSRF Configs #########
+
+atlas.rest-csrf.enabled=true
+atlas.rest-csrf.browser-useragents-regex=^Mozilla.*,^Opera.*,^Chrome.*
+atlas.rest-csrf.methods-to-ignore=GET,OPTIONS,HEAD,TRACE
+atlas.rest-csrf.custom-header=X-XSRF-HEADER
+
+######### Atlas Metric/Stats configs #########
+
+atlas.metric.query.cache.ttlInSecs=900
diff --git a/dev-support/atlas-docker/config/atlas/postgres/solr/atlas-application.properties b/dev-support/atlas-docker/config/atlas/postgres/solr/atlas-application.properties
new file mode 100644
index 000000000..e15bac245
--- /dev/null
+++ b/dev-support/atlas-docker/config/atlas/postgres/solr/atlas-application.properties
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+######### Graph Database Configs #########
+
+atlas.graph.storage.backend=rdbms
+atlas.graph.storage.hbase.table=apache_atlas_janus
+atlas.graph.storage.hbase.compression-algorithm=NONE
+atlas.graph.graph.replace-instance-if-exists=true
+
+atlas.graph.storage.rdbms.jpa.hikari.driverClassName=org.postgresql.Driver
+atlas.graph.storage.rdbms.jpa.hikari.jdbcUrl=jdbc:postgresql://atlas-db/atlas
+atlas.graph.storage.rdbms.jpa.hikari.username=atlas
+atlas.graph.storage.rdbms.jpa.hikari.password=atlasR0cks!
+atlas.graph.storage.rdbms.jpa.hikari.maximumPoolSize=40
+atlas.graph.storage.rdbms.jpa.hikari.minimumIdle=5
+atlas.graph.storage.rdbms.jpa.hikari.idleTimeout=300000
+atlas.graph.storage.rdbms.jpa.hikari.connectionTestQuery=select 1
+atlas.graph.storage.rdbms.jpa.hikari.maxLifetime=1800000
+atlas.graph.storage.rdbms.jpa.hikari.connectionTimeout=30000
+atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.dialect=org.eclipse.persistence.platform.database.PostgreSQLPlatform
+
+atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.rdbms.RdbmsBasedAuditRepository
+atlas.audit.hbase.tablename=apache_atlas_entity_audit
+atlas.audit.zookeeper.session.timeout.ms=1000
+atlas.audit.hbase.zookeeper.quorum=atlas-zk.example.com:2181
+
+######### Graph Search Index #########
+
+atlas.graph.index.search.backend=solr
+atlas.graph.index.search.solr.mode=http
+atlas.graph.index.search.solr.http-urls=http://atlas-solr.example.com:8983/solr
+atlas.graph.index.search.solr.zookeeper-connect-timeout=60000
+atlas.graph.index.search.solr.zookeeper-session-timeout=60000
+atlas.graph.index.search.solr.wait-searcher=false
+atlas.graph.index.search.max-result-set-size=150
+
+######### Notification Configs #########
+
+atlas.notification.embedded=false
+atlas.kafka.data=${sys:atlas.home}/data/kafka
+atlas.kafka.zookeeper.connect=atlas-zk.example.com:2181
+atlas.kafka.bootstrap.servers=atlas-kafka.example.com:9092
+atlas.kafka.zookeeper.session.timeout.ms=400
+atlas.kafka.zookeeper.connection.timeout.ms=200
+atlas.kafka.zookeeper.sync.time.ms=20
+atlas.kafka.auto.commit.interval.ms=1000
+atlas.kafka.hook.group.id=atlas
+atlas.kafka.enable.auto.commit=false
+atlas.kafka.auto.offset.reset=earliest
+atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
+atlas.kafka.poll.timeout.ms=1000
+atlas.notification.create.topics=true
+atlas.notification.replicas=1
+atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES
+atlas.notification.log.failed.messages=true
+atlas.notification.consumer.retry.interval=500
+atlas.notification.hook.retry.interval=1000
+
+######### Security Properties #########
+
+atlas.enableTLS=false
+atlas.authentication.method.kerberos=false
+atlas.authentication.method.file=true
+atlas.authentication.method.ldap.type=none
+atlas.authentication.method.file.filename=${sys:atlas.home}/conf/users-credentials.properties
+
+######### Server Properties #########
+
+atlas.rest.address=http://localhost:21000
+atlas.server.ha.enabled=false
+
+######### Atlas Authorization #########
+
+atlas.authorizer.impl=simple
+atlas.authorizer.simple.authz.policy.file=atlas-simple-authz-policy.json
+
+######### CSRF Configs #########
+
+atlas.rest-csrf.enabled=true
+atlas.rest-csrf.browser-useragents-regex=^Mozilla.*,^Opera.*,^Chrome.*
+atlas.rest-csrf.methods-to-ignore=GET,OPTIONS,HEAD,TRACE
+atlas.rest-csrf.custom-header=X-XSRF-HEADER
+
+######### Atlas Metric/Stats configs #########
+
+atlas.metric.query.cache.ttlInSecs=900
diff --git a/dev-support/atlas-docker/docker-compose.atlas-common.yml b/dev-support/atlas-docker/docker-compose.atlas-common.yml
index 15375cd2a..a14ba6263 100644
--- a/dev-support/atlas-docker/docker-compose.atlas-common.yml
+++ b/dev-support/atlas-docker/docker-compose.atlas-common.yml
@@ -10,6 +10,51 @@ services:
- atlas
ports:
- "8983:8983"
+ healthcheck:
+ test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/admin/info/system || exit 1"]
+ interval: 10s
+ timeout: 10s
+ retries: 12
+ start_period: 30s
+
+ atlas-opensearch:
+ build:
+ context: .
+ dockerfile: Dockerfile.atlas-opensearch
+ image: atlas-opensearch
+ container_name: atlas-opensearch
+ hostname: atlas-opensearch.example.com
+ networks:
+ - atlas
+ ports:
+ - "9200:9200"
+ - "9600:9600"
+ healthcheck:
+ test: ["CMD-SHELL", "curl -sf 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s' || exit 1"]
+ interval: 10s
+ timeout: 10s
+ retries: 12
+ start_period: 30s
+
+ atlas-opensearch-dashboards:
+ image: opensearchproject/opensearch-dashboards:3.7.0
+ container_name: atlas-opensearch-dashboards
+ hostname: atlas-opensearch-dashboards.example.com
+ networks:
+ - atlas
+ environment:
+ - OPENSEARCH_HOSTS=["http://atlas-opensearch:9200"]
+ - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true
+ - I18N_LOCALE=en
+ ports:
+ # Host 15601 avoids Chrome localStorage conflicts with other Kibana/OSD on 5601
+ - "15601:5601"
+ healthcheck:
+ test: ["CMD-SHELL", "curl -sf http://localhost:5601/api/status || exit 1"]
+ interval: 10s
+ timeout: 10s
+ retries: 12
+ start_period: 30s
atlas-kafka:
build:
diff --git a/dev-support/atlas-docker/docker-compose.atlas-index.yml b/dev-support/atlas-docker/docker-compose.atlas-index.yml
new file mode 100644
index 000000000..0beb1cd48
--- /dev/null
+++ b/dev-support/atlas-docker/docker-compose.atlas-index.yml
@@ -0,0 +1,10 @@
+services:
+ solr:
+ extends:
+ file: docker-compose.atlas-common.yml
+ service: atlas-solr
+
+ opensearch:
+ extends:
+ file: docker-compose.atlas-common.yml
+ service: atlas-opensearch
diff --git a/dev-support/atlas-docker/docker-compose.atlas.yml b/dev-support/atlas-docker/docker-compose.atlas.yml
index 646b8cc3a..55c5bf1f9 100644
--- a/dev-support/atlas-docker/docker-compose.atlas.yml
+++ b/dev-support/atlas-docker/docker-compose.atlas.yml
@@ -16,7 +16,7 @@ services:
- atlas
volumes:
- ./data:/home/atlas/data
- - ./config/atlas/${ATLAS_BACKEND}/atlas-application.properties:/opt/atlas/conf/atlas-application.properties:ro
+ - ./config/atlas/${ATLAS_BACKEND}/${ATLAS_INDEX_BACKEND}/atlas-application.properties:/opt/atlas/conf/atlas-application.properties:ro
- ./config/atlas/users-credentials.properties:/opt/atlas/conf/users-credentials.properties:ro
ports:
- "21000:21000"
@@ -25,12 +25,13 @@ services:
condition: service_healthy
atlas-kafka:
condition: service_started
- atlas-solr:
- condition: service_started
+ atlas-index:
+ condition: service_healthy
atlas-zk:
condition: service_started
environment:
- ATLAS_BACKEND
+ - ATLAS_INDEX_BACKEND
- ATLAS_SERVER_JAVA_VERSION
- ATLAS_VERSION
command:
@@ -41,14 +42,26 @@ services:
service: ${ATLAS_BACKEND}
file: docker-compose.atlas-backend.yml
- atlas-kafka:
+ atlas-index:
+ extends:
+ service: ${ATLAS_INDEX_BACKEND}
+ file: docker-compose.atlas-index.yml
+
+ atlas-opensearch-dashboards:
extends:
- service: atlas-kafka
file: docker-compose.atlas-common.yml
+ service: atlas-opensearch-dashboards
+ profiles:
+ - opensearch-index
+ depends_on:
+ atlas-index:
+ condition: service_healthy
+ environment:
+ - OPENSEARCH_HOSTS=["http://atlas-index:9200"]
- atlas-solr:
+ atlas-kafka:
extends:
- service: atlas-solr
+ service: atlas-kafka
file: docker-compose.atlas-common.yml
atlas-zk:
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
index 323161b21..3b3905f91 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphIndexClient.java
@@ -17,10 +17,12 @@
*/
package org.apache.atlas.repository.graphdb;
+import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Represents a graph client work with indices used by Jansgraph.
@@ -42,6 +44,19 @@ public interface AtlasGraphIndexClient {
*/
List<String> getSuggestions(String prefixString, String indexFieldName);
+ /**
+ * Weighted quick search for OpenSearch backends. Non-OpenSearch implementations return an empty result.
+ * <p>
+ * Backend failures (e.g. OpenSearch unavailable) and query-building failures (e.g. an unsupported operator)
+ * are surfaced as {@link AtlasBaseException} rather than being converted to an empty result, so that an outage
+ * is not mistaken for a legitimate zero-hit search.
+ *
+ * @param quickSearchContext quick-search inputs (query, filters, pagination)
+ * @return matching entity GUIDs in score order and total hit count
+ * @throws AtlasBaseException if the query cannot be built or the backend request fails
+ */
+ QuickSearchResult quickSearch(QuickSearchContext quickSearchContext) throws AtlasBaseException;
+
/**
* The implementers should apply the search weights for the passed in properties.
*
@@ -58,6 +73,15 @@ public interface AtlasGraphIndexClient {
*/
void applySuggestionFields(String collectionName, List<String> suggestionProperties);
+ /**
+ * Registers OpenSearch index fields that use a text mapping with a {@code keyword} subfield for terms queries.
+ * Non-OpenSearch implementations ignore this call.
+ *
+ * @param collectionName mixed index collection name
+ * @param indexFieldNames index field names that require {@code .keyword} for terms aggregations/suggestions
+ */
+ void applyKeywordSubfieldFields(String collectionName, Set<String> indexFieldNames);
+
/**
* Returns status of index client
*
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
index 55f2d7629..99ace6564 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
@@ -153,7 +153,20 @@ public interface AtlasGraphManagement extends AutoCloseable {
* @param isStringField
* @return the index field name used for the given property
*/
- String addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey, boolean isStringField);
+ default String addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey, boolean isStringField) {
+ return addMixedIndex(vertexIndex, propertyKey, isStringField, false);
+ }
+
+ /**
+ * Adds a property key to the given index in the graph.
+ *
+ * @param vertexIndex vertex mixed index name
+ * @param propertyKey property key to index
+ * @param isStringField when true, maps as OpenSearch/Solr keyword/string
+ * @param withKeywordSubfield when true on OpenSearch, maps as text with a {@code keyword} subfield for terms aggs
+ * @return the index field name used for the given property
+ */
+ String addMixedIndex(String vertexIndex, AtlasPropertyKey propertyKey, boolean isStringField, boolean withKeywordSubfield);
/**
* Gets the index field name for the vertex property.
@@ -161,7 +174,14 @@ public interface AtlasGraphManagement extends AutoCloseable {
* @param propertyKey
* @return the encoded name for the index
*/
- String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey, boolean isStringField);
+ default String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey, boolean isStringField) {
+ return getIndexFieldName(indexName, propertyKey, isStringField, false);
+ }
+
+ /**
+ * Gets the index field name for the vertex property.
+ */
+ String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey, boolean isStringField, boolean withKeywordSubfield);
/**
* Set consistency to ConsistencyModifier.LOCK for all vertex and edge indexes.
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/QuickSearchContext.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/QuickSearchContext.java
new file mode 100644
index 000000000..77a20de1b
--- /dev/null
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/QuickSearchContext.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb;
+
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.type.AtlasEntityType;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Inputs for OpenSearch-native weighted quick search.
+ */
+public class QuickSearchContext {
+ private final String queryString;
+ private final FilterCriteria filterCriteria;
+ private final Set<AtlasEntityType> entityTypes;
+ private final Set<String> classificationTypeNames;
+ private final Map<String, String> indexFieldNameCache;
+ private final boolean excludeDeletedEntities;
+ private final boolean includeSubTypes;
+ private final int offset;
+ private final int limit;
+
+ public QuickSearchContext(String queryString, FilterCriteria filterCriteria, Set<AtlasEntityType> entityTypes,
+ Set<String> classificationTypeNames, Map<String, String> indexFieldNameCache,
+ boolean excludeDeletedEntities, boolean includeSubTypes, int offset, int limit) {
+ this.queryString = queryString;
+ this.filterCriteria = filterCriteria;
+ this.entityTypes = entityTypes;
+ this.classificationTypeNames = classificationTypeNames;
+ this.indexFieldNameCache = indexFieldNameCache;
+ this.excludeDeletedEntities = excludeDeletedEntities;
+ this.includeSubTypes = includeSubTypes;
+ this.offset = offset;
+ this.limit = limit;
+ }
+
+ public String getQueryString() {
+ return queryString;
+ }
+
+ public FilterCriteria getFilterCriteria() {
+ return filterCriteria;
+ }
+
+ public Set<AtlasEntityType> getEntityTypes() {
+ return entityTypes;
+ }
+
+ public Set<String> getClassificationTypeNames() {
+ return classificationTypeNames;
+ }
+
+ public Map<String, String> getIndexFieldNameCache() {
+ return indexFieldNameCache;
+ }
+
+ public boolean isExcludeDeletedEntities() {
+ return excludeDeletedEntities;
+ }
+
+ public boolean isIncludeSubTypes() {
+ return includeSubTypes;
+ }
+
+ public int getOffset() {
+ return offset;
+ }
+
+ public int getLimit() {
+ return limit;
+ }
+}
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/QuickSearchResult.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/QuickSearchResult.java
new file mode 100644
index 000000000..b8ab9a4f3
--- /dev/null
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/QuickSearchResult.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * OpenSearch weighted quick-search response: entity GUIDs in score order plus total hit count.
+ */
+public class QuickSearchResult {
+ private final List<String> entityGuids;
+ private final long totalCount;
+
+ public QuickSearchResult(List<String> entityGuids, long totalCount) {
+ this.entityGuids = entityGuids == null ? Collections.emptyList() : entityGuids;
+ this.totalCount = totalCount;
+ }
+
+ public List<String> getEntityGuids() {
+ return entityGuids;
+ }
+
+ public long getTotalCount() {
+ return totalCount;
+ }
+}
diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml
index 4221210f6..b142f62ed 100644
--- a/graphdb/janus/pom.xml
+++ b/graphdb/janus/pom.xml
@@ -56,6 +56,12 @@
<artifactId>hbase-shaded-client-fixed</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>janusgraph-opensearch</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>janusgraph-rdbms</artifactId>
@@ -336,6 +342,13 @@
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
@@ -347,6 +360,11 @@
<version>${surefire.version}</version>
<configuration>
<skip>${skipUTs}</skip>
+ <excludes>
+ <!-- Requires a live OpenSearch cluster (Testcontainers); only run via the opensearch-it
+ profile so normal builds/JDKs are not affected. -->
+ <exclude>**/OpenSearchRuntimeInitTest.java</exclude>
+ </excludes>
</configuration>
</plugin>
@@ -374,4 +392,63 @@
</plugins>
</build>
+ <profiles>
+ <profile>
+ <!-- OpenSearch integration tests run against a real OpenSearch server provisioned by Testcontainers
+ (Docker). Mirrors the repository/pom.xml opensearch-it profile: this module already carries
+ janusgraph-opensearch, atlas-testtools and testcontainers as regular/test dependencies, so no
+ additional dependencies are needed here — only the surefire include/exclude override. -->
+ <id>opensearch-it</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration combine.self="override">
+ <skip>${skipUTs}</skip>
+ <!-- combine.self="override" (needed so the base module's exclude of
+ OpenSearchRuntimeInitTest.java above does not re-apply and cancel out the include
+ below) replaces the ENTIRE effective configuration for this plugin, including
+ whatever this module inherits from the root pom.xml pluginManagement, so the
+ JDK17 module opens flags (argLine) must be re-declared here too. Without this,
+ AtlasJanusGraphDatabase's reflective static init fails with InaccessibleObjectException
+ because the forked test JVM no longer gets the java.lang reflective access flag. -->
+ <argLine>${jdk.argLine} ${argLine}
+ -Djava.awt.headless=true -Dproject.version=${project.version}
+ -Dhadoop.tmp.dir="${project.build.directory}/tmp-hadoop-${user.name}"
+ -Xmx1024m -Djava.net.preferIPv4Stack=true ${atlas.surefire.options}</argLine>
+ <includes>
+ <include>**/*OpenSearch*IT.java</include>
+ <include>**/OpenSearchRuntimeInitTest.java</include>
+ </includes>
+ <systemPropertyVariables>
+ <opensearch.docker.image>${opensearch.docker.image}</opensearch.docker.image>
+ <opensearch.docker.version>${opensearch.docker.version}</opensearch.docker.version>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <!-- The root pom.xml binds maven-failsafe-plugin globally with forkCount=0 ("don't fork
+ the IT until there's sandbox capability for IT env"), which also means its argLine
+ (containing the JDK17 module opens flags) has no effect, since argLine only applies
+ to a forked JVM. Failsafe's default **/*IT.java pattern already matches these
+ OpenSearch ITs and re-runs them (redundantly with the surefire execution above) at
+ the integration-test/verify phase, so it also needs a real fork here to load
+ AtlasJanusGraphDatabase (JDK17 reflective static init) and the docker version
+ system properties, without changing the forkCount=0 default for every other module. -->
+ <configuration>
+ <forkCount>1</forkCount>
+ <systemPropertyVariables>
+ <opensearch.docker.image>${opensearch.docker.image}</opensearch.docker.image>
+ <opensearch.docker.version>${opensearch.docker.version}</opensearch.docker.version>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
</project>
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
index 7da9b2c94..3bd2469f3 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
@@ -39,6 +39,7 @@ import org.janusgraph.core.schema.JanusGraphManagement;
import org.janusgraph.diskstorage.StandardIndexProvider;
import org.janusgraph.diskstorage.StandardStoreManager;
import org.janusgraph.diskstorage.es.ElasticSearch7Index;
+import org.janusgraph.diskstorage.opensearch.AtlasOpenSearchIndex;
import org.janusgraph.diskstorage.hbase.HBaseStoreManager;
import org.janusgraph.diskstorage.rdbms.RdbmsStoreManager;
import org.janusgraph.diskstorage.solr.Solr6Index;
@@ -78,6 +79,7 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
public static final String SOLR_ZOOKEEPER_URLS = "atlas.graph.index.search.solr.zookeeper-urls";
public static final String INDEX_BACKEND_LUCENE = "lucene";
public static final String INDEX_BACKEND_ES = "elasticsearch";
+ public static final String INDEX_BACKEND_OPENSEARCH = "opensearch";
public static final String GRAPH_TX_LOG_CONF = "tx.log-tx";
public static final String GRAPH_TX_LOG_VERBOSE_CONF = "tx.recovery.verbose";
public static final String GRAPH_TX_LOG_TTL_CONF = "log.tx.ttl";
@@ -221,10 +223,8 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
@VisibleForTesting
static JanusGraph initJanusGraph(Configuration config) {
- org.apache.commons.configuration2.Configuration conf2 = createConfiguration2(config);
-
try {
- return JanusGraphFactory.open(conf2);
+ return JanusGraphFactory.open(config);
} catch (JanusGraphException e) {
LOG.warn("JanusGraphException: {}", e.getMessage());
@@ -233,7 +233,7 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
config.addProperty("graph.allow-upgrade", true);
- return JanusGraphFactory.open(conf2);
+ return JanusGraphFactory.open(config);
} else {
throw new RuntimeException(e);
}
@@ -454,6 +454,40 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
}
}
+ private static void addOpenSearchIndex() {
+ try {
+ Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES");
+
+ field.setAccessible(true);
+
+ Method getDeclaredFields0 = Class.class.getDeclaredMethod("getDeclaredFields0", boolean.class);
+ getDeclaredFields0.setAccessible(true);
+ Field[] fields = (Field[]) getDeclaredFields0.invoke(Field.class, false);
+ Field modifiersField = null;
+ for (Field each : fields) {
+ if ("modifiers".equals(each.getName())) {
+ modifiersField = each;
+ break;
+ }
+ }
+
+ modifiersField.setAccessible(true);
+ modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+
+ Map<String, String> customMap = new HashMap<>(StandardIndexProvider.getAllProviderClasses());
+
+ customMap.put(INDEX_BACKEND_OPENSEARCH, AtlasOpenSearchIndex.class.getName());
+
+ ImmutableMap<String, String> immap = ImmutableMap.copyOf(customMap);
+
+ field.set(null, immap);
+
+ LOG.debug("Injected opensearch index - {}", AtlasOpenSearchIndex.class.getName());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
private static void updateGlobalConfiguration(Map<String, Object> map) {
JanusGraph graph = null;
JanusGraphManagement managementSystem = null;
@@ -527,6 +561,8 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
addSolr6Index();
addElasticSearch7Index();
+
+ addOpenSearchIndex();
}
public static void logArgs() {
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
index 8b81d54ef..1ab331317 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
@@ -24,6 +24,8 @@ import org.apache.atlas.model.discovery.AtlasAggregationEntry;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AggregationContext;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
+import org.apache.atlas.repository.graphdb.QuickSearchContext;
+import org.apache.atlas.repository.graphdb.QuickSearchResult;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
@@ -44,6 +46,8 @@ import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.janusgraph.diskstorage.es.ElasticSearch7Index;
import org.janusgraph.diskstorage.es.ElasticSearchClient;
+import org.janusgraph.diskstorage.opensearch.AtlasOpenSearchIndex;
+import org.janusgraph.diskstorage.opensearch.OpenSearchClient;
import org.janusgraph.diskstorage.solr.Solr6Index;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,7 +70,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphIndexClient.class);
private static final FreqComparator FREQ_COMPARATOR = new FreqComparator();
- private static final int DEFAULT_SUGGESTION_COUNT = 5;
+ static final int DEFAULT_SUGGESTION_COUNT = 5;
private static final int MIN_FACET_COUNT_REQUIRED = 1;
private static final String TERMS_PREFIX = "terms.prefix";
private static final String TERMS_FIELD = "terms.fl";
@@ -162,6 +166,12 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
@Override
public Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(AggregationContext aggregationContext) {
+ String idxBackEnd = configuration != null ? configuration.getString(ApplicationProperties.INDEX_BACKEND_CONF) : null;
+
+ if (ApplicationProperties.INDEX_BACKEND_OPENSEARCH.equals(idxBackEnd)) {
+ return AtlasOpenSearchIndexClient.getAggregatedMetrics(aggregationContext, configuration);
+ }
+
SolrClient solrClient = null;
try {
@@ -266,6 +276,12 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
@Override
public List<String> getSuggestions(String prefixString, String indexFieldName) {
+ String idxBackEnd = configuration != null ? configuration.getString(ApplicationProperties.INDEX_BACKEND_CONF) : null;
+
+ if (ApplicationProperties.INDEX_BACKEND_OPENSEARCH.equals(idxBackEnd)) {
+ return AtlasOpenSearchIndexClient.getSuggestions(prefixString, indexFieldName, configuration);
+ }
+
SolrClient solrClient = null;
try {
@@ -322,8 +338,27 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return Collections.emptyList();
}
+ @Override
+ public QuickSearchResult quickSearch(QuickSearchContext quickSearchContext) throws AtlasBaseException {
+ String idxBackEnd = configuration != null ? configuration.getString(ApplicationProperties.INDEX_BACKEND_CONF) : null;
+
+ if (ApplicationProperties.INDEX_BACKEND_OPENSEARCH.equals(idxBackEnd)) {
+ return AtlasOpenSearchIndexClient.quickSearch(quickSearchContext, configuration);
+ }
+
+ return new QuickSearchResult(Collections.emptyList(), 0L);
+ }
+
@Override
public void applySearchWeight(String collectionName, Map<String, Integer> indexFieldName2SearchWeightMap) {
+ String idxBackEnd = configuration != null ? configuration.getString(ApplicationProperties.INDEX_BACKEND_CONF) : null;
+
+ if (ApplicationProperties.INDEX_BACKEND_OPENSEARCH.equals(idxBackEnd)) {
+ AtlasOpenSearchIndexClient.applySearchWeight(indexFieldName2SearchWeightMap);
+
+ return;
+ }
+
SolrClient solrClient = null;
try {
@@ -395,6 +430,14 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
@Override
public void applySuggestionFields(String collectionName, List<String> suggestionProperties) {
+ String idxBackEnd = configuration != null ? configuration.getString(ApplicationProperties.INDEX_BACKEND_CONF) : null;
+
+ if (ApplicationProperties.INDEX_BACKEND_OPENSEARCH.equals(idxBackEnd)) {
+ AtlasOpenSearchIndexClient.applySuggestionFields(suggestionProperties);
+
+ return;
+ }
+
SolrClient solrClient = null;
try {
@@ -421,6 +464,15 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
LOG.info("Applied suggestion fields request handler for collection {}.", collectionName);
}
+ @Override
+ public void applyKeywordSubfieldFields(String collectionName, Set<String> indexFieldNames) {
+ String idxBackEnd = configuration != null ? configuration.getString(ApplicationProperties.INDEX_BACKEND_CONF) : null;
+
+ if (ApplicationProperties.INDEX_BACKEND_OPENSEARCH.equals(idxBackEnd)) {
+ AtlasOpenSearchIndexClient.setKeywordSubfieldIndexFields(indexFieldNames);
+ }
+ }
+
public boolean isHealthy() {
boolean isHealthy = false;
long currentTime = System.currentTimeMillis();
@@ -431,6 +483,8 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
isHealthy = isSolrHealthy();
} else if (ApplicationProperties.INDEX_BACKEND_ELASTICSEARCH.equals(idxBackEnd)) {
isHealthy = isElasticsearchHealthy();
+ } else if (ApplicationProperties.INDEX_BACKEND_OPENSEARCH.equals(idxBackEnd)) {
+ isHealthy = isOpenSearchHealthy();
}
LOG.info("indexBackEnd={}; isHealthy={}", idxBackEnd, isHealthy);
@@ -460,7 +514,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
" 'terms.limit': 5 , \n" +
" 'terms.fl' : \n" +
" [\n" +
- " %s \n" +
+ " %x \n" +
" ] \n" +
" }\n" +
" 'components': " + "[ \n" +
@@ -483,6 +537,13 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
return client != null && client.indexExists(janusVertexIndex);
}
+ private boolean isOpenSearchHealthy() throws IOException {
+ OpenSearchClient client = AtlasOpenSearchIndex.getOpenSearchClient();
+ String janusVertexIndex = ApplicationProperties.DEFAULT_INDEX_NAME + "_" + Constants.VERTEX_INDEX;
+
+ return client != null && client.indexExists(janusVertexIndex);
+ }
+
private SolrResponse updateFreeTextRequestHandler(SolrClient solrClient, String collectionName, Map<String, Integer> indexFieldName2SearchWeightMap, Solr6Index.Mode mode) throws IOException, SolrServerException, AtlasBaseException {
String searchWeightString = generateSearchWeightString(indexFieldName2SearchWeightMap);
String payLoadString = generatePayLoadForFreeText("update-requesthandler", searchWeightString);
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
index 824733e01..c58eabde9 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
@@ -17,6 +17,7 @@
*/
package org.apache.atlas.repository.graphdb.janus;
+import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
@@ -57,6 +58,7 @@ import org.janusgraph.graphdb.log.StandardTransactionLogProcessor;
import org.janusgraph.graphdb.transaction.StandardJanusGraphTx;
import org.janusgraph.graphdb.types.IndexType;
import org.janusgraph.graphdb.types.MixedIndexType;
+import org.janusgraph.graphdb.types.ParameterType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -84,6 +86,21 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
private static final boolean lockEnabled = AtlasConfiguration.STORAGE_CONSISTENCY_LOCK_ENABLED.getBoolean();
private static final Parameter<?>[] STRING_PARAMETER_ARRAY = new Parameter[] {Mapping.STRING.asParameter()};
+ private static final Parameter<?>[] TEXT_WITH_KEYWORD_SUBFIELD_PARAMETERS = new Parameter[] {
+ Mapping.TEXT.asParameter(),
+ Parameter.of(ParameterType.customParameterName("fields"), createKeywordSubfieldsParameter())
+ };
+
+ private static Map<String, Object> createKeywordSubfieldsParameter() {
+ Map<String, Object> keywordMapping = new HashMap<>();
+ keywordMapping.put("type", "keyword");
+
+ Map<String, Object> fields = new HashMap<>();
+ fields.put("keyword", keywordMapping);
+
+ return fields;
+ }
+
private static final char[] RESERVED_CHARS = {'{', '}', '"', '$', Token.SEPARATOR_CHAR};
private final AtlasJanusGraph graph;
@@ -328,26 +345,24 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
}
@Override
- public String addMixedIndex(String indexName, AtlasPropertyKey propertyKey, boolean isStringField) {
+ public String addMixedIndex(String indexName, AtlasPropertyKey propertyKey, boolean isStringField, boolean withKeywordSubfield) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(propertyKey);
JanusGraphIndex janusGraphIndex = management.getGraphIndex(indexName);
+ Parameter<?>[] indexParameters = resolveIndexParameters(isStringField, withKeywordSubfield);
- if (isStringField) {
- management.addIndexKey(janusGraphIndex, janusKey, Mapping.STRING.asParameter());
-
- LOG.debug("created a string type for {} with janueKey {}.", propertyKey.getName(), janusKey);
- } else {
+ if (indexParameters.length == 0) {
management.addIndexKey(janusGraphIndex, janusKey);
-
LOG.debug("created a default type for {} with janueKey {}.", propertyKey.getName(), janusKey);
+ } else {
+ management.addIndexKey(janusGraphIndex, janusKey, indexParameters);
+ LOG.debug("created indexed type for {} with janueKey {} using {} parameter(s).",
+ propertyKey.getName(), janusKey, indexParameters.length);
}
- String encodedName;
+ String encodedName = graph.getIndexFieldName(propertyKey, janusGraphIndex, indexParameters);
- if (isStringField) {
- encodedName = graph.getIndexFieldName(propertyKey, janusGraphIndex, STRING_PARAMETER_ARRAY);
- } else {
- encodedName = graph.getIndexFieldName(propertyKey, janusGraphIndex);
+ if (usesKeywordSubfieldMapping(isStringField, withKeywordSubfield)) {
+ AtlasOpenSearchIndexClient.registerKeywordSubfieldField(encodedName);
}
LOG.info("property '{}' is encoded to '{}'.", propertyKey.getName(), encodedName);
@@ -356,13 +371,34 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
}
@Override
- public String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey, boolean isStringField) {
+ public String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey, boolean isStringField, boolean withKeywordSubfield) {
JanusGraphIndex janusGraphIndex = management.getGraphIndex(indexName);
+ return graph.getIndexFieldName(propertyKey, janusGraphIndex, resolveIndexParameters(isStringField, withKeywordSubfield));
+ }
+
+ private static Parameter<?>[] resolveIndexParameters(boolean isStringField, boolean withKeywordSubfield) {
if (isStringField) {
- return graph.getIndexFieldName(propertyKey, janusGraphIndex, STRING_PARAMETER_ARRAY);
- } else {
- return graph.getIndexFieldName(propertyKey, janusGraphIndex);
+ return STRING_PARAMETER_ARRAY;
+ }
+
+ if (usesKeywordSubfieldMapping(isStringField, withKeywordSubfield)) {
+ return TEXT_WITH_KEYWORD_SUBFIELD_PARAMETERS;
+ }
+
+ return new Parameter[0];
+ }
+
+ private static boolean usesKeywordSubfieldMapping(boolean isStringField, boolean withKeywordSubfield) {
+ return withKeywordSubfield && !isStringField && isOpenSearchIndexBackend();
+ }
+
+ private static boolean isOpenSearchIndexBackend() {
+ try {
+ return ApplicationProperties.INDEX_BACKEND_OPENSEARCH.equals(
+ ApplicationProperties.get().getString(ApplicationProperties.INDEX_BACKEND_CONF));
+ } catch (Exception e) {
+ return false;
}
}
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchIndexClient.java
new file mode 100644
index 000000000..11e242790
--- /dev/null
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchIndexClient.java
@@ -0,0 +1,672 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.AtlasAggregationEntry;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AggregationContext;
+import org.apache.atlas.repository.graphdb.QuickSearchContext;
+import org.apache.atlas.repository.graphdb.QuickSearchResult;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.lang3.StringUtils;
+import org.janusgraph.diskstorage.opensearch.AtlasOpenSearchIndex;
+import org.janusgraph.diskstorage.opensearch.OpenSearchClient;
+import org.janusgraph.diskstorage.opensearch.rest.RestSearchHit;
+import org.janusgraph.diskstorage.opensearch.rest.RestSearchResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * OpenSearch-native Atlas discovery helper for suggestions and aggregations. Uses
+ * {@link OpenSearchClient#search} with terms aggregations and keeps suggestion-field configuration
+ * in memory.
+ */
+public final class AtlasOpenSearchIndexClient {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasOpenSearchIndexClient.class);
+
+ private static final String ATLAS_INDEX_NAME_CONF = "atlas.graph.index.search.index-name";
+
+ private static final int SUGGESTION_BUCKET_MULTIPLE = 4;
+ private static final int MIN_AGG_DOC_COUNT = 1;
+ private static final int MAX_AGG_BUCKET_SIZE = 1000;
+
+ /** OpenSearch {@code terms.include} regex metacharacters (Lucence regex dialect). */
+ private static final String TERMS_INCLUDE_REGEX_METACHARACTERS = "\\.[]{}()*+-?^$|_";
+
+ private static volatile Set<String> suggestionIndexFields = Collections.emptySet();
+ private static volatile Map<String, Integer> searchWeightByIndexField = Collections.emptyMap();
+ private static volatile Set<String> keywordSubfieldIndexFields = Collections.emptySet();
+
+ private AtlasOpenSearchIndexClient() {
+ }
+
+ public static void applySearchWeight(Map<String, Integer> indexFieldName2SearchWeightMap) {
+ if (indexFieldName2SearchWeightMap == null || indexFieldName2SearchWeightMap.isEmpty()) {
+ searchWeightByIndexField = Collections.emptyMap();
+ } else {
+ searchWeightByIndexField = Collections.unmodifiableMap(new HashMap<>(indexFieldName2SearchWeightMap));
+ }
+
+ LOG.info("Applied OpenSearch search weights (count={}).", searchWeightByIndexField.size());
+ }
+
+ static Map<String, Integer> getSearchWeightByIndexField() {
+ return searchWeightByIndexField;
+ }
+
+ public static QuickSearchResult quickSearch(QuickSearchContext quickSearchContext, Configuration configuration) throws AtlasBaseException {
+ OpenSearchClient client = AtlasOpenSearchIndex.getOpenSearchClient();
+
+ if (client == null) {
+ // Not an OpenSearch-backed deployment: an empty result is the documented contract, not a backend failure.
+ LOG.warn("The indexing system is not OpenSearch based. Will return empty quick-search results.");
+
+ return new QuickSearchResult(Collections.emptyList(), 0L);
+ }
+
+ try {
+ Map<String, Object> query = new AtlasOpenSearchQueryBuilder()
+ .withEntityTypes(quickSearchContext.getEntityTypes())
+ .withQueryString(quickSearchContext.getQueryString())
+ .withCriteria(quickSearchContext.getFilterCriteria())
+ .withExcludedDeletedEntities(quickSearchContext.isExcludeDeletedEntities())
+ .withIncludeSubTypes(quickSearchContext.isIncludeSubTypes())
+ .withCommonIndexFieldNames(quickSearchContext.getIndexFieldNameCache())
+ .withClassificationTypeNames(quickSearchContext.getClassificationTypeNames())
+ .withSearchWeights(searchWeightByIndexField)
+ .buildDiscoveryQuery();
+
+ Map<String, Object> requestBody = new HashMap<>();
+
+ requestBody.put("query", query);
+ requestBody.put("from", quickSearchContext.getOffset());
+ requestBody.put("size", quickSearchContext.getLimit());
+ requestBody.put("track_total_hits", true);
+
+ String physicalIndex = resolveVertexIndexName(configuration);
+
+ LOG.debug("OpenSearch weighted quick-search: index={}, from={}, size={}", physicalIndex,
+ quickSearchContext.getOffset(), quickSearchContext.getLimit());
+
+ org.janusgraph.diskstorage.opensearch.OpenSearchResponse osResponse =
+ client.search(physicalIndex, requestBody, false);
+
+ if (!(osResponse instanceof RestSearchResponse)) {
+ // An unexpected response type indicates a backend/protocol problem; surface it rather than
+ // silently returning zero results (which could be mistaken for a legitimate empty search).
+ throw new AtlasBaseException("Unexpected OpenSearch quick-search response type: "
+ + (osResponse == null ? "null" : osResponse.getClass().getName()));
+ }
+
+ RestSearchResponse response = (RestSearchResponse) osResponse;
+ long total = response.getTotalHitCount();
+ List<String> guids = new ArrayList<>();
+
+ if (response.getHits() != null && CollectionUtils.isNotEmpty(response.getHits().getHits())) {
+ for (RestSearchHit hit : response.getHits().getHits()) {
+ String guid = resolveEntityGuid(hit);
+
+ if (StringUtils.isNotEmpty(guid)) {
+ guids.add(guid);
+ }
+ }
+ }
+
+ return new QuickSearchResult(guids, total);
+ } catch (AtlasBaseException e) {
+ // Includes query-building failures such as unsupported operators — propagate, do not swallow.
+ LOG.error("Error building or executing OpenSearch weighted quick search.", e);
+
+ throw e;
+ } catch (IOException e) {
+ // Backend outage/transport failure: propagate so callers do not mistake it for a zero-hit search.
+ LOG.error("OpenSearch backend failure during weighted quick search.", e);
+
+ throw new AtlasBaseException("OpenSearch quick search failed", e);
+ }
+ }
+
+ static String resolveEntityGuid(RestSearchHit hit) {
+ if (hit == null) {
+ return null;
+ }
+
+ Map<String, Object> source = hit.getSource();
+
+ if (source != null) {
+ Object guid = source.get("__guid");
+
+ if (guid != null) {
+ return String.valueOf(guid);
+ }
+ }
+
+ return hit.getId();
+ }
+
+ public static void applySuggestionFields(List<String> suggestionProperties) {
+ if (CollectionUtils.isEmpty(suggestionProperties)) {
+ suggestionIndexFields = Collections.emptySet();
+ } else {
+ suggestionIndexFields = Collections.unmodifiableSet(new LinkedHashSet<>(suggestionProperties));
+ }
+
+ LOG.info("Applied OpenSearch suggestion fields (count={}).", suggestionIndexFields.size());
+ }
+
+ public static void registerKeywordSubfieldField(String indexFieldName) {
+ if (StringUtils.isEmpty(indexFieldName)) {
+ return;
+ }
+
+ Set<String> updated = new LinkedHashSet<>(keywordSubfieldIndexFields);
+
+ updated.add(indexFieldName);
+ keywordSubfieldIndexFields = Collections.unmodifiableSet(updated);
+
+ LOG.debug("Registered OpenSearch keyword subfield for index field {}", indexFieldName);
+ }
+
+ public static void setKeywordSubfieldIndexFields(Set<String> indexFieldNames) {
+ if (CollectionUtils.isEmpty(indexFieldNames)) {
+ keywordSubfieldIndexFields = Collections.emptySet();
+ } else {
+ keywordSubfieldIndexFields = Collections.unmodifiableSet(new LinkedHashSet<>(indexFieldNames));
+ }
+
+ LOG.info("Applied OpenSearch keyword subfield fields (count={}).", keywordSubfieldIndexFields.size());
+ }
+
+ static Set<String> getKeywordSubfieldIndexFields() {
+ return keywordSubfieldIndexFields;
+ }
+
+ static void clearKeywordSubfieldFieldsForTests() {
+ keywordSubfieldIndexFields = Collections.emptySet();
+ }
+
+ static boolean usesKeywordSubfield(String indexFieldName) {
+ return StringUtils.isNotEmpty(indexFieldName) && keywordSubfieldIndexFields.contains(indexFieldName);
+ }
+
+ static Set<String> getSuggestionIndexFields() {
+ return suggestionIndexFields;
+ }
+
+ public static Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(AggregationContext aggregationContext,
+ Configuration configuration) {
+ OpenSearchClient client = AtlasOpenSearchIndex.getOpenSearchClient();
+
+ if (client == null) {
+ LOG.warn("The indexing system is not OpenSearch based. Will return empty Aggregation metrics.");
+
+ return Collections.emptyMap();
+ }
+
+ Set<String> aggregationCommonFields = aggregationContext.getAggregationFieldNames();
+ Set<AtlasAttribute> aggregationAttributes = aggregationContext.getAggregationAttributes();
+ Map<String, String> indexFieldNameCache = aggregationContext.getIndexFieldNameCache();
+
+ if (CollectionUtils.isEmpty(aggregationCommonFields) && CollectionUtils.isEmpty(aggregationAttributes)) {
+ LOG.warn("There are no aggregation fields or attributes provided. Will return empty metrics.");
+
+ return Collections.emptyMap();
+ }
+
+ try {
+ Map<String, Object> query = new AtlasOpenSearchQueryBuilder()
+ .withEntityTypes(aggregationContext.getSearchForEntityTypes())
+ .withQueryString(aggregationContext.getQueryString())
+ .withCriteria(aggregationContext.getFilterCriteria())
+ .withExcludedDeletedEntities(aggregationContext.isExcludeDeletedEntities())
+ .withIncludeSubTypes(aggregationContext.isIncludeSubTypes())
+ .withCommonIndexFieldNames(indexFieldNameCache)
+ .withSearchWeights(searchWeightByIndexField)
+ .buildDiscoveryQuery();
+
+ Map<String, Object> aggs = new HashMap<>();
+ Map<String, String> aggNameToPropertyKeyName = new HashMap<>();
+ int aggIdx = 0;
+
+ if (CollectionUtils.isNotEmpty(aggregationCommonFields)) {
+ for (String propertyName : aggregationCommonFields) {
+ String indexFieldName = indexFieldNameCache.get(propertyName);
+
+ if (StringUtils.isEmpty(indexFieldName)) {
+ continue;
+ }
+
+ String aggName = "agg_" + aggIdx++;
+
+ aggNameToPropertyKeyName.put(aggName, propertyName);
+ aggs.put(aggName, termsAgg(indexFieldName));
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(aggregationAttributes)) {
+ for (AtlasAttribute attribute : aggregationAttributes) {
+ String indexFieldName = attribute.getIndexFieldName();
+
+ if (StringUtils.isEmpty(indexFieldName)) {
+ indexFieldName = attribute.getQualifiedName();
+ }
+
+ if (StringUtils.isEmpty(indexFieldName)) {
+ continue;
+ }
+
+ String aggName = "agg_" + aggIdx++;
+
+ aggNameToPropertyKeyName.put(aggName, attribute.getQualifiedName());
+ aggs.put(aggName, termsAgg(indexFieldName));
+ }
+ }
+
+ if (aggs.isEmpty()) {
+ return Collections.emptyMap();
+ }
+
+ Map<String, Object> requestBody = new HashMap<>();
+
+ requestBody.put("size", 0);
+ requestBody.put("query", query);
+ requestBody.put("aggs", aggs);
+
+ String physicalIndex = resolveVertexIndexName(configuration);
+
+ LOG.debug("OpenSearch aggregation query: index={}, aggs={}", physicalIndex, aggNameToPropertyKeyName.values());
+
+ org.janusgraph.diskstorage.opensearch.OpenSearchResponse osResponse =
+ client.search(physicalIndex, requestBody, false);
+
+ if (!(osResponse instanceof RestSearchResponse)) {
+ LOG.warn("OpenSearch aggregation response type {}", osResponse.getClass().getName());
+
+ return Collections.emptyMap();
+ }
+
+ RestSearchResponse response = (RestSearchResponse) osResponse;
+ Map<String, Object> aggResponse = response.getAggregations();
+
+ if (aggResponse == null) {
+ return Collections.emptyMap();
+ }
+
+ Map<String, List<AtlasAggregationEntry>> ret = new HashMap<>();
+
+ for (Map.Entry<String, String> entry : aggNameToPropertyKeyName.entrySet()) {
+ Map<String, Object> aggResult = (Map<String, Object>) aggResponse.get(entry.getKey());
+ List<Map<String, Object>> buckets = aggResult == null ? null : (List<Map<String, Object>>) aggResult.get("buckets");
+ List<AtlasAggregationEntry> entries = new ArrayList<>();
+
+ if (CollectionUtils.isNotEmpty(buckets)) {
+ for (Map<String, Object> bucket : buckets) {
+ Object key = bucket.get("key");
+ Object docCount = bucket.get("doc_count");
+
+ if (key != null) {
+ entries.add(new AtlasAggregationEntry(String.valueOf(key),
+ docCount == null ? 0L : ((Number) docCount).longValue()));
+ }
+ }
+ }
+
+ ret.put(entry.getValue(), entries);
+ }
+
+ return ret;
+ } catch (AtlasBaseException | IOException e) {
+ LOG.error("Error encountered in getting the aggregation metrics. Will return empty aggregation.", e);
+ }
+
+ return Collections.emptyMap();
+ }
+
+ public static List<String> getSuggestions(String prefixString, String indexFieldName, Configuration configuration) {
+ OpenSearchClient client = AtlasOpenSearchIndex.getOpenSearchClient();
+
+ if (client == null) {
+ LOG.warn("The indexing system is not OpenSearch based. Suggestions feature will not be available.");
+
+ return Collections.emptyList();
+ }
+
+ List<String> fieldsToQuery = resolveFieldsToQuery(indexFieldName);
+
+ if (fieldsToQuery.isEmpty()) {
+ LOG.info("No index field name or suggestion fields configured for OpenSearch suggestions.");
+
+ return Collections.emptyList();
+ }
+
+ List<String> aggregationFields = resolveAggregationCompatibleFields(fieldsToQuery);
+
+ if (aggregationFields.isEmpty()) {
+ LOG.info("No aggregation-compatible suggestion fields available for OpenSearch suggestions.");
+
+ return Collections.emptyList();
+ }
+
+ try {
+ Map<String, Object> aggs = buildSuggestionsTermsAggs(aggregationFields, prefixString);
+
+ if (aggs.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ Map<String, Object> requestBody = new HashMap<>();
+
+ requestBody.put("size", 0);
+ requestBody.put("query", buildSuggestionsFilterQuery());
+ requestBody.put("aggs", aggs);
+
+ String physicalIndex = resolveVertexIndexName(configuration);
+
+ LOG.debug("OpenSearch suggestions query: index={}, prefix={}, fields={}, aggs={}",
+ physicalIndex, prefixString, aggregationFields, aggs.keySet());
+
+ org.janusgraph.diskstorage.opensearch.OpenSearchResponse osResponse =
+ client.search(physicalIndex, requestBody, false);
+
+ if (!(osResponse instanceof RestSearchResponse)) {
+ LOG.warn("OpenSearch suggestions response type {}", osResponse.getClass().getName());
+
+ return Collections.emptyList();
+ }
+
+ RestSearchResponse response = (RestSearchResponse) osResponse;
+ Map<String, AtlasJanusGraphIndexClient.TermFreq> termsMap =
+ collectTermsFromAggregations(response.getAggregations(), aggs.keySet());
+
+ return AtlasJanusGraphIndexClient.getTopTerms(termsMap);
+ } catch (IOException e) {
+ LOG.error("Error encountered in generating the suggestions. Ignoring the error", e);
+ }
+
+ return Collections.emptyList();
+ }
+
+ private static List<String> resolveFieldsToQuery(String indexFieldName) {
+ if (StringUtils.isNotEmpty(indexFieldName)) {
+ return Collections.singletonList(indexFieldName);
+ }
+
+ if (CollectionUtils.isNotEmpty(suggestionIndexFields)) {
+ return new ArrayList<>(suggestionIndexFields);
+ }
+
+ return Collections.emptyList();
+ }
+
+ /**
+ * Resolves Atlas index field names to OpenSearch {@code terms} aggregation targets.
+ * TEXT fields without a registered {@code .keyword} subfield are skipped to avoid fielddata errors.
+ */
+ static List<String> resolveAggregationCompatibleFields(List<String> indexFieldNames) {
+ List<String> ret = new ArrayList<>();
+
+ for (String indexFieldName : indexFieldNames) {
+ if (StringUtils.isEmpty(indexFieldName)) {
+ continue;
+ }
+
+ if (resolveTermsAggregationFieldName(indexFieldName) != null) {
+ ret.add(indexFieldName);
+ } else {
+ LOG.warn("Skipping OpenSearch suggestion field '{}' — not aggregation-compatible "
+ + "(TEXT field without keyword subfield).", indexFieldName);
+ }
+ }
+
+ return ret;
+ }
+
+ /**
+ * @return OpenSearch physical field for {@code terms} aggregation, or {@code null} if the field must be skipped.
+ */
+ static String resolveTermsAggregationFieldName(String indexFieldName) {
+ if (StringUtils.isEmpty(indexFieldName)) {
+ return null;
+ }
+
+ if (usesKeywordSubfield(indexFieldName)) {
+ return toOpenSearchFieldName(indexFieldName) + ".keyword";
+ }
+
+ // Legacy JanusGraph mapping: system fields as native keyword (no .keyword subfield).
+ if (Constants.STATE_PROPERTY_KEY.equals(indexFieldName)
+ || Constants.ENTITY_TYPE_PROPERTY_KEY.equals(indexFieldName)
+ || Constants.LABELS_PROPERTY_KEY.equals(indexFieldName)
+ || Constants.CLASSIFICATION_TEXT_KEY.equals(indexFieldName)) {
+ return toOpenSearchFieldName(indexFieldName);
+ }
+
+ if (isLikelyTextFieldWithoutKeywordSubfield(indexFieldName)) {
+ return null;
+ }
+
+ return toOpenSearchFieldName(indexFieldName);
+ }
+
+ /**
+ * Heuristic for misconfigured TEXT suggestion fields: high-weight entity attributes that are not STRING
+ * mappings should always have a {@code .keyword} subfield registered. When missing, skip rather than
+ * aggregating on a raw {@code text} field.
+ */
+ private static boolean isLikelyTextFieldWithoutKeywordSubfield(String indexFieldName) {
+ if (usesKeywordSubfield(indexFieldName)) {
+ return false;
+ }
+
+ if (Constants.STATE_PROPERTY_KEY.equals(indexFieldName)
+ || Constants.ENTITY_TYPE_PROPERTY_KEY.equals(indexFieldName)
+ || Constants.LABELS_PROPERTY_KEY.equals(indexFieldName)
+ || Constants.CLASSIFICATION_TEXT_KEY.equals(indexFieldName)) {
+ return keywordSubfieldIndexFields.isEmpty();
+ }
+
+ return false;
+ }
+
+ /**
+ * JanusGraph mixed-index documents use {@code \u2022} as the type/attribute delimiter in OpenSearch field names.
+ * Atlas callers may supply either the encoded or dot-separated property form.
+ */
+ static String toOpenSearchFieldName(String indexFieldName) {
+ if (StringUtils.isEmpty(indexFieldName) || indexFieldName.indexOf('\u2022') >= 0) {
+ return indexFieldName;
+ }
+
+ int dotIdx = indexFieldName.indexOf('.');
+
+ if (dotIdx > 0) {
+ return indexFieldName.substring(0, dotIdx) + '\u2022' + indexFieldName.substring(dotIdx + 1);
+ }
+
+ return indexFieldName;
+ }
+
+ static String toOpenSearchTermsFieldName(String indexFieldName) {
+ String resolved = resolveTermsAggregationFieldName(indexFieldName);
+
+ if (resolved != null) {
+ return resolved;
+ }
+
+ return toOpenSearchFieldName(indexFieldName);
+ }
+
+ /**
+ * Converts a user prefix to an OpenSearch {@code terms.include} regex (prefix match).
+ * Case-sensitive — keyword fields preserve indexed term casing, matching Solr /terms behavior.
+ */
+ static String toTermsIncludePattern(String prefixString) {
+ if (StringUtils.isEmpty(prefixString)) {
+ return ".*";
+ }
+
+ StringBuilder sb = new StringBuilder(prefixString.length() + 2);
+
+ for (int i = 0; i < prefixString.length(); i++) {
+ char c = prefixString.charAt(i);
+
+ if (TERMS_INCLUDE_REGEX_METACHARACTERS.indexOf(c) >= 0) {
+ sb.append('\\');
+ }
+
+ sb.append(c);
+ }
+
+ sb.append(".*");
+
+ return sb.toString();
+ }
+
+ static Map<String, Object> buildSuggestionsFilterQuery() {
+ List<Map<String, Object>> mustNotClauses = new ArrayList<>();
+ String stateTermsField = resolveTermsAggregationFieldName(Constants.STATE_PROPERTY_KEY);
+
+ if (StringUtils.isEmpty(stateTermsField)) {
+ stateTermsField = toOpenSearchFieldName(Constants.STATE_PROPERTY_KEY);
+ }
+
+ mustNotClauses.add(singleKeyMap("term",
+ singleKeyMap(stateTermsField, AtlasEntity.Status.DELETED.name())));
+
+ return singleKeyMap("bool", singleKeyMap("must_not", mustNotClauses));
+ }
+
+ static Map<String, Object> buildSuggestionsTermsAggs(List<String> indexFieldNames, String prefixString) {
+ Map<String, Object> aggs = new HashMap<>();
+ int idx = 0;
+
+ for (String indexFieldName : indexFieldNames) {
+ String osField = resolveTermsAggregationFieldName(indexFieldName);
+
+ if (StringUtils.isEmpty(osField)) {
+ continue;
+ }
+
+ aggs.put("sugg_" + idx++, suggestionsTermsAgg(osField, prefixString));
+ }
+
+ return aggs;
+ }
+
+ private static Map<String, Object> suggestionsTermsAgg(String openSearchFieldName, String prefixString) {
+ Map<String, Object> termsSpec = new HashMap<>();
+
+ termsSpec.put("field", openSearchFieldName);
+ termsSpec.put("size", AtlasJanusGraphIndexClient.DEFAULT_SUGGESTION_COUNT * SUGGESTION_BUCKET_MULTIPLE);
+ termsSpec.put("min_doc_count", MIN_AGG_DOC_COUNT);
+
+ if (StringUtils.isNotEmpty(prefixString)) {
+ // Solr /terms uses prefix matching on indexed terms; terms.include expects a regex.
+ termsSpec.put("include", toTermsIncludePattern(prefixString));
+ }
+
+ return Collections.singletonMap("terms", termsSpec);
+ }
+
+ static Map<String, AtlasJanusGraphIndexClient.TermFreq> collectTermsFromAggregations(Map<String, Object> aggregations,
+ Set<String> aggNames) {
+ Map<String, AtlasJanusGraphIndexClient.TermFreq> termsMap = new HashMap<>();
+
+ if (aggregations == null || CollectionUtils.isEmpty(aggNames)) {
+ return termsMap;
+ }
+
+ for (String aggName : aggNames) {
+ Map<String, Object> aggResult = (Map<String, Object>) aggregations.get(aggName);
+ List<Map<String, Object>> buckets = aggResult == null ? null : (List<Map<String, Object>>) aggResult.get("buckets");
+
+ mergeTermBuckets(termsMap, buckets);
+ }
+
+ return termsMap;
+ }
+
+ static void mergeTermBuckets(Map<String, AtlasJanusGraphIndexClient.TermFreq> termsMap,
+ List<Map<String, Object>> buckets) {
+ if (CollectionUtils.isEmpty(buckets)) {
+ return;
+ }
+
+ for (Map<String, Object> bucket : buckets) {
+ Object key = bucket.get("key");
+ Object docCount = bucket.get("doc_count");
+
+ if (key == null) {
+ continue;
+ }
+
+ String term = String.valueOf(key);
+ long freq = docCount == null ? 0L : ((Number) docCount).longValue();
+ AtlasJanusGraphIndexClient.TermFreq existing = termsMap.get(term);
+
+ if (existing == null) {
+ termsMap.put(term, new AtlasJanusGraphIndexClient.TermFreq(term, freq));
+ } else {
+ existing.addFreq(freq);
+ }
+ }
+ }
+
+ private static Map<String, Object> singleKeyMap(String key, Object value) {
+ Map<String, Object> ret = new HashMap<>();
+
+ ret.put(key, value);
+
+ return ret;
+ }
+
+ static String resolveVertexIndexName(Configuration configuration) {
+ String indexName = ApplicationProperties.DEFAULT_INDEX_NAME;
+
+ if (configuration != null) {
+ indexName = configuration.getString(ATLAS_INDEX_NAME_CONF,
+ configuration.getString(ApplicationProperties.OPENSEARCH_INDEX_NAME_CONF, ApplicationProperties.DEFAULT_INDEX_NAME));
+ }
+
+ return indexName + "_" + Constants.VERTEX_INDEX;
+ }
+
+ private static Map<String, Object> termsAgg(String indexFieldName) {
+ Map<String, Object> terms = new HashMap<>();
+
+ terms.put("field", toOpenSearchTermsFieldName(indexFieldName));
+ terms.put("size", MAX_AGG_BUCKET_SIZE);
+ terms.put("min_doc_count", MIN_AGG_DOC_COUNT);
+
+ return Collections.singletonMap("terms", terms);
+ }
+}
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchQueryBuilder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchQueryBuilder.java
new file mode 100644
index 000000000..eafd2a223
--- /dev/null
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasOpenSearchQueryBuilder.java
@@ -0,0 +1,727 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graphdb.janus;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.Operator;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.janusgraph.diskstorage.opensearch.OpenSearchConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.graphdb.janus.AtlasSolrQueryBuilder.CUSTOM_ATTR_SEARCH_FORMAT;
+import static org.apache.atlas.repository.graphdb.janus.AtlasSolrQueryBuilder.CUSTOM_ATTR_SEPARATOR;
+
+/**
+ * Builds an OpenSearch Query DSL {@code query} clause from the same Atlas inputs as
+ * {@link AtlasSolrQueryBuilder} (free-text query, entity-type filter, exclude-deleted flag,
+ * {@link FilterCriteria} tree). Used for OpenSearch discovery aggregations/suggestions filters.
+ */
+public class AtlasOpenSearchQueryBuilder {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasOpenSearchQueryBuilder.class);
+
+ private Set<AtlasEntityType> entityTypes;
+ private String queryString;
+ private FilterCriteria criteria;
+ private boolean excludeDeletedEntities;
+ private boolean includeSubtypes;
+ private Map<String, String> indexFieldNameCache;
+ private Map<String, Integer> searchWeights;
+ private Set<String> classificationTypeNames;
+
+ public AtlasOpenSearchQueryBuilder withEntityTypes(Set<AtlasEntityType> searchForEntityTypes) {
+ this.entityTypes = searchForEntityTypes;
+
+ return this;
+ }
+
+ public AtlasOpenSearchQueryBuilder withQueryString(String queryString) {
+ this.queryString = queryString;
+
+ return this;
+ }
+
+ public AtlasOpenSearchQueryBuilder withCriteria(FilterCriteria criteria) {
+ this.criteria = criteria;
+
+ return this;
+ }
+
+ public AtlasOpenSearchQueryBuilder withExcludedDeletedEntities(boolean excludeDeletedEntities) {
+ this.excludeDeletedEntities = excludeDeletedEntities;
+
+ return this;
+ }
+
+ public AtlasOpenSearchQueryBuilder withIncludeSubTypes(boolean includeSubTypes) {
+ this.includeSubtypes = includeSubTypes;
+
+ return this;
+ }
+
+ public AtlasOpenSearchQueryBuilder withCommonIndexFieldNames(Map<String, String> indexFieldNameCache) {
+ this.indexFieldNameCache = indexFieldNameCache;
+
+ return this;
+ }
+
+ public AtlasOpenSearchQueryBuilder withSearchWeights(Map<String, Integer> searchWeights) {
+ this.searchWeights = searchWeights;
+
+ return this;
+ }
+
+ public AtlasOpenSearchQueryBuilder withClassificationTypeNames(Set<String> classificationTypeNames) {
+ this.classificationTypeNames = classificationTypeNames;
+
+ return this;
+ }
+
+ /**
+ * @return OpenSearch Query DSL {@code query} clause for discovery (quick search hits and aggregations):
+ * Solr edismax-style {@code dis_max} over weighted fields for plain terms and wildcards; structural
+ * filters in {@code filter}.
+ */
+ public Map<String, Object> buildDiscoveryQuery() throws AtlasBaseException {
+ return buildWeightedQuickSearch();
+ }
+
+ /**
+ * @return OpenSearch Query DSL {@code query} clause for weighted quick search: Solr edismax-style
+ * {@code dis_max} when weights are configured (or {@code query_string} fallback); structural filters
+ * in {@code filter}.
+ */
+ public Map<String, Object> buildWeightedQuickSearch() throws AtlasBaseException {
+ List<Map<String, Object>> mustClauses = new ArrayList<>();
+ List<Map<String, Object>> filterClauses = new ArrayList<>();
+ List<Map<String, Object>> mustNotClauses = new ArrayList<>();
+
+ if (StringUtils.isNotEmpty(queryString)) {
+ Map<String, Object> textClause = buildFreeTextClause(queryString.trim());
+
+ if (textClause != null) {
+ mustClauses.add(textClause);
+ }
+ }
+
+ if (excludeDeletedEntities) {
+ String rawStateFieldName = indexFieldNameCache != null
+ ? indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY) : null;
+
+ if (StringUtils.isEmpty(rawStateFieldName)) {
+ throw new AtlasBaseException(String.format("There is no index field name defined for attribute '%s'",
+ Constants.STATE_PROPERTY_KEY));
+ }
+
+ // Exact-match on __state must target the keyword field when the mapping uses a text+keyword subfield;
+ // an analyzed text field would index "DELETED" lower-cased and miss the term filter.
+ mustNotClauses.add(singleKeyMap("term", singleKeyMap(toKeywordField(rawStateFieldName), AtlasEntity.Status.DELETED.name())));
+ }
+
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
+ filterClauses.add(buildEntityTypeClause());
+ }
+
+ if (CollectionUtils.isNotEmpty(classificationTypeNames)) {
+ Map<String, Object> classificationClause = buildClassificationTypeClause();
+
+ if (classificationClause != null) {
+ filterClauses.add(classificationClause);
+ }
+ }
+
+ if (criteria != null) {
+ Map<String, Object> criteriaClause = buildCriteria(criteria);
+
+ if (criteriaClause != null) {
+ filterClauses.add(criteriaClause);
+ }
+ }
+
+ if (mustClauses.isEmpty() && filterClauses.isEmpty() && mustNotClauses.isEmpty()) {
+ return singleKeyMap("match_all", new HashMap<>());
+ }
+
+ Map<String, Object> bool = new HashMap<>();
+
+ if (!mustClauses.isEmpty()) {
+ bool.put("must", mustClauses);
+ }
+
+ if (!filterClauses.isEmpty()) {
+ bool.put("filter", filterClauses);
+ }
+
+ if (!mustNotClauses.isEmpty()) {
+ bool.put("must_not", mustNotClauses);
+ }
+
+ return singleKeyMap("bool", bool);
+ }
+
+ private Map<String, Object> buildFreeTextClause(String trimmedQuery) {
+ if (searchWeights != null && !searchWeights.isEmpty()) {
+ List<Map<String, Object>> disMaxQueries = buildDisMaxWeightedQueries(trimmedQuery);
+
+ if (!disMaxQueries.isEmpty()) {
+ Map<String, Object> disMax = new HashMap<>();
+
+ disMax.put("queries", disMaxQueries);
+ disMax.put("tie_breaker", 0.0);
+
+ return singleKeyMap("dis_max", disMax);
+ }
+ }
+
+ LOG.debug("No search weights configured; using query_string fallback for quick search.");
+
+ return buildWeightedQueryStringClause(trimmedQuery);
+ }
+
+ /**
+ * One dis_max sub-query per weighted index field — mirrors Solr edismax {@code qf=name^10 owner^3}.
+ */
+ private List<Map<String, Object>> buildDisMaxWeightedQueries(String trimmedQuery) {
+ List<Map<String, Object>> queries = new ArrayList<>();
+
+ if (searchWeights == null || searchWeights.isEmpty()) {
+ return queries;
+ }
+
+ boolean trailingWildcard = trimmedQuery.endsWith("*")
+ && trimmedQuery.indexOf('*') == trimmedQuery.length() - 1
+ && trimmedQuery.indexOf('?') < 0;
+ String prefixTerm = trailingWildcard ? trimmedQuery.substring(0, trimmedQuery.length() - 1) : trimmedQuery;
+
+ for (Map.Entry<String, Integer> entry : searchWeights.entrySet()) {
+ if (StringUtils.isEmpty(entry.getKey()) || entry.getValue() == null) {
+ continue;
+ }
+
+ String fieldName = toOsField(entry.getKey());
+ float boost = entry.getValue().floatValue();
+
+ Map<String, Object> fieldQuery = new HashMap<>();
+
+ fieldQuery.put("query", prefixTerm);
+ fieldQuery.put("boost", boost);
+
+ if (trailingWildcard) {
+ Map<String, Object> prefixParams = new HashMap<>();
+
+ prefixParams.put("value", prefixTerm);
+ prefixParams.put("boost", boost);
+ queries.add(singleKeyMap("prefix", singleKeyMap(fieldName, prefixParams)));
+ } else if (containsWildcard(trimmedQuery)) {
+ fieldQuery.put("query", trimmedQuery);
+ fieldQuery.put("default_field", fieldName);
+ fieldQuery.put("default_operator", "AND");
+ queries.add(singleKeyMap("query_string", fieldQuery));
+ } else if (containsQueryStringMetacharacters(trimmedQuery)) {
+ Map<String, Object> qsParams = new HashMap<>();
+
+ qsParams.put("query", escapeFreeTextQuery(trimmedQuery));
+ qsParams.put("default_field", fieldName);
+ qsParams.put("default_operator", "AND");
+ qsParams.put("boost", boost);
+ queries.add(singleKeyMap("query_string", qsParams));
+ } else {
+ queries.add(singleKeyMap("match", singleKeyMap(fieldName, fieldQuery)));
+ }
+ }
+
+ if (!trailingWildcard && !containsWildcard(trimmedQuery)) {
+ Map<String, Object> allFieldQuery = new HashMap<>();
+
+ allFieldQuery.put("query", trimmedQuery);
+ queries.add(singleKeyMap("match", singleKeyMap(OpenSearchConstants.CUSTOM_ALL_FIELD, allFieldQuery)));
+ }
+
+ return queries;
+ }
+
+ private Map<String, Object> buildWeightedQueryStringClause(String trimmedQuery) {
+ Map<String, Object> queryStringParams = new HashMap<>();
+
+ queryStringParams.put("query", escapeFreeTextQuery(trimmedQuery));
+ queryStringParams.put("default_operator", "AND");
+
+ List<String> weightedFields = buildWeightedFieldList();
+
+ if (!weightedFields.isEmpty()) {
+ queryStringParams.put("fields", weightedFields);
+ }
+
+ return singleKeyMap("query_string", queryStringParams);
+ }
+
+ private List<String> buildWeightedFieldList() {
+ List<String> weightedFields = new ArrayList<>();
+
+ if (searchWeights == null || searchWeights.isEmpty()) {
+ return weightedFields;
+ }
+
+ for (Map.Entry<String, Integer> entry : searchWeights.entrySet()) {
+ if (StringUtils.isNotEmpty(entry.getKey()) && entry.getValue() != null) {
+ weightedFields.add(toOsField(entry.getKey()) + "^" + entry.getValue());
+ }
+ }
+
+ return weightedFields;
+ }
+
+ static boolean containsWildcard(String query) {
+ if (StringUtils.isEmpty(query)) {
+ return false;
+ }
+
+ for (int i = 0; i < query.length(); i++) {
+ char c = query.charAt(i);
+
+ if (c == '\\' && i + 1 < query.length()) {
+ i++;
+
+ continue;
+ }
+
+ if (c == '*' || c == '?') {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private static String escapeFreeTextQuery(String query) {
+ if (StringUtils.isEmpty(query)) {
+ return query;
+ }
+
+ return AtlasAttribute.escapeIndexQueryValue(query, containsWildcard(query));
+ }
+
+ private static boolean containsQueryStringMetacharacters(String query) {
+ if (StringUtils.isEmpty(query) || containsWildcard(query)) {
+ return false;
+ }
+
+ for (int i = 0; i < query.length(); i++) {
+ char c = query.charAt(i);
+
+ if (c == '\\' && i + 1 < query.length()) {
+ i++;
+
+ continue;
+ }
+
+ switch (c) {
+ case ':':
+ case '+':
+ case '!':
+ case '(':
+ case ')':
+ case '{':
+ case '}':
+ case '[':
+ case ']':
+ case '^':
+ case '"':
+ case '~':
+ case '/':
+ case '&':
+ case '|':
+ return true;
+ default:
+ break;
+ }
+ }
+
+ return false;
+ }
+
+ private Map<String, Object> buildClassificationTypeClause() {
+ // terms on classification-name fields is an exact-match filter; resolve to the keyword subfield when mapped.
+ String classIndexFieldName = toKeywordField(indexFieldNameCache.get(CLASSIFICATION_NAMES_KEY));
+ String propagatedIndexFieldName = toKeywordField(indexFieldNameCache.get(PROPAGATED_CLASSIFICATION_NAMES_KEY));
+
+ if (StringUtils.isEmpty(classIndexFieldName) || StringUtils.isEmpty(propagatedIndexFieldName)) {
+ LOG.warn("Missing index field names for classification filters; skipping OpenSearch classification filter.");
+
+ return null;
+ }
+
+ List<String> types = new ArrayList<>(classificationTypeNames);
+
+ List<Map<String, Object>> shouldClauses = new ArrayList<>();
+
+ shouldClauses.add(singleKeyMap("terms", singleKeyMap(classIndexFieldName, types)));
+ shouldClauses.add(singleKeyMap("terms", singleKeyMap(propagatedIndexFieldName, types)));
+
+ return singleKeyMap("bool", singleKeyMap("should", shouldClauses));
+ }
+
+ /**
+ * @return OpenSearch Query DSL {@code query} clause for legacy callers; prefer {@link #buildDiscoveryQuery()}.
+ */
+ public Map<String, Object> build() throws AtlasBaseException {
+ List<Map<String, Object>> mustClauses = new ArrayList<>();
+ List<Map<String, Object>> mustNotClauses = new ArrayList<>();
+
+ if (StringUtils.isNotEmpty(queryString)) {
+ LOG.debug("Initial query string is {}.", queryString);
+
+ Map<String, Object> queryStringParams = new HashMap<>();
+
+ queryStringParams.put("query", escapeFreeTextQuery(queryString.trim()));
+ queryStringParams.put("default_operator", "AND");
+
+ mustClauses.add(singleKeyMap("query_string", queryStringParams));
+ }
+
+ if (excludeDeletedEntities) {
+ String rawStateFieldName = indexFieldNameCache != null
+ ? indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY) : null;
+
+ if (StringUtils.isEmpty(rawStateFieldName)) {
+ throw new AtlasBaseException(String.format("There is no index field name defined for attribute '%s'",
+ Constants.STATE_PROPERTY_KEY));
+ }
+
+ mustNotClauses.add(singleKeyMap("term", singleKeyMap(toKeywordField(rawStateFieldName), AtlasEntity.Status.DELETED.name())));
+ }
+
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
+ mustClauses.add(buildEntityTypeClause());
+ }
+
+ if (criteria != null) {
+ Map<String, Object> criteriaClause = buildCriteria(criteria);
+
+ if (criteriaClause != null) {
+ mustClauses.add(criteriaClause);
+ }
+ }
+
+ if (mustClauses.isEmpty() && mustNotClauses.isEmpty()) {
+ return singleKeyMap("match_all", new HashMap<>());
+ }
+
+ Map<String, Object> bool = new HashMap<>();
+
+ if (!mustClauses.isEmpty()) {
+ bool.put("must", mustClauses);
+ }
+
+ if (!mustNotClauses.isEmpty()) {
+ bool.put("must_not", mustNotClauses);
+ }
+
+ return singleKeyMap("bool", bool);
+ }
+
+ private Map<String, Object> buildEntityTypeClause() {
+ // terms on __typeName is an exact-match filter — resolve to the keyword subfield when the mapping requires it.
+ String typeIndexFieldName = toKeywordField(indexFieldNameCache.get(Constants.ENTITY_TYPE_PROPERTY_KEY));
+ Set<String> typesToSearch = new HashSet<>();
+
+ for (AtlasEntityType type : entityTypes) {
+ if (includeSubtypes) {
+ typesToSearch.addAll(type.getTypeAndAllSubTypes());
+ } else {
+ typesToSearch.add(type.getTypeName());
+ }
+ }
+
+ return singleKeyMap("terms", singleKeyMap(typeIndexFieldName, new ArrayList<>(typesToSearch)));
+ }
+
+ private Map<String, Object> buildCriteria(FilterCriteria criteria) throws AtlasBaseException {
+ List<FilterCriteria> criterion = criteria.getCriterion();
+
+ if (StringUtils.isNotEmpty(criteria.getAttributeName()) && CollectionUtils.isEmpty(criterion)) {
+ return buildLeafCriteria(criteria);
+ } else if (CollectionUtils.isNotEmpty(criterion)) {
+ List<Map<String, Object>> childClauses = new ArrayList<>();
+
+ for (FilterCriteria childCriteria : criterion) {
+ Map<String, Object> childClause = buildCriteria(childCriteria);
+
+ if (childClause != null) {
+ childClauses.add(childClause);
+ }
+ }
+
+ if (childClauses.isEmpty()) {
+ return null;
+ }
+
+ String condition = criteria.getCondition() != null ? criteria.getCondition().name() : FilterCriteria.Condition.AND.name();
+ boolean isAnd = FilterCriteria.Condition.AND.name().equalsIgnoreCase(condition);
+
+ return singleKeyMap("bool", singleKeyMap(isAnd ? "must" : "should", childClauses));
+ }
+
+ return null;
+ }
+
+ private Map<String, Object> buildLeafCriteria(FilterCriteria criteria) throws AtlasBaseException {
+ String attributeName = criteria.getAttributeName();
+ String attributeValue = criteria.getAttributeValue();
+ Operator operator = criteria.getOperator();
+
+ List<Map<String, Object>> orClauses = new ArrayList<>();
+ Set<String> indexAttributes = new HashSet<>();
+
+ for (AtlasEntityType type : entityTypes) {
+ String rawIndexAttributeName = getIndexAttributeName(type, attributeName);
+ String indexAttributeName = toOsField(rawIndexAttributeName);
+
+ if (!indexAttributes.contains(indexAttributeName)) {
+ indexAttributes.add(indexAttributeName);
+
+ if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) {
+ if (operator.equals(Operator.CONTAINS)) {
+ operator = Operator.EQ;
+ } else if (operator.equals(Operator.NOT_CONTAINS)) {
+ operator = Operator.NEQ;
+ }
+
+ attributeValue = getIndexQueryAttributeValue(attributeValue);
+ }
+
+ if (attributeValue != null) {
+ attributeValue = attributeValue.trim();
+ }
+
+ boolean replaceWildcardChar = false;
+ AtlasStructDef.AtlasAttributeDef def = type.getAttributeDef(attributeName);
+
+ if (!isPipeSeparatedSystemAttribute(attributeName) && isWildCardOperator(operator)
+ && def.getTypeName().equalsIgnoreCase(AtlasBaseTypeDef.ATLAS_TYPE_STRING)) {
+ if (def.getIndexType() == null && AtlasAttribute.hastokenizeChar(attributeValue)) {
+ replaceWildcardChar = true;
+ }
+ }
+
+ Map<String, Object> clause = buildOperatorClause(rawIndexAttributeName, operator, attributeValue, replaceWildcardChar);
+
+ if (clause != null) {
+ orClauses.add(clause);
+ }
+ }
+ }
+
+ if (orClauses.isEmpty()) {
+ return null;
+ }
+
+ return orClauses.size() == 1 ? orClauses.get(0) : singleKeyMap("bool", singleKeyMap("should", orClauses));
+ }
+
+ private Map<String, Object> buildOperatorClause(String rawIndexFieldName, Operator operator, String attributeValue,
+ boolean replaceWildCard) throws AtlasBaseException {
+ if (operator == null) {
+ return null;
+ }
+
+ // Free-text/wildcard/range operators target the analyzed (default) physical field; exact-match term
+ // operators (EQ/NEQ) must target the keyword subfield when the mapping uses text+keyword.
+ String indexFieldName = toOsField(rawIndexFieldName);
+ String exactFieldName = toKeywordField(rawIndexFieldName);
+
+ switch (operator) {
+ case EQ:
+ return singleKeyMap("term", singleKeyMap(exactFieldName, attributeValue));
+ case NEQ:
+ return singleKeyMap("bool", singleKeyMap("must_not", singleKeyMap("term", singleKeyMap(exactFieldName, attributeValue))));
+ case STARTS_WITH:
+ return wildcardClause(indexFieldName, toWildcardPattern(attributeValue, replaceWildCard, true, false));
+ case ENDS_WITH:
+ return wildcardClause(indexFieldName, toWildcardPattern(attributeValue, replaceWildCard, false, true));
+ case CONTAINS:
+ return wildcardClause(indexFieldName, toWildcardPattern(attributeValue, replaceWildCard, true, true));
+ case NOT_CONTAINS:
+ return singleKeyMap("bool", singleKeyMap("must_not", wildcardClause(indexFieldName,
+ toWildcardPattern(attributeValue, replaceWildCard, true, true))));
+ case IS_NULL:
+ return singleKeyMap("bool", singleKeyMap("must_not", singleKeyMap("exists", singleKeyMap("field", indexFieldName))));
+ case NOT_NULL:
+ return singleKeyMap("exists", singleKeyMap("field", indexFieldName));
+ case LT:
+ return singleKeyMap("range", singleKeyMap(indexFieldName, singleKeyMap("lt", attributeValue)));
+ case GT:
+ return singleKeyMap("range", singleKeyMap(indexFieldName, singleKeyMap("gt", attributeValue)));
+ case LTE:
+ return singleKeyMap("range", singleKeyMap(indexFieldName, singleKeyMap("lte", attributeValue)));
+ case GTE:
+ return singleKeyMap("range", singleKeyMap(indexFieldName, singleKeyMap("gte", attributeValue)));
+ case IN:
+ case LIKE:
+ case CONTAINS_ANY:
+ case CONTAINS_ALL:
+ default:
+ // Operator parity with the Solr quick-search contract (AtlasSolrQueryBuilder): IN, LIKE, CONTAINS_ANY,
+ // CONTAINS_ALL, TIME_RANGE and NOT_EMPTY are NOT supported by the Solr index-query builder for this
+ // path either (they fall through to the same "not supported" branch there). We intentionally do not
+ // invent OpenSearch-only semantics for them so the two backends stay behaviorally identical. This
+ // exception now propagates to the caller (see AtlasOpenSearchIndexClient#quickSearch) rather than being
+ // swallowed into an empty result.
+ String msg = String.format("%s is not supported operation.", operator.getSymbol());
+
+ LOG.error(msg);
+
+ throw new AtlasBaseException(msg);
+ }
+ }
+
+ private static Map<String, Object> wildcardClause(String indexFieldName, String pattern) {
+ return singleKeyMap("wildcard", singleKeyMap(indexFieldName, pattern));
+ }
+
+ /**
+ * OpenSearch wildcard patterns use {@code *} and {@code ?}; do not apply Solr quote/escape helpers.
+ */
+ private static String toWildcardPattern(String attributeValue, boolean replaceWildCard, boolean prefixStar, boolean suffixStar) {
+ if (attributeValue == null) {
+ return replaceWildCard ? "" : (prefixStar ? "*" : "") + (suffixStar ? "*" : "");
+ }
+
+ String escaped = escapeOpenSearchWildcard(attributeValue);
+
+ if (replaceWildCard) {
+ return escaped;
+ }
+
+ StringBuilder sb = new StringBuilder();
+
+ if (prefixStar) {
+ sb.append('*');
+ }
+
+ sb.append(escaped);
+
+ if (suffixStar) {
+ sb.append('*');
+ }
+
+ return sb.toString();
+ }
+
+ private static String escapeOpenSearchWildcard(String value) {
+ StringBuilder sb = new StringBuilder(value.length());
+
+ for (int i = 0; i < value.length(); i++) {
+ char c = value.charAt(i);
+
+ if (c == '*' || c == '?' || c == '\\') {
+ sb.append('\\');
+ }
+
+ sb.append(c);
+ }
+
+ return sb.toString();
+ }
+
+ private static boolean isPipeSeparatedSystemAttribute(String attrName) {
+ return StringUtils.equals(attrName, CLASSIFICATION_NAMES_KEY) ||
+ StringUtils.equals(attrName, PROPAGATED_CLASSIFICATION_NAMES_KEY) ||
+ StringUtils.equals(attrName, LABELS_PROPERTY_KEY) ||
+ StringUtils.equals(attrName, CUSTOM_ATTRIBUTES_PROPERTY_KEY);
+ }
+
+ private static boolean isWildCardOperator(Operator operator) {
+ return operator == Operator.CONTAINS ||
+ operator == Operator.STARTS_WITH ||
+ operator == Operator.ENDS_WITH ||
+ operator == Operator.NOT_CONTAINS;
+ }
+
+ private static String getIndexQueryAttributeValue(String attributeValue) {
+ if (StringUtils.isNotEmpty(attributeValue)) {
+ int separatorIdx = attributeValue.indexOf(CUSTOM_ATTR_SEPARATOR);
+ String key = separatorIdx != -1 ? attributeValue.substring(0, separatorIdx) : null;
+
+ if (key != null) {
+ String value = attributeValue.substring(separatorIdx + 1);
+
+ return String.format(CUSTOM_ATTR_SEARCH_FORMAT, key, value);
+ }
+ }
+
+ return attributeValue;
+ }
+
+ private String getIndexAttributeName(AtlasEntityType type, String attrName) throws AtlasBaseException {
+ AtlasAttribute ret = type.getAttribute(attrName);
+
+ if (ret == null) {
+ throw new AtlasBaseException(String.format("Received unknown attribute '%s' for type '%s'.", attrName, type.getTypeName()));
+ }
+
+ String indexFieldName = ret.getIndexFieldName();
+
+ if (indexFieldName == null) {
+ throw new AtlasBaseException(String.format("Received non-index attribute %s for type %s.", attrName, type.getTypeName()));
+ }
+
+ return indexFieldName;
+ }
+
+ static String toOsField(String indexFieldName) {
+ return AtlasOpenSearchIndexClient.toOpenSearchFieldName(indexFieldName);
+ }
+
+ /**
+ * Resolves the physical OpenSearch field for exact-match ({@code term}/{@code terms}) operations, appending the
+ * {@code .keyword} subfield when the field was registered as text+keyword (see
+ * {@link AtlasOpenSearchIndexClient#toOpenSearchTermsFieldName(String)}). For fields mapped as native keyword or
+ * as analyzed text without a keyword subfield, the bare field is returned (preserving prior behavior).
+ */
+ static String toKeywordField(String indexFieldName) {
+ return AtlasOpenSearchIndexClient.toOpenSearchTermsFieldName(indexFieldName);
+ }
+
+ private static Map<String, Object> singleKeyMap(String key, Object value) {
+ Map<String, Object> ret = new HashMap<>();
+
+ ret.put(key, value);
+
+ return ret;
+ }
+}
diff --git a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/opensearch/AtlasOpenSearchIndex.java b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/opensearch/AtlasOpenSearchIndex.java
new file mode 100644
index 000000000..0d46a4883
--- /dev/null
+++ b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/opensearch/AtlasOpenSearchIndex.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.janusgraph.diskstorage.opensearch;
+
+import org.janusgraph.diskstorage.BackendException;
+import org.janusgraph.diskstorage.configuration.Configuration;
+import org.janusgraph.graphdb.configuration.PreInitializeConfigOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Field;
+
+/**
+ * Thin Atlas/JanusGraph integration wrapper around {@link OpenSearchIndex} that exposes the
+ * underlying {@link OpenSearchClient} for Atlas discovery-layer operations.
+ */
+@PreInitializeConfigOptions
+public class AtlasOpenSearchIndex extends OpenSearchIndex {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasOpenSearchIndex.class);
+
+ private static volatile AtlasOpenSearchIndex instance;
+
+ private final OpenSearchClient client;
+
+ public AtlasOpenSearchIndex(Configuration config) throws BackendException {
+ super(config);
+
+ OpenSearchClient openSearchClient = null;
+
+ try {
+ Field fld = OpenSearchIndex.class.getDeclaredField("client");
+
+ fld.setAccessible(true);
+
+ openSearchClient = (OpenSearchClient) fld.get(this);
+ } catch (Exception excp) {
+ LOG.warn("Failed to get OpenSearchClient", excp);
+ }
+
+ this.client = openSearchClient;
+
+ AtlasOpenSearchIndex.instance = this;
+ }
+
+ public static OpenSearchClient getOpenSearchClient() {
+ AtlasOpenSearchIndex index = AtlasOpenSearchIndex.instance;
+
+ return index != null ? index.client : null;
+ }
+
+ @Override
+ public void close() throws org.janusgraph.diskstorage.BackendException {
+ // Clear the shared static reference before closing the underlying client so that discovery-layer callers
+ // (AtlasOpenSearchIndexClient) never obtain a client that is about to be, or already, closed. Clear it only
+ // when it still points to this instance: a short-lived bulk-loading index closing must not wipe a different
+ // (e.g. primary) graph's live client reference.
+ clearInstanceIfCurrent(this);
+
+ super.close();
+ }
+
+ /**
+ * Compare-and-clear of the shared static instance. Package/test visible so the concurrency-sensitive
+ * "only clear when the static still points to this instance" rule can be verified without a live cluster.
+ *
+ * @return {@code true} if the static reference was cleared, {@code false} if it pointed elsewhere (or was null)
+ */
+ static synchronized boolean clearInstanceIfCurrent(AtlasOpenSearchIndex index) {
+ if (instance == index) {
+ instance = null;
+
+ return true;
+ }
+
+ return false;
+ }
+
+ static AtlasOpenSearchIndex getInstanceForTests() {
+ return instance;
+ }
+
+ static void setInstanceForTests(AtlasOpenSearchIndex index) {
+ instance = index;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/pom.xml b/graphdb/janusgraph-opensearch/pom.xml
new file mode 100644
index 000000000..d854a060b
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/pom.xml
@@ -0,0 +1,162 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+~ Licensed to the Apache Software Foundation (ASF) under one
+~ or more contributor license agreements. See the NOTICE file
+~ distributed with this work for additional information
+~ regarding copyright ownership. The ASF licenses this file
+~ to you under the Apache License, Version 2.0 (the
+~ "License"); you may not use this file except in compliance
+~ with the License. You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-graphdb</artifactId>
+ <version>3.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>janusgraph-opensearch</artifactId>
+ <name>JanusGraph opensearch indexing store</name>
+
+ <properties>
+ <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
+ <checkstyle.skip>true</checkstyle.skip>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.databind.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.janusgraph</groupId>
+ <artifactId>janusgraph-core</artifactId>
+ <version>${janusgraph.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.noggit</groupId>
+ <artifactId>noggit</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.janusgraph</groupId>
+ <artifactId>janusgraph-mixed-index-utils</artifactId>
+ <version>${janusgraph.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.opensearch.client</groupId>
+ <artifactId>opensearch-rest-client</artifactId>
+ <version>${opensearch.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-testtools</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.janusgraph</groupId>
+ <artifactId>janusgraph-inmemory</artifactId>
+ <version>${janusgraph.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <filtering>true</filtering>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <includes>
+ <!-- Docker-free unit tests only (idempotent create, scroll cleanup, scroll wiring). The
+ Testcontainers ITs (**/*OpenSearch*IT.java) are intentionally NOT included here — they
+ run only under the opensearch-it profile below, so the normal docker-build job does not
+ unintentionally start OpenSearch containers. -->
+ <include>**/*Test.java</include>
+ </includes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <!-- OpenSearch integration tests run against a real OpenSearch server provisioned by Testcontainers
+ (Docker). Activated explicitly by the dedicated opensearch-it CI job; must not run as part of the
+ default build/test lifecycle. -->
+ <id>opensearch-it</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration combine.self="override">
+ <includes>
+ <include>**/*OpenSearch*IT.java</include>
+ </includes>
+ <!-- Propagate the configurable OpenSearch server image/version to the forked test JVM so
+ the same IT suite can validate 3.7 (default) and 3.8 without code changes. -->
+ <systemPropertyVariables>
+ <opensearch.docker.image>${opensearch.docker.image}</opensearch.docker.image>
+ <opensearch.docker.version>${opensearch.docker.version}</opensearch.docker.version>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchClient.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchClient.java
new file mode 100644
index 000000000..c783ed7fa
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchClient.java
@@ -0,0 +1,77 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch;
+
+import org.janusgraph.diskstorage.opensearch.mapping.IndexMapping;
+import org.janusgraph.diskstorage.opensearch.script.OSScriptResponse;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public interface OpenSearchClient extends Closeable {
+
+ OpenSearchMajorVersion getMajorVersion();
+
+ void clusterHealthRequest(String timeout) throws IOException;
+
+ boolean indexExists(String indexName) throws IOException;
+
+ boolean isIndex(String indexName);
+
+ boolean isAlias(String aliasName);
+
+ void createStoredScript(String scriptName, Map<String,Object> script) throws IOException;
+
+ OSScriptResponse getStoredScript(String scriptName) throws IOException;
+
+ void createIndex(String indexName, Map<String,Object> settings) throws IOException;
+
+ void updateIndexSettings(String indexName, Map<String,Object> settings) throws IOException;
+
+ void updateClusterSettings(Map<String,Object> settings) throws IOException;
+
+ Map getIndexSettings(String indexName) throws IOException;
+
+ void createMapping(String indexName, String typeName, Map<String,Object> mapping) throws IOException;
+
+ IndexMapping getMapping(String indexName, String typeName) throws IOException;
+
+ void deleteIndex(String indexName) throws IOException;
+
+ void clearStore(String indexName, String storeName) throws IOException;
+
+ void bulkRequest(List<OpenSearchMutation> requests, String ingestPipeline) throws IOException;
+
+ long countTotal(String indexName, Map<String,Object> requestData) throws IOException;
+
+ Number min(String indexName, Map<String,Object> requestData, String fieldName, Class<? extends Number> expectedType) throws IOException;
+
+ Number max(String indexName, Map<String,Object> requestData, String fieldName, Class<? extends Number> expectedType) throws IOException;
+
+ double avg(String indexName, Map<String,Object> requestData, String fieldName) throws IOException;
+
+ Number sum(String indexName, Map<String,Object> requestData, String fieldName, Class<? extends Number> expectedType) throws IOException;
+
+ OpenSearchResponse search(String indexName, Map<String,Object> request, boolean useScroll) throws IOException;
+
+ OpenSearchResponse search(String scrollId) throws IOException;
+
+ void deleteScroll(String scrollId) throws IOException;
+
+ void addAlias(String alias, String index) throws IOException;
+
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchConstants.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchConstants.java
new file mode 100644
index 000000000..1abaf137a
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchConstants.java
@@ -0,0 +1,31 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch;
+
+public class OpenSearchConstants {
+
+ public static final String OS_DOC_KEY = "doc";
+ public static final String OS_UPSERT_KEY = "upsert";
+ public static final String OS_SCRIPT_KEY = "script";
+ public static final String OS_SOURCE_KEY = "source";
+ public static final String OS_ID_KEY = "id";
+ public static final String OS_PARAMS_KEY = "params";
+ public static final String OS_PARAMS_FIELDS_KEY = "fields";
+ public static final String OS_LANG_KEY = "lang";
+ public static final String OS_TYPE_KEY = "type";
+ public static final String OS_ANALYZER = "analyzer";
+ public static final String OS_GEO_COORDS_KEY = "coordinates";
+ public static final String CUSTOM_ALL_FIELD = "all";
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchIndex.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchIndex.java
new file mode 100644
index 000000000..10a6aa4ae
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchIndex.java
@@ -0,0 +1,1566 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import org.janusgraph.core.Cardinality;
+import org.janusgraph.core.JanusGraphException;
+import org.janusgraph.core.attribute.Cmp;
+import org.janusgraph.core.attribute.Geo;
+import org.janusgraph.core.attribute.Geoshape;
+import org.janusgraph.core.attribute.Text;
+import org.janusgraph.core.schema.Mapping;
+import org.janusgraph.core.schema.Parameter;
+import org.janusgraph.diskstorage.BackendException;
+import org.janusgraph.diskstorage.BaseTransaction;
+import org.janusgraph.diskstorage.BaseTransactionConfig;
+import org.janusgraph.diskstorage.BaseTransactionConfigurable;
+import org.janusgraph.diskstorage.PermanentBackendException;
+import org.janusgraph.diskstorage.TemporaryBackendException;
+import org.janusgraph.diskstorage.configuration.ConfigNamespace;
+import org.janusgraph.diskstorage.configuration.ConfigOption;
+import org.janusgraph.diskstorage.configuration.Configuration;
+import org.janusgraph.diskstorage.opensearch.compat.AbstractOSCompat;
+import org.janusgraph.diskstorage.opensearch.compat.OSCompatUtils;
+import org.janusgraph.diskstorage.opensearch.mapping.IndexMapping;
+import org.janusgraph.diskstorage.opensearch.rest.util.HttpAuthTypes;
+import org.janusgraph.diskstorage.opensearch.script.OSScriptResponse;
+import org.janusgraph.diskstorage.indexing.IndexEntry;
+import org.janusgraph.diskstorage.indexing.IndexFeatures;
+import org.janusgraph.diskstorage.indexing.IndexMutation;
+import org.janusgraph.diskstorage.indexing.IndexProvider;
+import org.janusgraph.diskstorage.indexing.IndexQuery;
+import org.janusgraph.diskstorage.indexing.KeyInformation;
+import org.janusgraph.diskstorage.indexing.RawQuery;
+import org.janusgraph.diskstorage.mixed.utils.MixedIndexUtilsConfigOptions;
+import org.janusgraph.diskstorage.mixed.utils.processor.CircleProcessor;
+import org.janusgraph.diskstorage.util.DefaultTransaction;
+import org.janusgraph.graphdb.configuration.PreInitializeConfigOptions;
+import org.janusgraph.graphdb.database.serialize.AttributeUtils;
+import org.janusgraph.graphdb.query.JanusGraphPredicate;
+import org.janusgraph.graphdb.query.QueryUtil;
+import org.janusgraph.graphdb.query.condition.And;
+import org.janusgraph.graphdb.query.condition.Condition;
+import org.janusgraph.graphdb.query.condition.Not;
+import org.janusgraph.graphdb.query.condition.Or;
+import org.janusgraph.graphdb.query.condition.PredicateCondition;
+import org.janusgraph.graphdb.tinkerpop.optimize.step.Aggregation;
+import org.janusgraph.graphdb.types.ParameterType;
+import org.janusgraph.util.datastructures.IterablesUtil;
+import org.locationtech.spatial4j.shape.Rectangle;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import static org.janusgraph.diskstorage.configuration.ConfigOption.disallowEmpty;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_DOC_KEY;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_GEO_COORDS_KEY;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_LANG_KEY;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_SCRIPT_KEY;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_TYPE_KEY;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.GRAPH_NAME;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_MAX_RESULT_SET_SIZE;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_NAME;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_NS;
+
+/**
+ * @author Matthias Broecheler (me@matthiasb.com)
+ */
+
+@PreInitializeConfigOptions
+public class OpenSearchIndex implements IndexProvider {
+
+ private static final Logger log = LoggerFactory.getLogger(OpenSearchIndex.class);
+
+ private static final String STRING_MAPPING_SUFFIX = "__STRING";
+
+ public static final ConfigNamespace OPENSEARCH_NS =
+ new ConfigNamespace(INDEX_NS, "opensearch", "OpenSearch index configuration");
+
+ public static final ConfigOption<String> INTERFACE =
+ new ConfigOption<>(OPENSEARCH_NS, "interface",
+ "Interface for connecting to OpenSearch. " +
+ "TRANSPORT_CLIENT and NODE were previously supported, but now are required to migrate to REST_CLIENT. " +
+ "See the JanusGraph upgrade instructions for more details.",
+ ConfigOption.Type.MASKABLE, String.class, OpenSearchSetup.REST_CLIENT.toString(),
+ disallowEmpty(String.class));
+
+ public static final ConfigOption<String> HEALTH_REQUEST_TIMEOUT =
+ new ConfigOption<>(OPENSEARCH_NS, "health-request-timeout",
+ "When JanusGraph initializes its ES backend, JanusGraph waits up to this duration for the " +
+ "ES cluster health to reach at least yellow status. " +
+ "This string should be formatted as a natural number followed by the lowercase letter " +
+ "\"s\", e.g. 3s or 60s.", ConfigOption.Type.MASKABLE, "30s");
+
+ public static final ConfigOption<String> BULK_REFRESH =
+ new ConfigOption<>(OPENSEARCH_NS, "bulk-refresh",
+ "OpenSearch bulk API refresh setting used to control when changes made by this request are made " +
+ "visible to search", ConfigOption.Type.MASKABLE, "false");
+
+ public static final ConfigNamespace OS_CREATE_NS =
+ new ConfigNamespace(OPENSEARCH_NS, "create", "Settings related to index creation");
+
+ public static final ConfigOption<Long> CREATE_SLEEP =
+ new ConfigOption<>(OS_CREATE_NS, "sleep",
+ "How long to sleep, in milliseconds, between the successful completion of a (blocking) index " +
+ "creation request and the first use of that index. This only applies when creating an index in ES, " +
+ "which typically only happens the first time JanusGraph is started on top of ES. If the index JanusGraph is " +
+ "configured to use already exists, then this setting has no effect.", ConfigOption.Type.MASKABLE, 200L);
+
+ public static final ConfigNamespace OS_CREATE_EXTRAS_NS =
+ new ConfigNamespace(OS_CREATE_NS, "ext", "Overrides for arbitrary settings applied at index creation.\n" +
+ "See [OpenSearch](../index-backend/opensearch.md#index-creation-options), The full list of possible setting is available at " +
+ "[OpenSearch index settings](https://www.elastic.co/guide/en/opensearch/reference/current/index-modules.html#index-modules-settings).");
+
+ public static final ConfigOption<Boolean> USE_EXTERNAL_MAPPINGS =
+ new ConfigOption<>(OS_CREATE_NS, "use-external-mappings",
+ "Whether JanusGraph should make use of an external mapping when registering an index.", ConfigOption.Type.MASKABLE, false);
+
+ public static final ConfigOption<Integer> NUMBER_OF_REPLICAS =
+ new ConfigOption<>(OS_CREATE_EXTRAS_NS, "number_of_replicas",
+ "The number of replicas each primary shard has", ConfigOption.Type.MASKABLE, 1);
+
+ public static final ConfigOption<Integer> NUMBER_OF_SHARDS =
+ new ConfigOption<>(OS_CREATE_EXTRAS_NS, "number_of_shards",
+ "The number of primary shards that an index should have." +
+ "Default value is 5 on ES 6 and 1 on ES 7", ConfigOption.Type.MASKABLE, Integer.class);
+
+
+ public static final ConfigOption<Boolean> ALLOW_MAPPING_UPDATE =
+ new ConfigOption<>(OS_CREATE_NS, "allow-mapping-update",
+ "Whether JanusGraph should allow a mapping update when registering an index. " +
+ "Only applicable when " + USE_EXTERNAL_MAPPINGS.getName() + " is true.", ConfigOption.Type.MASKABLE, false);
+
+ public static final ConfigOption<Boolean> USE_ALL_FIELD =
+ new ConfigOption<>(OPENSEARCH_NS, "use-all-field",
+ "Whether JanusGraph should add an \"all\" field mapping. When enabled field mappings will " +
+ "include a \"copy_to\" parameter referencing the \"all\" field. This is supported since OpenSearch 6.x " +
+ " and is required when using wildcard fields starting in OpenSearch 6.x.", ConfigOption.Type.GLOBAL_OFFLINE, true);
+
+ public static final ConfigOption<Integer> OS_SCROLL_KEEP_ALIVE =
+ new ConfigOption<>(OPENSEARCH_NS, "scroll-keep-alive",
+ "How long (in seconds) opensearch should keep alive the scroll context.", ConfigOption.Type.GLOBAL_OFFLINE, 60);
+
+ public static final ConfigNamespace OS_INGEST_PIPELINES =
+ new ConfigNamespace(OPENSEARCH_NS, "ingest-pipeline", "Ingest pipeline applicable to a store of an index.");
+
+ public static final ConfigNamespace SSL_NS =
+ new ConfigNamespace(OPENSEARCH_NS, "ssl", "OpenSearch SSL configuration");
+
+ public static final ConfigOption<Boolean> SSL_ENABLED =
+ new ConfigOption<>(SSL_NS, "enabled",
+ "Controls use of the SSL connection to OpenSearch.", ConfigOption.Type.LOCAL, false);
+
+ public static final ConfigOption<Boolean> SSL_DISABLE_HOSTNAME_VERIFICATION =
+ new ConfigOption<>(SSL_NS, "disable-hostname-verification",
+ "Disables the SSL hostname verification if set to true. Hostname verification is enabled by default.",
+ ConfigOption.Type.LOCAL, false);
+
+ public static final ConfigOption<Boolean> SSL_ALLOW_SELF_SIGNED_CERTIFICATES =
+ new ConfigOption<>(SSL_NS, "allow-self-signed-certificates",
+ "Controls the accepting of the self-signed SSL certificates.",
+ ConfigOption.Type.LOCAL, false);
+
+ public static final ConfigNamespace SSL_TRUSTSTORE_NS =
+ new ConfigNamespace(SSL_NS, "truststore", "Configuration options for SSL Truststore.");
+
+ public static final ConfigOption<String> SSL_TRUSTSTORE_LOCATION =
+ new ConfigOption<>(SSL_TRUSTSTORE_NS, "location",
+ "Marks the location of the SSL Truststore.", ConfigOption.Type.LOCAL, "");
+
+ public static final ConfigOption<String> SSL_TRUSTSTORE_PASSWORD =
+ new ConfigOption<>(SSL_TRUSTSTORE_NS, "password",
+ "The password to access SSL Truststore.", ConfigOption.Type.LOCAL, "", Objects::nonNull);
+
+ public static final ConfigNamespace SSL_KEYSTORE_NS =
+ new ConfigNamespace(SSL_NS, "keystore", "Configuration options for SSL Keystore.");
+
+ public static final ConfigOption<String> SSL_KEYSTORE_LOCATION =
+ new ConfigOption<>(SSL_KEYSTORE_NS, "location",
+ "Marks the location of the SSL Keystore.", ConfigOption.Type.LOCAL, "");
+
+ public static final ConfigOption<String> SSL_KEYSTORE_PASSWORD =
+ new ConfigOption<>(SSL_KEYSTORE_NS, "storepassword",
+ "The password to access SSL Keystore.", ConfigOption.Type.LOCAL, "", Objects::nonNull);
+
+ public static final ConfigOption<String> SSL_KEY_PASSWORD =
+ new ConfigOption<>(SSL_KEYSTORE_NS, "keypassword",
+ "The password to access the key in the SSL Keystore. If the option is not present, the value of \"storepassword\" is used.",
+ ConfigOption.Type.LOCAL, "", Objects::nonNull);
+
+ public static final ConfigNamespace OS_HTTP_NS =
+ new ConfigNamespace(OPENSEARCH_NS, "http", "Configuration options for HTTP(S) transport.");
+
+ public static final ConfigNamespace OS_HTTP_AUTH_NS =
+ new ConfigNamespace(OS_HTTP_NS, "auth", "Configuration options for HTTP(S) authentication.");
+
+ public static final ConfigOption<String> OS_HTTP_AUTH_TYPE =
+ new ConfigOption<>(OS_HTTP_AUTH_NS, "type",
+ "Authentication type to be used for HTTP(S) access. Available options are `NONE`, `BASIC` and `CUSTOM`.",
+ ConfigOption.Type.LOCAL, HttpAuthTypes.NONE.toString());
+
+ public static final ConfigNamespace OS_HTTP_AUTH_BASIC_NS =
+ new ConfigNamespace(OS_HTTP_AUTH_NS, "basic", "Configuration options for HTTP(S) Basic authentication.");
+
+ public static final ConfigOption<String> OS_HTTP_AUTH_USERNAME =
+ new ConfigOption<>(OS_HTTP_AUTH_BASIC_NS, "username",
+ "Username for HTTP(S) authentication.", ConfigOption.Type.LOCAL, "");
+
+ public static final ConfigOption<String> OS_HTTP_AUTH_PASSWORD =
+ new ConfigOption<>(OS_HTTP_AUTH_BASIC_NS, "password",
+ "Password for HTTP(S) authentication.", ConfigOption.Type.LOCAL, "");
+
+ public static final ConfigOption<String> OS_HTTP_AUTH_REALM = new ConfigOption<>(OS_HTTP_AUTH_BASIC_NS,
+ "realm", "Realm value for HTTP(S) authentication. If empty, any realm is accepted.",
+ ConfigOption.Type.LOCAL, "");
+
+ public static final ConfigNamespace OS_HTTP_AUTH_CUSTOM_NS =
+ new ConfigNamespace(OS_HTTP_AUTH_NS, "custom", "Configuration options for custom HTTP(S) authenticator.");
+
+ public static final ConfigOption<String> OS_HTTP_AUTHENTICATOR_CLASS = new ConfigOption<>(
+ OS_HTTP_AUTH_CUSTOM_NS, "authenticator-class", "Authenticator fully qualified class name.",
+ ConfigOption.Type.LOCAL, "");
+
+ public static final ConfigOption<String[]> OS_HTTP_AUTHENTICATOR_ARGS = new ConfigOption<>(
+ OS_HTTP_AUTH_CUSTOM_NS, "authenticator-args", "Comma-separated custom authenticator constructor arguments.",
+ ConfigOption.Type.LOCAL, new String[0]);
+
+ public static final ConfigOption<Boolean> SETUP_MAX_OPEN_SCROLL_CONTEXTS =
+ new ConfigOption<>(OPENSEARCH_NS, "setup-max-open-scroll-contexts",
+ "Whether JanusGraph should setup max_open_scroll_context to maximum value for the cluster or not.",
+ ConfigOption.Type.MASKABLE, true);
+
+ public static final ConfigOption<Boolean> USE_MAPPING_TYPES =
+ new ConfigOption<>(OPENSEARCH_NS, "use-mapping-types",
+ "OpenSearch 2.x uses typeless indices. This option is retained for compatibility but has no effect.",
+ ConfigOption.Type.MASKABLE, false);
+
+ public static final ConfigOption<Long> CLIENT_KEEP_ALIVE =
+ new ConfigOption<>(OPENSEARCH_NS, "client-keep-alive",
+ "Set a keep-alive timeout (in milliseconds)",
+ ConfigOption.Type.GLOBAL_OFFLINE, Long.class);
+
+ public static final ConfigOption<Integer> RETRY_ON_CONFLICT =
+ new ConfigOption<>(OPENSEARCH_NS, "retry_on_conflict",
+ "Specify how many times should the operation be retried when a conflict occurs.", ConfigOption.Type.MASKABLE, 0);
+
+ public static final ConfigOption<Boolean> ENABLE_INDEX_STORE_NAMES_CACHE =
+ new ConfigOption<>(OPENSEARCH_NS, "enable_index_names_cache",
+ "Enables cache for generated index store names. " +
+ "It is recommended to always enable index store names cache unless you have more then 50000 indexes " +
+ "per index store.", ConfigOption.Type.MASKABLE, true);
+
+ public static final ConfigOption<Integer> CONNECT_TIMEOUT =
+ new ConfigOption<>(OPENSEARCH_NS, "connect-timeout",
+ "Sets the maximum connection timeout (in milliseconds).", ConfigOption.Type.MASKABLE,
+ Integer.class, 1000);
+
+ public static final ConfigOption<Integer> SOCKET_TIMEOUT =
+ new ConfigOption<>(OPENSEARCH_NS, "socket-timeout",
+ "Sets the maximum socket timeout (in milliseconds).", ConfigOption.Type.MASKABLE,
+ Integer.class, 30000);
+
+ public static final ConfigOption<Integer> RETRY_LIMIT =
+ new ConfigOption<>(OPENSEARCH_NS, "retry-limit",
+ "Sets the number of attempts for configured retryable error codes.", ConfigOption.Type.LOCAL,
+ Integer.class, 0);
+
+ public static final ConfigOption<Long> RETRY_INITIAL_WAIT =
+ new ConfigOption<>(OPENSEARCH_NS, "retry-initial-wait",
+ "Sets the initial retry wait time (in milliseconds) before exponential backoff.",
+ ConfigOption.Type.LOCAL, Long.class, 1L);
+
+ public static final ConfigOption<Long> RETRY_MAX_WAIT =
+ new ConfigOption<>(OPENSEARCH_NS, "retry-max-wait",
+ "Sets the max retry wait time (in milliseconds).", ConfigOption.Type.LOCAL,
+ Long.class, 1000L);
+
+ public static final ConfigOption<String[]> RETRY_ERROR_CODES =
+ new ConfigOption<>(OPENSEARCH_NS, "retry-error-codes",
+ "Comma separated list of OpenSearch REST client ResponseException error codes to retry. " +
+ "E.g. \"408,429\"", ConfigOption.Type.LOCAL, String[].class, new String[0]);
+
+ public static final ConfigOption<Integer> BULK_CHUNK_SIZE_LIMIT_BYTES =
+ new ConfigOption<>(OPENSEARCH_NS, "bulk-chunk-size-limit-bytes",
+ "The total size limit in bytes of a bulk request. Mutation batches in excess of this limit will be " +
+ "chunked to this size. If a single bulk item exceeds this limit an exception will be thrown after the " +
+ "smaller bulk items are submitted. Ensure that this limit is always less than or equal to the configured " +
+ "limit of `http.max_content_length` on the OpenSearch servers. For more information, refer to the " +
+ "[OpenSearch documentation](https://www.elastic.co/guide/en/opensearch/reference/master/modules-network.html).",
+ ConfigOption.Type.LOCAL, Integer.class, 100_000_000);
+
+ public static final int HOST_PORT_DEFAULT = 9200;
+
+ /**
+ * Default tree_levels used when creating geo_shape mappings.
+ */
+ public static final int DEFAULT_GEO_MAX_LEVELS = 20;
+
+ /**
+ * Default distance_error_pct used when creating geo_shape mappings.
+ */
+ public static final double DEFAULT_GEO_DIST_ERROR_PCT = 0.025;
+
+ private static final String PARAMETERIZED_DELETION_SCRIPT = parameterizedScriptPrepare("",
+ "for (field in params.fields) {",
+ " if (field.cardinality == 'SINGLE') {",
+ " ctx._source.remove(field.name);",
+ " } else if (ctx._source.containsKey(field.name)) {",
+ " def fieldIndex = ctx._source[field.name].indexOf(field.value);",
+ " if (fieldIndex >= 0 && fieldIndex < ctx._source[field.name].size()) {",
+ " ctx._source[field.name].remove(fieldIndex);",
+ " }",
+ " }",
+ "}");
+
+ private static final String PARAMETERIZED_ADDITION_SCRIPT = parameterizedScriptPrepare("",
+ "for (field in params.fields) {",
+ " if (ctx._source[field.name] == null) {",
+ " ctx._source[field.name] = [];",
+ " }",
+ " if (field.cardinality != 'SET' || ctx._source[field.name].indexOf(field.value) == -1) {",
+ " ctx._source[field.name].add(field.value);",
+ " }",
+ "}");
+
+ static final String INDEX_NAME_SEPARATOR = "_";
+ private static final String SCRIPT_ID_SEPARATOR = "-";
+
+ private static final String MAX_OPEN_SCROLL_CONTEXT_PARAMETER = "search.max_open_scroll_context";
+ private static final Map<String, Object> MAX_RESULT_WINDOW = ImmutableMap.of("index.max_result_window", Integer.MAX_VALUE);
+
+ private static final Parameter[] NULL_PARAMETERS = null;
+
+ private static final String TRACK_TOTAL_HITS_PARAMETER = "track_total_hits";
+ private static final Parameter[] TRACK_TOTAL_HITS_DISABLED_PARAMETERS = new Parameter[]{new Parameter<>(TRACK_TOTAL_HITS_PARAMETER, false)};
+ private static final Map<String, Object> TRACK_TOTAL_HITS_DISABLED_REQUEST_BODY = ImmutableMap.of(TRACK_TOTAL_HITS_PARAMETER, false);
+
+ private final Function<String, String> generateIndexStoreNameFunction = this::generateIndexStoreName;
+ private final Map<String, String> indexStoreNamesCache = new ConcurrentHashMap<>();
+ private final boolean indexStoreNameCacheEnabled;
+
+ private final AbstractOSCompat compat;
+ private final OpenSearchClient client;
+ private final String indexName;
+ private final int batchSize;
+ private final boolean useExternalMappings;
+ private final boolean allowMappingUpdate;
+ private final Map<String, Object> indexSetting;
+ private final long createSleep;
+ private final boolean useAllField;
+ private final Map<String, Object> ingestPipelines;
+ private final boolean useMappingTypes;
+ private final String parameterizedAdditionScriptId;
+ private final String parameterizedDeletionScriptId;
+ private final boolean supportsGeoShapePrefixTree;
+ private final CircleProcessor bdbCircleProcessor;
+
+ public OpenSearchIndex(Configuration config) throws BackendException {
+ indexName = determineIndexName(config);
+ parameterizedAdditionScriptId = generateScriptId("add");
+ parameterizedDeletionScriptId = generateScriptId("del");
+ useAllField = config.get(USE_ALL_FIELD);
+ useExternalMappings = config.get(USE_EXTERNAL_MAPPINGS);
+ allowMappingUpdate = config.get(ALLOW_MAPPING_UPDATE);
+ createSleep = config.get(CREATE_SLEEP);
+ ingestPipelines = config.getSubset(OS_INGEST_PIPELINES);
+ useMappingTypes = config.get(USE_MAPPING_TYPES);
+ indexStoreNameCacheEnabled = config.get(ENABLE_INDEX_STORE_NAMES_CACHE);
+ batchSize = config.get(INDEX_MAX_RESULT_SET_SIZE);
+ log.debug("Configured ES query nb result by query to {}", batchSize);
+ bdbCircleProcessor = MixedIndexUtilsConfigOptions.buildBKDCircleProcessor(config);
+
+ client = interfaceConfiguration(config).getClient();
+ supportsGeoShapePrefixTree = client.getMajorVersion().supportsTypelessWireFormat();
+
+ checkClusterHealth(config.get(HEALTH_REQUEST_TIMEOUT));
+
+ compat = OSCompatUtils.acquireCompatForVersion(client.getMajorVersion());
+
+ indexSetting = OpenSearchSetup.getSettingsFromJanusGraphConf(config);
+
+ setupMaxOpenScrollContextsIfNeeded(config);
+
+ setupStoredScripts();
+ }
+
+ public static String determineIndexName(Configuration config) {
+ return !config.has(INDEX_NAME) && config.has(GRAPH_NAME)
+ ? config.get(GRAPH_NAME)
+ : config.get(INDEX_NAME);
+ }
+
+ private void checkClusterHealth(String healthCheck) throws BackendException {
+ try {
+ client.clusterHealthRequest(healthCheck);
+ } catch (final IOException e) {
+ throw new PermanentBackendException(e.getMessage(), e);
+ }
+ }
+
+ private void setupStoredScripts() throws PermanentBackendException {
+ setupStoredScriptIfNeeded(parameterizedAdditionScriptId, PARAMETERIZED_ADDITION_SCRIPT);
+ setupStoredScriptIfNeeded(parameterizedDeletionScriptId, PARAMETERIZED_DELETION_SCRIPT);
+ }
+
+ private void setupStoredScriptIfNeeded(String storedScriptId, String source) throws PermanentBackendException {
+
+ ImmutableMap<String, Object> preparedScript = compat.prepareScript(source).build();
+
+ String lang = (String) ((ImmutableMap<String, Object>) preparedScript.get(OS_SCRIPT_KEY)).get(OS_LANG_KEY);
+
+ try {
+ OSScriptResponse esScriptResponse = client.getStoredScript(storedScriptId);
+
+ if(Boolean.FALSE.equals(esScriptResponse.getFound()) || !Objects.equals(lang, esScriptResponse.getScript().getLang()) ||
+ !Objects.equals(source, esScriptResponse.getScript().getSource())){
+ client.createStoredScript(storedScriptId, preparedScript);
+ }
+
+ } catch (final IOException e) {
+ throw new PermanentBackendException(e.getMessage(), e);
+ }
+ }
+
+ private void setupMaxOpenScrollContextsIfNeeded(Configuration config) throws PermanentBackendException {
+
+ if(client.getMajorVersion().supportsTypelessWireFormat()){
+
+ boolean setupMaxOpenScrollContexts;
+
+ if(config.has(SETUP_MAX_OPEN_SCROLL_CONTEXTS)){
+ setupMaxOpenScrollContexts = config.get(SETUP_MAX_OPEN_SCROLL_CONTEXTS);
+ } else {
+ setupMaxOpenScrollContexts = SETUP_MAX_OPEN_SCROLL_CONTEXTS.getDefaultValue();
+ }
+
+ if(setupMaxOpenScrollContexts){
+
+ Map<String, Object> settings = ImmutableMap.of("persistent",
+ ImmutableMap.of(MAX_OPEN_SCROLL_CONTEXT_PARAMETER, Integer.MAX_VALUE));
+
+ try {
+ client.updateClusterSettings(settings);
+ } catch (final IOException e) {
+ throw new PermanentBackendException(e.getMessage(), e);
+ }
+ }
+ }
+ }
+
+ /**
+ * If ES already contains this instance's target index, then do nothing.
+ * Otherwise, create the index, then wait {@link #CREATE_SLEEP}.
+ * <p>
+ * The {@code client} field must point to a live, connected client.
+ * The {@code indexName} field must be non-null and point to the name
+ * of the index to check for existence or create.
+ *
+ * @param index index name
+ * @throws IOException if the index status could not be checked or index could not be created
+ */
+ private void checkForOrCreateIndex(String index) throws IOException {
+ Objects.requireNonNull(client);
+ Objects.requireNonNull(index);
+
+ // Create index if it does not useExternalMappings and if it does not already exist
+ if (!useExternalMappings && !client.indexExists(index)) {
+ boolean created = createIndexIdempotent(client, index, indexSetting);
+
+ if (created) {
+ client.updateIndexSettings(index, MAX_RESULT_WINDOW);
+ try {
+ log.debug("Sleeping {} ms after {} index creation returned from actionGet()", createSleep, index);
+ Thread.sleep(createSleep);
+ } catch (final InterruptedException e) {
+ throw new JanusGraphException("Interrupted while waiting for index to settle in", e);
+ }
+ }
+ }
+ Preconditions.checkState(client.indexExists(index), "Could not create index: %s",index);
+ client.addAlias(indexName, index);
+ }
+
+ /**
+ * Creates {@code index} idempotently so that concurrently starting Atlas nodes (HA) do not fail with a
+ * {@code resource_already_exists_exception}.
+ * <p>
+ * Index creation is inherently a check-then-act race: two nodes may both observe that the index is absent and
+ * both attempt to create it. Exactly one succeeds; the loser's create request fails. This is safe as long as the
+ * desired final state (the index existing) is reached. We therefore only treat a failed create as success when a
+ * follow-up existence check confirms the index is now present — i.e. another node created it. Any other failure
+ * (index still absent) is rethrown so real errors are not masked.
+ *
+ * @return {@code true} if this call created the index, {@code false} if it was created concurrently by another node
+ * @throws IOException if creation failed and the index does not exist afterwards
+ */
+ static boolean createIndexIdempotent(OpenSearchClient client, String index, Map<String, Object> indexSetting) throws IOException {
+ try {
+ client.createIndex(index, indexSetting);
+
+ return true;
+ } catch (final IOException createException) {
+ // Another node may have created the index between our existence check and this create request.
+ // Only swallow the error if the index actually exists now; otherwise surface the original failure.
+ if (client.indexExists(index)) {
+ log.info("Index {} was created concurrently by another node; treating create as successful.", index);
+
+ return false;
+ }
+
+ log.error("Failed to create index {} and it does not exist afterwards.", index, createException);
+
+ throw createException;
+ }
+ }
+
+
+ /**
+ * Configure OpenSearchIndex's ES client. See{@link org.janusgraph.diskstorage.opensearch.OpenSearchSetup} for more
+ * information.
+ *
+ * @param config a config passed to OpenSearchIndex's constructor
+ * @return a client object open and ready for use
+ */
+ private OpenSearchSetup.Connection interfaceConfiguration(Configuration config) {
+ final OpenSearchSetup clientMode = ConfigOption.getEnumValue(config.get(INTERFACE), OpenSearchSetup.class);
+
+ try {
+ return clientMode.connect(config);
+ } catch (final IOException e) {
+ throw new JanusGraphException(e);
+ }
+ }
+
+ private BackendException convert(Exception esException) {
+ if (esException instanceof InterruptedException) {
+ return new TemporaryBackendException("Interrupted while waiting for response", esException);
+ } else {
+ return new PermanentBackendException("Unknown exception while executing index operation", esException);
+ }
+ }
+
+ private static String getDualMappingName(String key) {
+ return key + STRING_MAPPING_SUFFIX;
+ }
+
+ private String generateScriptId(String uniqueScriptSuffix){
+ return indexName + SCRIPT_ID_SEPARATOR + uniqueScriptSuffix;
+ }
+
+ private String generateIndexStoreName(String store){
+ return indexName + INDEX_NAME_SEPARATOR + store.toLowerCase();
+ }
+
+ private String getIndexStoreName(String store) {
+
+ if(indexStoreNameCacheEnabled){
+ return indexStoreNamesCache.computeIfAbsent(store, generateIndexStoreNameFunction);
+ }
+
+ return generateIndexStoreName(store);
+ }
+
+ @Override
+ public void register(String store, String key, KeyInformation information,
+ BaseTransaction tx) throws BackendException {
+ final Class<?> dataType = information.getDataType();
+ final Mapping map = Mapping.getMapping(information);
+ Preconditions.checkArgument(map==Mapping.DEFAULT || AttributeUtils.isString(dataType) ||
+ AttributeUtils.isGeo(dataType) && (map==Mapping.BKD || map==Mapping.PREFIX_TREE),
+ "Specified illegal mapping [%s] for data type [%s]",map,dataType);
+ if(map==Mapping.PREFIX_TREE && !supportsGeoShapePrefixTree){
+ String error = "PREFIX_TREE geo_shape indexing strategy is not supported. " +
+ "Please, select BKD strategy for geo_shape index. Key ["+key+"].";
+ log.error(error);
+ throw new IllegalArgumentException(error);
+ }
+ final String indexStoreName = getIndexStoreName(store);
+ if (useExternalMappings) {
+ try {
+ //We check if the externalMapping have the property 'key'
+ final IndexMapping mappings = client.getMapping(indexStoreName, store);
+ if (mappings == null || (!mappings.isDynamic() && !mappings.getProperties().containsKey(key))) {
+ //Error if it is not dynamic and have not the property 'key'
+ throw new PermanentBackendException("The external mapping for index '"+ indexStoreName + "' and type '" + store + "' do not have property '" + key + "'");
+ } else if (allowMappingUpdate && mappings.isDynamic()) {
+ //If it is dynamic, we push the unknown property 'key'
+ this.pushMapping(store, key, information);
+ }
+ } catch (final IOException e) {
+ throw new PermanentBackendException(e);
+ }
+ } else {
+ try {
+ checkForOrCreateIndex(indexStoreName);
+ } catch (final IOException e) {
+ throw new PermanentBackendException(e);
+ }
+ this.pushMapping(store, key, information);
+ }
+ }
+
+ /**
+ * Push mapping to OpenSearch
+ * @param store the type in the index
+ * @param key the name of the property in the index
+ * @param information information of the key
+ */
+ private void pushMapping(String store, String key,
+ KeyInformation information) throws AssertionError, BackendException {
+ final Class<?> dataType = information.getDataType();
+ Mapping map = Mapping.getMapping(information);
+ final Map<String,Object> properties = new HashMap<>();
+ if (AttributeUtils.isString(dataType)) {
+ if (map==Mapping.DEFAULT) map=Mapping.TEXT;
+ log.debug("Registering string type for {} with mapping {}", key, map);
+ final String stringAnalyzer
+ = ParameterType.STRING_ANALYZER.findParameter(information.getParameters(), null);
+ final String textAnalyzer = ParameterType.TEXT_ANALYZER.findParameter(information.getParameters(), null);
+ // use keyword type for string mappings unless custom string analyzer is provided
+ final Map<String,Object> stringMapping
+ = stringAnalyzer == null ? compat.createKeywordMapping() : compat.createTextMapping(stringAnalyzer);
+ switch (map) {
+ case STRING:
+ properties.put(key, stringMapping);
+ break;
+ case TEXT:
+ properties.put(key, compat.createTextMapping(textAnalyzer));
+ break;
+ case TEXTSTRING:
+ properties.put(key, compat.createTextMapping(textAnalyzer));
+ properties.put(getDualMappingName(key), stringMapping);
+ break;
+ default: throw new AssertionError("Unexpected mapping: "+map);
+ }
+ } else if (dataType == Float.class) {
+ log.debug("Registering float type for {}", key);
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "float"));
+ } else if (dataType == Double.class) {
+ log.debug("Registering double type for {}", key);
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "double"));
+ } else if (dataType == Byte.class) {
+ log.debug("Registering byte type for {}", key);
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "byte"));
+ } else if (dataType == Short.class) {
+ log.debug("Registering short type for {}", key);
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "short"));
+ } else if (dataType == Integer.class) {
+ log.debug("Registering integer type for {}", key);
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "integer"));
+ } else if (dataType == Long.class) {
+ log.debug("Registering long type for {}", key);
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "long"));
+ } else if (dataType == Boolean.class) {
+ log.debug("Registering boolean type for {}", key);
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "boolean"));
+ } else if (dataType == Geoshape.class) {
+ switch (map) {
+ case BKD:
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "geo_shape"));
+ break;
+ case PREFIX_TREE:
+ final int maxLevels = ParameterType.INDEX_GEO_MAX_LEVELS.findParameter(information.getParameters(),
+ DEFAULT_GEO_MAX_LEVELS);
+ final double distErrorPct
+ = ParameterType.INDEX_GEO_DIST_ERROR_PCT.findParameter(information.getParameters(),
+ DEFAULT_GEO_DIST_ERROR_PCT);
+ log.debug("Registering geo_shape type for {} with tree_levels={} and distance_error_pct={}", key,
+ maxLevels, distErrorPct);
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "geo_shape",
+ "tree", "quadtree",
+ "tree_levels", maxLevels,
+ "distance_error_pct", distErrorPct));
+ break;
+ default:
+ log.debug("Registering geo_point type for {}", key);
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "geo_point"));
+ }
+ } else if (dataType == Date.class || dataType == Instant.class) {
+ log.debug("Registering date type for {}", key);
+ properties.put(key, ImmutableMap.of(OS_TYPE_KEY, "date"));
+ } else if (dataType == UUID.class) {
+ log.debug("Registering uuid type for {}", key);
+ properties.put(key, compat.createKeywordMapping());
+ }
+
+ if (useAllField) {
+ // add custom all field mapping if it doesn't exist
+ properties.put(OpenSearchConstants.CUSTOM_ALL_FIELD, compat.createTextMapping(null));
+
+ // add copy_to for custom all field mapping
+ if (properties.containsKey(key) && dataType != Geoshape.class) {
+ final Map<String, Object> mapping = new HashMap<>(((Map<String, Object>) properties.get(key)));
+ mapping.put("copy_to", OpenSearchConstants.CUSTOM_ALL_FIELD);
+ properties.put(key, mapping);
+ }
+ }
+
+ final List<Parameter> customParameters = ParameterType.getCustomParameters(information.getParameters());
+
+ if (properties.containsKey(key) && !customParameters.isEmpty()) {
+ final Map<String, Object> mapping = new HashMap<>(((Map<String, Object>) properties.get(key)));
+ customParameters.forEach(p -> mapping.put(p.key(), p.value()));
+ properties.put(key, mapping);
+ }
+
+ final Map<String,Object> mapping = ImmutableMap.of("properties", properties);
+
+ try {
+ client.createMapping(getIndexStoreName(store), store, mapping);
+ } catch (final Exception e) {
+ throw convert(e);
+ }
+ }
+
+ private static Mapping getStringMapping(KeyInformation information) {
+ assert AttributeUtils.isString(information.getDataType());
+ Mapping map = Mapping.getMapping(information);
+ if (map==Mapping.DEFAULT) map = Mapping.TEXT;
+ return map;
+ }
+
+ private static boolean hasDualStringMapping(KeyInformation information) {
+ return AttributeUtils.isString(information.getDataType()) && getStringMapping(information)==Mapping.TEXTSTRING;
+ }
+
+ public Map<String, Object> getNewDocument(final List<IndexEntry> additions,
+ KeyInformation.StoreRetriever information) throws BackendException {
+ // JSON writes duplicate fields one after another, which forces us
+ // at this stage to make de-duplication on the IndexEntry list. We don't want to pay the
+ // price map storage on the Mutation level because none of other backends need that.
+
+ final Multimap<String, IndexEntry> unique = LinkedListMultimap.create();
+ for (final IndexEntry e : additions) {
+ unique.put(e.field, e);
+ }
+
+ final Map<String, Object> doc = new HashMap<>();
+ for (final Map.Entry<String, Collection<IndexEntry>> add : unique.asMap().entrySet()) {
+ final KeyInformation keyInformation = information.get(add.getKey());
+ final Object value;
+ switch (keyInformation.getCardinality()) {
+ case SINGLE:
+ value = convertToEsType(Iterators.getLast(add.getValue().iterator()).value,
+ Mapping.getMapping(keyInformation));
+ break;
+ case SET:
+ case LIST:
+ value = add.getValue().stream()
+ .map(v -> convertToEsType(v.value, Mapping.getMapping(keyInformation)))
+ .filter(v -> {
+ Preconditions.checkArgument(!(v instanceof byte[]),
+ "Collections not supported for %s", add.getKey());
+ return true;
+ }).toArray();
+ break;
+ default:
+ value = null;
+ break;
+ }
+
+ doc.put(add.getKey(), value);
+ if (hasDualStringMapping(information.get(add.getKey())) && keyInformation.getDataType() == String.class) {
+ doc.put(getDualMappingName(add.getKey()), value);
+ }
+
+
+ }
+
+ return doc;
+ }
+
+ private Object convertToEsType(Object value, Mapping mapping) {
+ if (value instanceof Number) {
+ if (AttributeUtils.isWholeNumber((Number) value)) {
+ return ((Number) value).longValue();
+ } else { //double or float
+ return ((Number) value).doubleValue();
+ }
+ } else if (AttributeUtils.isString(value)) {
+ return value;
+ } else if (value instanceof Geoshape) {
+ return convertGeoshape((Geoshape) value, mapping);
+ } else if (value instanceof Date) {
+ return value;
+ } else if (value instanceof Instant) {
+ return Date.from((Instant) value);
+ } else if (value instanceof Boolean) {
+ return value;
+ } else if (value instanceof UUID) {
+ return value.toString();
+ } else throw new IllegalArgumentException("Unsupported type: " + value.getClass() + " (value: " + value + ")");
+ }
+
+ @SuppressWarnings("unchecked")
+ private Object convertGeoshape(Geoshape geoshape, Mapping mapping) {
+ if (geoshape.getType() == Geoshape.Type.POINT && Mapping.BKD != mapping && Mapping.PREFIX_TREE != mapping) {
+ final Geoshape.Point p = geoshape.getPoint();
+ return new double[]{p.getLongitude(), p.getLatitude()};
+ } else if (geoshape.getType() == Geoshape.Type.BOX) {
+ final Rectangle box = geoshape.getShape().getBoundingBox();
+ final Map<String,Object> map = new HashMap<>();
+ map.put("type", "envelope");
+ map.put("coordinates", new double[][] {{box.getMinX(),box.getMaxY()},{box.getMaxX(),box.getMinY()}});
+ return map;
+ } else if (geoshape.getType() == Geoshape.Type.CIRCLE) {
+ try {
+ if(Mapping.BKD.equals(mapping)){
+ geoshape = bdbCircleProcessor.process(geoshape);
+ if (geoshape.getType() != Geoshape.Type.CIRCLE){
+ return convertGeoshape(geoshape, mapping);
+ }
+ }
+ Map<String,Object> map = geoshape.toMap();
+ map.put("radius", map.get("radius") + ((Map<String, String>) map.remove("properties")).get("radius_units"));
+ return map;
+ } catch (final IOException e) {
+ throw new IllegalArgumentException("Invalid geoshape: " + geoshape, e);
+ }
+ } else {
+ try {
+ return geoshape.toMap();
+ } catch (final IOException e) {
+ throw new IllegalArgumentException("Invalid geoshape: " + geoshape, e);
+ }
+ }
+ }
+
+ @Override
+ public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever information,
+ BaseTransaction tx) throws BackendException {
+ final List<OpenSearchMutation> requests = new ArrayList<>();
+ try {
+ for (final Map.Entry<String, Map<String, IndexMutation>> stores : mutations.entrySet()) {
+ final List<OpenSearchMutation> requestByStore = new ArrayList<>();
+ final String storeName = stores.getKey();
+ final String indexStoreName = getIndexStoreName(storeName);
+ for (final Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
+ final String documentId = entry.getKey();
+ final IndexMutation mutation = entry.getValue();
+ assert mutation.isConsolidated();
+ Preconditions.checkArgument(!(mutation.isNew() && mutation.isDeleted()));
+ Preconditions.checkArgument(!mutation.isNew() || !mutation.hasDeletions());
+ Preconditions.checkArgument(!mutation.isDeleted() || !mutation.hasAdditions());
+ //Deletions first
+ if (mutation.hasDeletions()) {
+ if (mutation.isDeleted()) {
+ log.trace("Deleting entire document {}", documentId);
+ requestByStore.add(OpenSearchMutation.createDeleteRequest(indexStoreName, storeName,
+ documentId));
+ } else {
+ List<Map<String, Object>> params = getParameters(information.get(storeName),
+ mutation.getDeletions(), true);
+ Map doc = compat.prepareStoredScript(parameterizedDeletionScriptId, params).build();
+ log.trace("Deletion script {} with params {}", PARAMETERIZED_DELETION_SCRIPT, params);
+ requestByStore.add(OpenSearchMutation.createUpdateRequest(indexStoreName, storeName,
+ documentId, doc));
+ }
+ }
+ if (mutation.hasAdditions()) {
+ if (mutation.isNew()) { //Index
+ log.trace("Adding entire document {}", documentId);
+ final Map<String, Object> source = getNewDocument(mutation.getAdditions(),
+ information.get(storeName));
+ requestByStore.add(OpenSearchMutation.createIndexRequest(indexStoreName, storeName,
+ documentId, source));
+ } else {
+ final Map upsert;
+ if (!mutation.hasDeletions()) {
+ upsert = getNewDocument(mutation.getAdditions(), information.get(storeName));
+ } else {
+ upsert = null;
+ }
+
+ List<Map<String, Object>> params = getParameters(information.get(storeName),
+ mutation.getAdditions(), false, Cardinality.SINGLE);
+ if (!params.isEmpty()) {
+ ImmutableMap.Builder builder = compat.prepareStoredScript(parameterizedAdditionScriptId, params);
+ requestByStore.add(OpenSearchMutation.createUpdateRequest(indexStoreName, storeName,
+ documentId, builder, upsert));
+ log.trace("Adding script {} with params {}", PARAMETERIZED_ADDITION_SCRIPT, params);
+ }
+
+ final Map<String, Object> doc = getAdditionDoc(information, storeName, mutation);
+ if (!doc.isEmpty()) {
+ final ImmutableMap.Builder builder = ImmutableMap.builder().put(OS_DOC_KEY, doc);
+ requestByStore.add(OpenSearchMutation.createUpdateRequest(indexStoreName, storeName,
+ documentId, builder, upsert));
+ log.trace("Adding update {}", doc);
+ }
+ }
+ }
+ }
+ if (!requestByStore.isEmpty() && ingestPipelines.containsKey(storeName)) {
+ client.bulkRequest(requestByStore, String.valueOf(ingestPipelines.get(storeName)));
+ } else if (!requestByStore.isEmpty()) {
+ requests.addAll(requestByStore);
+ }
+ }
+ if (!requests.isEmpty()) {
+ client.bulkRequest(requests, null);
+ }
+ } catch (final Exception e) {
+ log.error("Failed to execute bulk OpenSearch mutation", e);
+ throw convert(e);
+ }
+ }
+
+ private List<Map<String, Object>> getParameters(KeyInformation.StoreRetriever storeRetriever,
+ List<IndexEntry> entries,
+ boolean deletion,
+ Cardinality... cardinalitiesToSkip) {
+ Set<Cardinality> cardinalityToSkipSet = Sets.newHashSet(cardinalitiesToSkip);
+ List<Map<String, Object>> result = new ArrayList<>();
+ for (IndexEntry entry : entries) {
+ KeyInformation info = storeRetriever.get(entry.field);
+ if (cardinalityToSkipSet.contains(info.getCardinality())) {
+ continue;
+ }
+ Object jsValue = deletion && info.getCardinality() == Cardinality.SINGLE ?
+ "" : convertToEsType(entry.value, Mapping.getMapping(info));
+ result.add(ImmutableMap.of("name", entry.field,
+ "value", jsValue,
+ "cardinality", info.getCardinality().name()));
+ if (hasDualStringMapping(info)) {
+ result.add(ImmutableMap.of("name", getDualMappingName(entry.field),
+ "value", jsValue,
+ "cardinality", info.getCardinality().name()));
+ }
+ }
+ return result;
+ }
+
+ private Map<String,Object> getAdditionDoc(KeyInformation.IndexRetriever information,
+ String store, IndexMutation mutation) throws PermanentBackendException {
+ final Map<String,Object> doc = new HashMap<>();
+ for (final IndexEntry e : mutation.getAdditions()) {
+ final KeyInformation keyInformation = information.get(store).get(e.field);
+ if (keyInformation.getCardinality() == Cardinality.SINGLE) {
+ doc.put(e.field, convertToEsType(e.value, Mapping.getMapping(keyInformation)));
+ if (hasDualStringMapping(keyInformation)) {
+ doc.put(getDualMappingName(e.field), convertToEsType(e.value, Mapping.getMapping(keyInformation)));
+ }
+ }
+ }
+
+ return doc;
+ }
+
+ @Override
+ public void restore(Map<String,Map<String, List<IndexEntry>>> documents, KeyInformation.IndexRetriever information,
+ BaseTransaction tx) throws BackendException {
+ final List<OpenSearchMutation> requests = new ArrayList<>();
+ try {
+ for (final Map.Entry<String, Map<String, List<IndexEntry>>> stores : documents.entrySet()) {
+ final List<OpenSearchMutation> requestByStore = new ArrayList<>();
+ final String store = stores.getKey();
+ final String indexStoreName = getIndexStoreName(store);
+ for (final Map.Entry<String, List<IndexEntry>> entry : stores.getValue().entrySet()) {
+ final String docID = entry.getKey();
+ final List<IndexEntry> content = entry.getValue();
+ if (content == null || content.size() == 0) {
+ // delete
+ if (log.isTraceEnabled())
+ log.trace("Deleting entire document {}", docID);
+
+ requestByStore.add(OpenSearchMutation.createDeleteRequest(indexStoreName, store, docID));
+ } else {
+ // Add
+ if (log.isTraceEnabled())
+ log.trace("Adding entire document {}", docID);
+ final Map<String, Object> source = getNewDocument(content, information.get(store));
+ requestByStore.add(OpenSearchMutation.createIndexRequest(indexStoreName, store, docID,
+ source));
+ }
+ }
+ if (!requestByStore.isEmpty() && ingestPipelines.containsKey(store)) {
+ client.bulkRequest(requestByStore, String.valueOf(ingestPipelines.get(store)));
+ } else if (!requestByStore.isEmpty()) {
+ requests.addAll(requestByStore);
+ }
+ }
+ if (!requests.isEmpty())
+ client.bulkRequest(requests, null);
+ } catch (final Exception e) {
+ throw convert(e);
+ }
+ }
+
+ private Map<String, Object> getRelationFromCmp(final Cmp cmp, String key, final Object value) {
+ switch (cmp) {
+ case EQUAL:
+ return compat.term(key, value);
+ case NOT_EQUAL:
+ return compat.boolMust(ImmutableList.of(compat.exists(key), compat.boolMustNot(compat.term(key, value))));
+ case LESS_THAN:
+ return compat.lt(key, value);
+ case LESS_THAN_EQUAL:
+ return compat.lte(key, value);
+ case GREATER_THAN:
+ return compat.gt(key, value);
+ case GREATER_THAN_EQUAL:
+ return compat.gte(key, value);
+ default:
+ throw new IllegalArgumentException("Unexpected relation: " + cmp);
+ }
+ }
+
+ public Map<String,Object> getFilter(Condition<?> condition, KeyInformation.StoreRetriever information) {
+ if (condition instanceof PredicateCondition) {
+ final PredicateCondition<String, ?> atom = (PredicateCondition) condition;
+ Object value = atom.getValue();
+ final String key = atom.getKey();
+ final JanusGraphPredicate predicate = atom.getPredicate();
+ if (value == null && predicate == Cmp.NOT_EQUAL) {
+ return compat.exists(key);
+ } else if (value instanceof Number) {
+ Preconditions.checkArgument(predicate instanceof Cmp,
+ "Relation not supported on numeric types: %s", predicate);
+ return getRelationFromCmp((Cmp) predicate, key, value);
+ } else if (value instanceof String) {
+
+ final Mapping mapping = getStringMapping(information.get(key));
+ final String fieldName;
+ if (mapping==Mapping.TEXT && !(Text.HAS_CONTAINS.contains(predicate) || predicate instanceof Cmp))
+ throw new IllegalArgumentException("Text mapped string values only support CONTAINS and Compare queries and not: " + predicate);
+ if (mapping==Mapping.STRING && Text.HAS_CONTAINS.contains(predicate))
+ throw new IllegalArgumentException("String mapped string values do not support CONTAINS queries: " + predicate);
+ if (mapping==Mapping.TEXTSTRING && !(Text.HAS_CONTAINS.contains(predicate) || (predicate instanceof Cmp && predicate != Cmp.EQUAL))) {
+ fieldName = getDualMappingName(key);
+ } else {
+ fieldName = key;
+ }
+
+ if (predicate == Text.CONTAINS || predicate == Cmp.EQUAL) {
+ return compat.match(fieldName, value);
+ } else if (predicate == Text.NOT_CONTAINS) {
+ return compat.boolMust(ImmutableList.of(compat.exists(fieldName),
+ compat.boolMustNot(compat.match(fieldName, value))));
+ } else if (predicate == Text.CONTAINS_PHRASE) {
+ return compat.matchPhrase(fieldName, value);
+ } else if (predicate == Text.NOT_CONTAINS_PHRASE) {
+ return compat.boolMust(ImmutableList.of(compat.exists(fieldName),
+ compat.boolMustNot(compat.matchPhrase(fieldName, value))));
+ } else if (predicate == Text.CONTAINS_PREFIX) {
+ if (!ParameterType.TEXT_ANALYZER.hasParameter(information.get(key).getParameters()))
+ value = ((String) value).toLowerCase();
+ return compat.prefix(fieldName, value);
+ } else if (predicate == Text.NOT_CONTAINS_PREFIX) {
+ if (!ParameterType.TEXT_ANALYZER.hasParameter(information.get(key).getParameters()))
+ value = ((String) value).toLowerCase();
+ return compat.boolMust(ImmutableList.of(compat.exists(fieldName),
+ compat.boolMustNot(compat.prefix(fieldName, value))));
+ } else if (predicate == Text.CONTAINS_REGEX) {
+ if (!ParameterType.TEXT_ANALYZER.hasParameter(information.get(key).getParameters()))
+ value = ((String) value).toLowerCase();
+ return compat.regexp(fieldName, value);
+ } else if (predicate == Text.NOT_CONTAINS_REGEX) {
+ if (!ParameterType.TEXT_ANALYZER.hasParameter(information.get(key).getParameters()))
+ value = ((String) value).toLowerCase();
+ return compat.boolMust(ImmutableList.of(compat.exists(fieldName),
+ compat.boolMustNot(compat.regexp(fieldName, value))));
+ } else if (predicate == Text.PREFIX) {
+ return compat.prefix(fieldName, value);
+ } else if (predicate == Text.NOT_PREFIX) {
+ return compat.boolMust(ImmutableList.of(compat.exists(fieldName),
+ compat.boolMustNot(compat.prefix(fieldName, value))));
+ } else if (predicate == Text.REGEX) {
+ return compat.regexp(fieldName, value);
+ } else if (predicate == Text.NOT_REGEX) {
+ return compat.boolMust(ImmutableList.of(compat.exists(fieldName),
+ compat.boolMustNot(compat.regexp(fieldName, value))));
+ } else if (predicate == Cmp.NOT_EQUAL) {
+ return compat.boolMust(ImmutableList.of(compat.exists(fieldName), compat.boolMustNot(compat.match(fieldName, value))));
+ } else if (predicate == Text.FUZZY || predicate == Text.CONTAINS_FUZZY) {
+ return compat.fuzzyMatch(fieldName, value);
+ } else if (predicate == Text.NOT_FUZZY || predicate == Text.NOT_CONTAINS_FUZZY) {
+ return compat.boolMust(ImmutableList.of(compat.exists(fieldName),
+ compat.boolMustNot(compat.fuzzyMatch(fieldName, value))));
+ } else if (predicate == Cmp.LESS_THAN) {
+ return compat.lt(fieldName, value);
+ } else if (predicate == Cmp.LESS_THAN_EQUAL) {
+ return compat.lte(fieldName, value);
+ } else if (predicate == Cmp.GREATER_THAN) {
+ return compat.gt(fieldName, value);
+ } else if (predicate == Cmp.GREATER_THAN_EQUAL) {
+ return compat.gte(fieldName, value);
+ } else
+ throw new IllegalArgumentException("Predicate is not supported for string value: " + predicate);
+ } else if (value instanceof Geoshape && Mapping.getMapping(information.get(key)) == Mapping.DEFAULT) {
+ // geopoint
+ final Geoshape shape = (Geoshape) value;
+ Preconditions.checkArgument(predicate instanceof Geo && predicate != Geo.CONTAINS,
+ "Relation not supported on geopoint types: %s", predicate);
+
+ final Map<String,Object> query;
+ switch (shape.getType()) {
+ case CIRCLE:
+ final Geoshape.Point center = shape.getPoint();
+ query = compat.geoDistance(key, center.getLatitude(), center.getLongitude(), shape.getRadius());
+ break;
+ case BOX:
+ final Geoshape.Point southwest = shape.getPoint(0);
+ final Geoshape.Point northeast = shape.getPoint(1);
+ query = compat.geoBoundingBox(key, southwest.getLatitude(), southwest.getLongitude(),
+ northeast.getLatitude(), northeast.getLongitude());
+ break;
+ case POLYGON:
+ final List<List<Double>> points = IntStream.range(0, shape.size())
+ .mapToObj(i -> ImmutableList.of(shape.getPoint(i).getLongitude(),
+ shape.getPoint(i).getLatitude()))
+ .collect(Collectors.toList());
+ query = compat.geoPolygon(key, points);
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported or invalid search shape type for geopoint: "
+ + shape.getType());
+ }
+
+ return predicate == Geo.DISJOINT ? compat.boolMustNot(query) : query;
+ } else if (value instanceof Geoshape) {
+ Preconditions.checkArgument(predicate instanceof Geo,
+ "Relation not supported on geoshape types: %s", predicate);
+ final Geoshape shape = (Geoshape) value;
+ final Map<String,Object> geo;
+ switch (shape.getType()) {
+ case CIRCLE:
+ final Geoshape.Point center = shape.getPoint();
+ geo = ImmutableMap.of(OS_TYPE_KEY, "circle",
+ OS_GEO_COORDS_KEY, ImmutableList.of(center.getLongitude(), center.getLatitude()),
+ "radius", shape.getRadius() + "km");
+ break;
+ case BOX:
+ final Geoshape.Point southwest = shape.getPoint(0);
+ final Geoshape.Point northeast = shape.getPoint(1);
+ geo = ImmutableMap.of(OS_TYPE_KEY, "envelope",
+ OS_GEO_COORDS_KEY,
+ ImmutableList.of(
+ ImmutableList.of(southwest.getLongitude(), northeast.getLatitude()),
+ ImmutableList.of(northeast.getLongitude(), southwest.getLatitude())));
+ break;
+ case LINE:
+ final List lineCoords = IntStream.range(0, shape.size())
+ .mapToObj(i -> ImmutableList.of(shape.getPoint(i).getLongitude(),
+ shape.getPoint(i).getLatitude()))
+ .collect(Collectors.toList());
+ geo = ImmutableMap.of(OS_TYPE_KEY, "linestring", OS_GEO_COORDS_KEY, lineCoords);
+ break;
+ case POLYGON:
+ final List polyCoords = IntStream.range(0, shape.size())
+ .mapToObj(i -> ImmutableList.of(shape.getPoint(i).getLongitude(),
+ shape.getPoint(i).getLatitude()))
+ .collect(Collectors.toList());
+ geo = ImmutableMap.of(OS_TYPE_KEY, "polygon", OS_GEO_COORDS_KEY,
+ ImmutableList.of(polyCoords));
+ break;
+ case POINT:
+ geo = ImmutableMap.of(OS_TYPE_KEY, "point",
+ OS_GEO_COORDS_KEY, ImmutableList.of(shape.getPoint().getLongitude(),
+ shape.getPoint().getLatitude()));
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported or invalid search shape type: "
+ + shape.getType());
+ }
+
+ return compat.geoShape(key, geo, (Geo) predicate);
+ } else if (value instanceof Date || value instanceof Instant) {
+ Preconditions.checkArgument(predicate instanceof Cmp,
+ "Relation not supported on date types: %s", predicate);
+
+ if (value instanceof Instant) {
+ value = Date.from((Instant) value);
+ }
+ return getRelationFromCmp((Cmp) predicate, key, value);
+ } else if (value instanceof Boolean) {
+ final Cmp numRel = (Cmp) predicate;
+ switch (numRel) {
+ case EQUAL:
+ return compat.term(key, value);
+ case NOT_EQUAL:
+ return compat.boolMust(ImmutableList.of(compat.exists(key), compat.boolMustNot(compat.term(key, value))));
+ default:
+ throw new IllegalArgumentException("Boolean types only support EQUAL or NOT_EQUAL");
+ }
+ } else if (value instanceof UUID) {
+ if (predicate == Cmp.EQUAL) {
+ return compat.term(key, value);
+ } else if (predicate == Cmp.NOT_EQUAL) {
+ return compat.boolMust(ImmutableList.of(compat.exists(key), compat.boolMustNot(compat.term(key, value))));
+ } else {
+ throw new IllegalArgumentException("Only equal or not equal is supported for UUIDs: "
+ + predicate);
+ }
+ } else throw new IllegalArgumentException("Unsupported type: " + value);
+ } else if (condition instanceof Not) {
+ return compat.boolMustNot(getFilter(((Not) condition).getChild(),information));
+ } else if (condition instanceof And) {
+ final List queries = IterablesUtil.stream(condition.getChildren())
+ .map(c -> getFilter(c,information)).collect(Collectors.toList());
+ return compat.boolMust(queries);
+ } else if (condition instanceof Or) {
+ final List queries = IterablesUtil.stream(condition.getChildren())
+ .map(c -> getFilter(c,information)).collect(Collectors.toList());
+ return compat.boolShould(queries);
+ } else throw new IllegalArgumentException("Invalid condition: " + condition);
+ }
+
+ @Override
+ public Stream<String> query(IndexQuery query, KeyInformation.IndexRetriever informations,
+ BaseTransaction tx) throws BackendException {
+ final OpenSearchRequest sr = new OpenSearchRequest();
+ final Map<String,Object> esQuery = getFilter(query.getCondition(), informations.get(query.getStore()));
+ sr.setQuery(compat.prepareQuery(esQuery));
+ if (!query.getOrder().isEmpty()) {
+ addOrderToQuery(informations, sr, query.getOrder(), query.getStore());
+ }
+ sr.setFrom(0);
+ if (query.hasLimit()) {
+ sr.setSize(Math.min(query.getLimit(), batchSize));
+ } else {
+ sr.setSize(batchSize);
+ }
+
+ sr.setDisableSourceRetrieval(true);
+
+ OpenSearchResponse response;
+ try {
+ final String indexStoreName = getIndexStoreName(query.getStore());
+ final boolean useScroll = sr.getSize() >= batchSize;
+ response = client.search(indexStoreName,
+ compat.createRequestBody(sr, useScroll? NULL_PARAMETERS : TRACK_TOTAL_HITS_DISABLED_PARAMETERS),
+ useScroll);
+ log.debug("First Executed query [{}] in {} ms", query.getCondition(), response.getTook());
+ final Iterator<RawQuery.Result<String>> resultIterator = getResultsIterator(useScroll, response, sr.getSize());
+ final Stream<RawQuery.Result<String>> toReturn = withScrollCleanup(resultIterator,
+ StreamSupport.stream(Spliterators.spliteratorUnknownSize(resultIterator, Spliterator.ORDERED), false));
+ return (query.hasLimit() ? toReturn.limit(query.getLimit()) : toReturn).map(RawQuery.Result::getResult);
+ } catch (final IOException | UncheckedIOException e) {
+ throw new PermanentBackendException(e);
+ }
+ }
+
+ private Iterator<RawQuery.Result<String>> getResultsIterator(boolean useScroll, OpenSearchResponse response, int windowSize){
+ return (useScroll)? new OpenSearchScroll(client, response, windowSize) : response.getResults().iterator();
+ }
+
+ /**
+ * Ensures the server-side scroll context is released when the returned stream is closed, including when a
+ * limit short-circuits consumption before the scroll is naturally exhausted. Shared by every {@code query(...)}
+ * overload that may back its result {@code Stream} with an {@link OpenSearchScroll}, so a new scroll-backed
+ * query path cannot be added later without this wiring. A no-op (returns {@code stream} unchanged) when
+ * {@code resultIterator} is not a scroll (i.e. the non-scroll, single-page result path).
+ * Package-visible for direct unit testing without a live cluster.
+ */
+ static Stream<RawQuery.Result<String>> withScrollCleanup(Iterator<RawQuery.Result<String>> resultIterator,
+ Stream<RawQuery.Result<String>> stream) {
+ if (resultIterator instanceof OpenSearchScroll) {
+ final OpenSearchScroll scroll = (OpenSearchScroll) resultIterator;
+
+ return stream.onClose(scroll::close);
+ }
+
+ return stream;
+ }
+
+ private String convertToEsDataType(Class<?> dataType, Mapping mapping) {
+ if(String.class.isAssignableFrom(dataType)) {
+ return "string";
+ }
+ else if (Integer.class.isAssignableFrom(dataType)) {
+ return "integer";
+ }
+ else if (Long.class.isAssignableFrom(dataType)) {
+ return "long";
+ }
+ else if (Float.class.isAssignableFrom(dataType)) {
+ return "float";
+ }
+ else if (Double.class.isAssignableFrom(dataType)) {
+ return "double";
+ }
+ else if (Boolean.class.isAssignableFrom(dataType)) {
+ return "boolean";
+ }
+ else if (Date.class.isAssignableFrom(dataType)) {
+ return "date";
+ }
+ else if (Instant.class.isAssignableFrom(dataType)) {
+ return "date";
+ }
+ else if (Geoshape.class.isAssignableFrom(dataType)) {
+ return mapping == Mapping.DEFAULT ? "geo_point" : "geo_shape";
+ }
+
+ return null;
+ }
+
+ private OpenSearchResponse runCommonQuery(RawQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx, int size,
+ boolean useScroll) throws BackendException{
+ final OpenSearchRequest sr = new OpenSearchRequest();
+ sr.setQuery(compat.queryString(query.getQuery()));
+ if (!query.getOrders().isEmpty()) {
+ addOrderToQuery(informations, sr, query.getOrders(), query.getStore());
+ }
+ sr.setFrom(0);
+ sr.setSize(size);
+ sr.setDisableSourceRetrieval(true);
+ try {
+ Map<String, Object> requestBody = compat.createRequestBody(sr, query.getParameters());
+ if(!useScroll) {
+ if (requestBody == null) {
+ requestBody = TRACK_TOTAL_HITS_DISABLED_REQUEST_BODY;
+ } else {
+ requestBody.put(TRACK_TOTAL_HITS_PARAMETER, false);
+ }
+ }
+ return client.search(
+ getIndexStoreName(query.getStore()),
+ requestBody,
+ useScroll);
+ } catch (final IOException | UncheckedIOException e) {
+ throw new PermanentBackendException(e);
+ }
+ }
+
+ private long runCountQuery(RawQuery query) throws BackendException{
+ try {
+ long countTotal = client.countTotal(
+ getIndexStoreName(query.getStore()),
+ compat.createRequestBody(compat.queryString(query.getQuery()), query.getParameters()));
+ return QueryUtil.applyOffsetWithQueryLimitAfterCount(countTotal, query.getOffset(), query);
+ } catch (final IOException | UncheckedIOException e) {
+ throw new PermanentBackendException(e);
+ }
+ }
+
+ private void addOrderToQuery(KeyInformation.IndexRetriever informations, OpenSearchRequest sr, final List<IndexQuery.OrderEntry> orders,
+ String store) {
+ for (final IndexQuery.OrderEntry orderEntry : orders) {
+ final String order = orderEntry.getOrder().name();
+ final KeyInformation information = informations.get(store).get(orderEntry.getKey());
+ final Mapping mapping = Mapping.getMapping(information);
+ final Class<?> datatype = orderEntry.getDatatype();
+ final String key = hasDualStringMapping(information) ? getDualMappingName(orderEntry.getKey()) : orderEntry.getKey();
+ sr.addSort(key, order.toLowerCase(), convertToEsDataType(datatype, mapping));
+ }
+ }
+
+ @Override
+ public Stream<RawQuery.Result<String>> query(RawQuery query, KeyInformation.IndexRetriever information,
+ BaseTransaction tx) throws BackendException {
+ final int size = query.hasLimit() ? Math.min(query.getLimit() + query.getOffset(), batchSize) : batchSize;
+ final boolean useScroll = size >= batchSize;
+ final OpenSearchResponse response = runCommonQuery(query, information, tx, size, useScroll);
+ log.debug("First Executed query [{}] in {} ms", query.getQuery(), response.getTook());
+ final Iterator<RawQuery.Result<String>> resultIterator = getResultsIterator(useScroll, response, size);
+ // Same server-side scroll-context release as query(IndexQuery, ...): without this, a RawQuery (e.g. Atlas'
+ // native "v.field:value" queries via graph.indexQuery(indexName, queryString)) that limits/skips results
+ // before the scroll is naturally exhausted leaks the OpenSearch scroll context.
+ final Stream<RawQuery.Result<String>> toReturn = withScrollCleanup(resultIterator,
+ StreamSupport.stream(Spliterators.spliteratorUnknownSize(resultIterator, Spliterator.ORDERED),
+ false).skip(query.getOffset()));
+ return query.hasLimit() ? toReturn.limit(query.getLimit()) : toReturn;
+ }
+
+ @Override
+ public Number queryAggregation(IndexQuery query, KeyInformation.IndexRetriever information, BaseTransaction tx, Aggregation aggregation) throws BackendException {
+ final OpenSearchRequest sr = new OpenSearchRequest();
+ final Map<String, Object> esQuery = getFilter(query.getCondition(), information.get(query.getStore()));
+ sr.setQuery(compat.prepareQuery(esQuery));
+ try {
+ final String indexName = getIndexStoreName(query.getStore());
+ final Map<String,Object> requestData = compat.createRequestBody(sr, null);
+ switch (aggregation.getType()) {
+ case COUNT: return QueryUtil.applyQueryLimitAfterCount(client.countTotal(indexName, requestData), query);
+ case MIN: return client.min(indexName, requestData, aggregation.getFieldName(), aggregation.getDataType());
+ case MAX: return client.max(indexName, requestData, aggregation.getFieldName(), aggregation.getDataType());
+ case AVG: return client.avg(indexName, requestData, aggregation.getFieldName());
+ case SUM: return client.sum(indexName, requestData, aggregation.getFieldName(), aggregation.getDataType());
+ default: throw new UnsupportedOperationException();
+ }
+ } catch (final IOException | UncheckedIOException e) {
+ throw new PermanentBackendException(e);
+ }
+ }
+
+ @Override
+ public Long totals(RawQuery query, KeyInformation.IndexRetriever information,
+ BaseTransaction tx) throws BackendException {
+ long startTime = System.currentTimeMillis();
+ long count = runCountQuery(query);
+ if(log.isDebugEnabled()){
+ log.debug("Executed count query [{}] in {} ms", query.getQuery(), System.currentTimeMillis() - startTime);
+ }
+ return count;
+ }
+
+ @Override
+ public boolean supports(KeyInformation information, JanusGraphPredicate janusgraphPredicate) {
+ final Class<?> dataType = information.getDataType();
+ final Mapping mapping = Mapping.getMapping(information);
+ if (mapping!=Mapping.DEFAULT && !AttributeUtils.isString(dataType) &&
+ !(AttributeUtils.isGeo(dataType) && (mapping==Mapping.BKD || mapping==Mapping.PREFIX_TREE))) return false;
+
+ if (Number.class.isAssignableFrom(dataType)) {
+ return janusgraphPredicate instanceof Cmp;
+ } else if (dataType == Geoshape.class) {
+ switch(mapping) {
+ case DEFAULT:
+ return janusgraphPredicate instanceof Geo && janusgraphPredicate != Geo.CONTAINS;
+ case BKD:
+ case PREFIX_TREE:
+ return janusgraphPredicate instanceof Geo;
+ }
+ } else if (AttributeUtils.isString(dataType)) {
+ switch(mapping) {
+ case DEFAULT:
+ case TEXT:
+ return janusgraphPredicate == Text.CONTAINS || janusgraphPredicate == Text.NOT_CONTAINS
+ || janusgraphPredicate == Text.CONTAINS_FUZZY || janusgraphPredicate == Text.NOT_CONTAINS_FUZZY
+ || janusgraphPredicate == Text.CONTAINS_PREFIX || janusgraphPredicate == Text.NOT_CONTAINS_PREFIX
+ || janusgraphPredicate == Text.CONTAINS_REGEX || janusgraphPredicate == Text.NOT_CONTAINS_REGEX
+ || janusgraphPredicate == Text.CONTAINS_PHRASE || janusgraphPredicate == Text.NOT_CONTAINS_PHRASE;
+ case STRING:
+ return janusgraphPredicate instanceof Cmp
+ || janusgraphPredicate==Text.REGEX || janusgraphPredicate==Text.NOT_REGEX
+ || janusgraphPredicate==Text.PREFIX || janusgraphPredicate==Text.NOT_PREFIX
+ || janusgraphPredicate == Text.FUZZY || janusgraphPredicate == Text.NOT_FUZZY;
+ case TEXTSTRING:
+ return janusgraphPredicate instanceof Text || janusgraphPredicate instanceof Cmp;
+ }
+ } else if (dataType == Date.class || dataType == Instant.class) {
+ return janusgraphPredicate instanceof Cmp;
+ } else if (dataType == Boolean.class) {
+ return janusgraphPredicate == Cmp.EQUAL || janusgraphPredicate == Cmp.NOT_EQUAL;
+ } else if (dataType == UUID.class) {
+ return janusgraphPredicate == Cmp.EQUAL || janusgraphPredicate==Cmp.NOT_EQUAL;
+ }
+ return false;
+ }
+
+
+ @Override
+ public boolean supports(KeyInformation information) {
+ final Class<?> dataType = information.getDataType();
+ final Mapping mapping = Mapping.getMapping(information);
+ if (Number.class.isAssignableFrom(dataType) || dataType == Date.class || dataType== Instant.class
+ || dataType == Boolean.class || dataType == UUID.class) {
+ return mapping == Mapping.DEFAULT;
+ } else if (AttributeUtils.isString(dataType)) {
+ return mapping == Mapping.DEFAULT || mapping == Mapping.STRING
+ || mapping == Mapping.TEXT || mapping == Mapping.TEXTSTRING;
+ } else if (AttributeUtils.isGeo(dataType)) {
+ return mapping == Mapping.DEFAULT || mapping == Mapping.BKD || mapping == Mapping.PREFIX_TREE && supportsGeoShapePrefixTree;
+ }
+ return false;
+ }
+
+ @Override
+ public String mapKey2Field(String key, KeyInformation information) {
+ IndexProvider.checkKeyValidity(key);
+ return key.replace(' ', IndexProvider.REPLACEMENT_CHAR).replace('.', IndexProvider.REPLACEMENT_CHAR);
+ }
+
+ @Override
+ public IndexFeatures getFeatures() {
+ return compat.getIndexFeatures();
+ }
+
+ @Override
+ public BaseTransactionConfigurable beginTransaction(BaseTransactionConfig config) throws BackendException {
+ return new DefaultTransaction(config);
+ }
+
+ @Override
+ public void close() throws BackendException {
+ try {
+ client.close();
+ } catch (final IOException e) {
+ throw new PermanentBackendException(e);
+ }
+
+ }
+
+ @Override
+ public void clearStorage() throws BackendException {
+ try {
+ client.deleteIndex(indexName);
+ } catch (final Exception e) {
+ throw new PermanentBackendException("Could not delete index " + indexName, e);
+ } finally {
+ close();
+ }
+ }
+
+ @Override
+ public void clearStore(String storeName) throws BackendException {
+ try {
+ client.clearStore(indexName, storeName);
+ } catch (final Exception e) {
+ throw new PermanentBackendException("Could not clear store " + indexName + "_" + storeName, e);
+ }
+ }
+
+ @Override
+ public boolean exists() throws BackendException {
+ try {
+ return client.indexExists(indexName);
+ } catch (final IOException e) {
+ throw new PermanentBackendException("Could not check if index " + indexName + " exists", e);
+ }
+ }
+
+ OpenSearchMajorVersion getVersion() {
+ return client.getMajorVersion();
+ }
+
+ boolean isUseMappingForES7(){
+ return useMappingTypes;
+ }
+
+ private static String parameterizedScriptPrepare(String ... lines){
+ return Arrays.stream(lines).map(String::trim).collect(Collectors.joining(""));
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchMajorVersion.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchMajorVersion.java
new file mode 100644
index 000000000..c74aafe8f
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchMajorVersion.java
@@ -0,0 +1,56 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public enum OpenSearchMajorVersion {
+
+ TWO(2),
+
+ THREE(3),
+
+ ;
+
+ static final Pattern PATTERN = Pattern.compile("(\\d+)\\.\\d+\\.\\d+.*");
+
+ final int value;
+
+ OpenSearchMajorVersion(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return value;
+ }
+
+ public static OpenSearchMajorVersion parse(final String value) {
+ final Matcher m = value != null ? PATTERN.matcher(value) : null;
+ switch (m != null && m.find() ? Integer.parseInt(m.group(1)) : -1) {
+ case 2:
+ return OpenSearchMajorVersion.TWO;
+ case 3:
+ return OpenSearchMajorVersion.THREE;
+ default:
+ throw new IllegalArgumentException("Unsupported OpenSearch server major version: " + value);
+ }
+ }
+
+ /** OpenSearch 2.x and 3.x use typeless index wire format (ES7+ semantics). */
+ public boolean supportsTypelessWireFormat() {
+ return this == TWO || this == THREE;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchMutation.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchMutation.java
new file mode 100644
index 000000000..c89d46c10
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchMutation.java
@@ -0,0 +1,90 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_UPSERT_KEY;
+
+public class OpenSearchMutation {
+
+ public enum RequestType {
+
+ INDEX,
+
+ UPDATE,
+
+ DELETE
+
+ }
+
+ private final RequestType requestType;
+
+ private final String index;
+
+ private final String type;
+
+ private final String id;
+
+ private final Map source;
+
+ private OpenSearchMutation(RequestType requestType, String index, String type, String id, Map source) {
+ this.requestType = requestType;
+ this.index = index;
+ this.type = type;
+ this.id = id;
+ this.source = source;
+ }
+
+ public static OpenSearchMutation createDeleteRequest(String index, String type, String id) {
+ return new OpenSearchMutation(RequestType.DELETE, index, type, id, null);
+ }
+
+ public static OpenSearchMutation createIndexRequest(String index, String type, String id, Map source) {
+ return new OpenSearchMutation(RequestType.INDEX, index, type, id, source);
+ }
+
+ public static OpenSearchMutation createUpdateRequest(String index, String type, String id, Map source) {
+ return new OpenSearchMutation(RequestType.UPDATE, index, type, id, source);
+ }
+
+ public static OpenSearchMutation createUpdateRequest(String index, String type, String id, ImmutableMap.Builder builder, Map upsert) {
+ final Map source = upsert == null ? builder.build() : builder.put(OS_UPSERT_KEY, upsert).build();
+ return new OpenSearchMutation(RequestType.UPDATE, index, type, id, source);
+ }
+
+ public RequestType getRequestType() {
+ return requestType;
+ }
+
+ public String getIndex() {
+ return index;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public Map getSource() {
+ return source;
+ }
+
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchRequest.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchRequest.java
new file mode 100644
index 000000000..57fc6ce62
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchRequest.java
@@ -0,0 +1,120 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonProperty;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class OpenSearchRequest {
+
+ private Map<String,Object> query;
+
+ private Integer size;
+
+ private Integer from;
+
+ private final List<Map<String,RestSortInfo>> sorts;
+
+ private List<String> fields;
+
+ private boolean disableSourceRetrieval;
+
+ public OpenSearchRequest() {
+ this.sorts = new ArrayList<>();
+ this.fields = new ArrayList<>();
+ }
+
+ public Map<String,Object> getQuery() {
+ return query;
+ }
+
+ public void setQuery(Map<String,Object> query) {
+ this.query = query;
+ }
+
+ public Integer getSize() {
+ return size;
+ }
+
+ public void setSize(Integer size) {
+ this.size = size;
+ }
+
+ public Integer getFrom() {
+ return from;
+ }
+
+ public void setFrom(Integer from) {
+ this.from = from;
+ }
+
+ public List<Map<String,RestSortInfo>> getSorts() {
+ return sorts;
+ }
+
+ public List<String> getFields() {
+ return fields;
+ }
+
+ public void setFields(List<String> fields) {
+ this.fields = fields;
+ }
+
+ public void addSort(String key, String order, String unmappedType) {
+ this.sorts.add(ImmutableMap.of(key, new RestSortInfo(order, unmappedType)));
+ }
+
+ public boolean isDisableSourceRetrieval() {
+ return disableSourceRetrieval;
+ }
+
+ public void setDisableSourceRetrieval(boolean disableSourceRetrieval) {
+ this.disableSourceRetrieval = disableSourceRetrieval;
+ }
+
+ public static class RestSortInfo {
+
+ String order;
+
+ @JsonProperty("unmapped_type")
+ String unmappedType;
+
+ public RestSortInfo(String order, String unmappedType) {
+ this.order = order;
+ this.unmappedType = unmappedType;
+ }
+
+ public String getOrder() {
+ return order;
+ }
+
+ public void setOrder(String order) {
+ this.order = order;
+ }
+
+ public String getUnmappedType() {
+ return unmappedType;
+ }
+
+ public void setUnmappedType(String unmappedType) {
+ this.unmappedType = unmappedType;
+ }
+
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchResponse.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchResponse.java
new file mode 100644
index 000000000..7654b7e0a
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchResponse.java
@@ -0,0 +1,57 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch;
+
+import org.janusgraph.diskstorage.indexing.RawQuery;
+
+import java.util.List;
+import java.util.stream.Stream;
+
+public class OpenSearchResponse {
+
+ private long took;
+
+ private String scrollId;
+
+ private List<RawQuery.Result<String>> results;
+
+ public long getTook() {
+ return took;
+ }
+
+ public void setTook(long took) {
+ this.took = took;
+ }
+
+ public Stream<RawQuery.Result<String>> getResults() {
+ return results.stream();
+ }
+
+ public void setResults(List<RawQuery.Result<String>> results) {
+ this.results = results;
+ }
+
+ public int numResults() {
+ return results.size();
+ }
+
+ public String getScrollId() {
+ return scrollId;
+ }
+
+ public void setScrollId(String scrollId) {
+ this.scrollId = scrollId;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchScroll.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchScroll.java
new file mode 100644
index 000000000..76d9dbc4c
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchScroll.java
@@ -0,0 +1,121 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch;
+
+import org.janusgraph.diskstorage.indexing.RawQuery;
+import org.janusgraph.diskstorage.indexing.RawQuery.Result;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+/**
+ * @author David Clement (david.clement90@laposte.net)
+ *
+ * <p>Iterates the results of a scrolled OpenSearch search. The server-side scroll context is a finite resource
+ * and must be released. The scroll context is deleted:
+ * <ul>
+ * <li>on normal exhaustion (last batch smaller than the batch size),</li>
+ * <li>on early termination (e.g. when the consuming {@code Stream} is limited and closed), via {@link #close()},</li>
+ * <li>on exception while fetching the next batch.</li>
+ * </ul>
+ * Deletion is best-effort: a failure to delete the scroll context is logged but never masks the original result
+ * or exception.</p>
+ */
+public class OpenSearchScroll implements Iterator<RawQuery.Result<String>>, Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(OpenSearchScroll.class);
+
+ private final BlockingQueue<RawQuery.Result<String>> queue;
+ private final OpenSearchClient client;
+ private final int batchSize;
+
+ private boolean isFinished;
+ private boolean scrollDeleted;
+ private String scrollId;
+
+ public OpenSearchScroll(OpenSearchClient client, OpenSearchResponse initialResponse, int nbDocByQuery) {
+ queue = new LinkedBlockingQueue<>();
+ this.client = client;
+ this.batchSize = nbDocByQuery;
+ update(initialResponse);
+ }
+
+ private void update(OpenSearchResponse response) {
+ response.getResults().forEach(queue::add);
+ this.scrollId = response.getScrollId();
+ this.isFinished = response.numResults() < this.batchSize;
+ if (isFinished) {
+ deleteScrollQuietly();
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ try {
+ if (!queue.isEmpty()) {
+ return true;
+ }
+ if (isFinished) {
+ return false;
+ }
+ final OpenSearchResponse res = client.search(scrollId);
+ update(res);
+ return res.numResults() > 0;
+ } catch (final IOException e) {
+ // Fetching the next batch failed: release the scroll context best-effort, then surface the original error.
+ deleteScrollQuietly();
+ throw new UncheckedIOException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public Result<String> next() {
+ if (hasNext()) {
+ return queue.remove();
+ }
+ throw new NoSuchElementException();
+ }
+
+ /**
+ * Releases the server-side scroll context. Safe to call multiple times and safe to call on a partially or fully
+ * consumed iterator. Intended to be wired to the consuming {@code Stream}'s close handler so that early
+ * termination (e.g. {@code stream.limit(n)}) does not leak scroll contexts.
+ */
+ @Override
+ public void close() {
+ deleteScrollQuietly();
+ }
+
+ private void deleteScrollQuietly() {
+ if (scrollDeleted || scrollId == null) {
+ return;
+ }
+ // Mark deleted before the call so a failure does not cause repeated delete attempts.
+ scrollDeleted = true;
+ try {
+ client.deleteScroll(scrollId);
+ } catch (final IOException e) {
+ // Best-effort cleanup: the scroll context will expire server-side on its own; do not mask the caller's flow.
+ log.warn("Failed to delete OpenSearch scroll context {} (will expire server-side).", scrollId, e);
+ }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchSetup.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchSetup.java
new file mode 100644
index 000000000..8bff44c86
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/OpenSearchSetup.java
@@ -0,0 +1,78 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch;
+
+import com.google.common.base.Preconditions;
+import org.janusgraph.diskstorage.configuration.Configuration;
+import org.janusgraph.diskstorage.opensearch.rest.RestClientSetup;
+import org.janusgraph.util.system.ConfigurationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Create an ES {@link org.opensearch.client.RestClient} from a JanusGraph
+ * {@link org.janusgraph.diskstorage.configuration.Configuration}.
+ * <p>
+ * Any key-value pairs under the {@link org.janusgraph.diskstorage.opensearch.OpenSearchIndex#OS_CREATE_EXTRAS_NS} namespace
+ * are copied into the OpenSearch settings builder. This allows overriding arbitrary
+ * ES settings from within the JanusGraph properties file.
+ * <p>
+ * Assumes that an ES cluster is already running. It does not attempt to start an
+ * embedded ES instance. It just connects to whatever hosts are given in
+ * {@link org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration#INDEX_HOSTS}.
+ */
+public enum OpenSearchSetup {
+
+ /**
+ * Create an ES RestClient connected to
+ * {@link org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration#INDEX_HOSTS}.
+ */
+ REST_CLIENT {
+ @Override
+ public Connection connect(Configuration config) throws IOException {
+ return new Connection(new RestClientSetup().connect(config));
+ }
+ };
+
+ static Map<String, Object> getSettingsFromJanusGraphConf(Configuration config) {
+ final Map<String, String> settings = ConfigurationUtil.getSettingsFromJanusGraphConf(config, OpenSearchIndex.OS_CREATE_EXTRAS_NS);
+ if(log.isDebugEnabled()){
+ settings.forEach((key, val) -> log.debug("[ES ext.* cfg] Set {}: {}", key, val));
+ log.debug("Loaded {} settings from the {} JanusGraph config namespace", settings.size(), OpenSearchIndex.OS_CREATE_EXTRAS_NS);
+ }
+ return new HashMap<>(settings);
+ }
+
+ private static final Logger log = LoggerFactory.getLogger(OpenSearchSetup.class);
+
+ public abstract Connection connect(Configuration config) throws IOException;
+
+ public static class Connection {
+
+ private final OpenSearchClient client;
+
+ public Connection(OpenSearchClient client) {
+ this.client = Preconditions.checkNotNull(client, "Unable to instantiate OpenSearch Client object");
+ }
+
+ public OpenSearchClient getClient() {
+ return client;
+ }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/AbstractOSCompat.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/AbstractOSCompat.java
new file mode 100644
index 000000000..7dd04c6ca
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/AbstractOSCompat.java
@@ -0,0 +1,235 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.compat;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.janusgraph.core.Cardinality;
+import org.janusgraph.core.attribute.Geo;
+import org.janusgraph.core.schema.Mapping;
+import org.janusgraph.core.schema.Parameter;
+import org.janusgraph.diskstorage.opensearch.OpenSearchRequest;
+import org.janusgraph.diskstorage.indexing.IndexFeatures;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_ANALYZER;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_ID_KEY;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_LANG_KEY;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_PARAMS_FIELDS_KEY;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_PARAMS_KEY;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_SCRIPT_KEY;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_SOURCE_KEY;
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.OS_TYPE_KEY;
+
+/**
+ * Base class for building OpenSearch mapping and query objects.
+ */
+public abstract class AbstractOSCompat {
+
+ static final Map<String,Object> MATCH_ALL = ImmutableMap.of("match_all", Collections.EMPTY_MAP);
+
+ static IndexFeatures.Builder coreFeatures() {
+ return new IndexFeatures.Builder()
+ .setDefaultStringMapping(Mapping.TEXT)
+ .supportedStringMappings(Mapping.TEXT, Mapping.TEXTSTRING, Mapping.STRING)
+ .setWildcardField("_all")
+ .supportsCardinality(Cardinality.SINGLE)
+ .supportsCardinality(Cardinality.LIST)
+ .supportsCardinality(Cardinality.SET)
+ .supportsNanoseconds()
+ .supportsCustomAnalyzer()
+ .supportsGeoExists()
+ .supportNotQueryNormalForm()
+ ;
+ }
+
+ public abstract IndexFeatures getIndexFeatures();
+
+ public Map<String,Object> createKeywordMapping() {
+ return ImmutableMap.of(OS_TYPE_KEY, "keyword");
+ }
+
+ public Map<String,Object> createTextMapping(String textAnalyzer) {
+ final ImmutableMap.Builder builder = ImmutableMap.builder().put(OS_TYPE_KEY, "text");
+ return (textAnalyzer != null ? builder.put(OS_ANALYZER, textAnalyzer) : builder).build();
+ }
+
+ public String scriptLang() {
+ return "painless";
+ }
+
+ public ImmutableMap.Builder<String, Object> prepareScript(String source) {
+ Map<String, Object> script = ImmutableMap.of(OS_SOURCE_KEY, source,
+ OS_LANG_KEY, scriptLang());
+ return ImmutableMap.<String, Object>builder().put(OS_SCRIPT_KEY, script);
+ }
+
+ public ImmutableMap.Builder<String, Object> prepareStoredScript(String scriptId, List<Map<String, Object>> fields) {
+ Map<String, Object> script = ImmutableMap.of(OS_ID_KEY, scriptId,
+ OS_PARAMS_KEY, ImmutableMap.of(OS_PARAMS_FIELDS_KEY, fields));
+ return ImmutableMap.<String, Object>builder().put(OS_SCRIPT_KEY, script);
+ }
+
+ public ImmutableMap.Builder<String, Object> prepareInlineScript(String source, List<Map<String, Object>> fields) {
+ Map<String, Object> script = ImmutableMap.of(OS_SOURCE_KEY, source,
+ OS_PARAMS_KEY, ImmutableMap.of(OS_PARAMS_FIELDS_KEY, fields),
+ OS_LANG_KEY, scriptLang());
+ return ImmutableMap.<String, Object>builder().put(OS_SCRIPT_KEY, script);
+ }
+
+ public Map<String,Object> prepareQuery(Map<String,Object> query) {
+ return query;
+ }
+
+ public Map<String,Object> term(String key, Object value) {
+ return ImmutableMap.of("term", ImmutableMap.of(key, value));
+ }
+
+ public Map<String,Object> contains(String key, List<String> terms) {
+ return boolMust(terms.stream().map(term -> term(key, term)).collect(Collectors.toList()));
+ }
+
+ public Map<String,Object> boolMust(List<Map<String,Object>> queries) {
+ return queries.size() > 1 ? ImmutableMap.of("bool", ImmutableMap.of("must", queries)) : queries.get(0);
+ }
+
+ public Map<String,Object> boolMustNot(Map<String,Object> query) {
+ return ImmutableMap.of("bool", ImmutableMap.of("must_not", query));
+ }
+
+ public Map<String,Object> boolShould(List<Map<String,Object>> queries) {
+ return ImmutableMap.of("bool", ImmutableMap.of("should", queries));
+ }
+
+ public Map<String,Object> boolFilter(Map<String,Object> query) {
+ return ImmutableMap.of("bool", ImmutableMap.of("must", MATCH_ALL, "filter", query));
+ }
+
+ public Map<String,Object> lt(String key, Object value) {
+ return ImmutableMap.of("range", ImmutableMap.of(key, ImmutableMap.of("lt", value)));
+ }
+
+ public Map<String,Object> lte(String key, Object value) {
+ return ImmutableMap.of("range", ImmutableMap.of(key, ImmutableMap.of("lte", value)));
+ }
+
+ public Map<String,Object> gt(String key, Object value) {
+ return ImmutableMap.of("range", ImmutableMap.of(key, ImmutableMap.of("gt", value)));
+ }
+
+ public Map<String,Object> gte(String key, Object value) {
+ return ImmutableMap.of("range", ImmutableMap.of(key, ImmutableMap.of("gte", value)));
+ }
+
+ public Map<String,Object> prefix(String key, Object value) {
+ return ImmutableMap.of("prefix", ImmutableMap.of(key, value));
+ }
+
+ public Map<String,Object> regexp(String key, Object value) {
+ return ImmutableMap.of("regexp", ImmutableMap.of(key, value));
+ }
+
+ public Map<String,Object> exists(String key) {
+ return ImmutableMap.of("exists", ImmutableMap.of("field", key));
+ }
+
+ public Map<String,Object> match(String key, Object value) {
+ return match(key, value, null);
+ }
+
+ public Map<String,Object> matchPhrase(String key, Object value) {
+ return ImmutableMap.of("match_phrase", ImmutableMap.of(key, value));
+ }
+
+ public Map<String,Object> fuzzyMatch(String key, Object value) {
+ return match(key, value, "AUTO");
+ }
+
+ public Map<String,Object> match(String key, Object value, String fuzziness) {
+ final ImmutableMap.Builder builder = ImmutableMap.builder().put("query", value);
+ builder.put("operator", "and");
+ if (fuzziness != null) builder.put("fuzziness", fuzziness);
+ return ImmutableMap.of("match", ImmutableMap.of(key, builder.build()));
+ }
+
+ public Map<String,Object> queryString(Object query) {
+ return ImmutableMap.of("query_string", ImmutableMap.of("query", query));
+ }
+
+ public Map<String,Object> geoDistance(String key, double lat, double lon, double radius) {
+ return filter(ImmutableMap.of("geo_distance", ImmutableMap.of("distance", radius + "km", key, ImmutableList.of(lon, lat))));
+ }
+
+ public Map<String,Object> geoBoundingBox(String key, double minLat, double minLon, double maxLat, double maxLon) {
+ return filter(ImmutableMap.of("geo_bounding_box", ImmutableMap.of(key,ImmutableMap.of(
+ "top_left", ImmutableList.of(minLon, maxLat),"bottom_right", ImmutableList.of(maxLon, minLat)))));
+ }
+
+ public Map<String,Object> geoPolygon(String key, List<List<Double>> points) {
+ return filter(ImmutableMap.of("geo_polygon", ImmutableMap.of(key, ImmutableMap.of("points", points))));
+ }
+
+ public Map<String,Object> geoShape(String key, Map<String,Object> geoShape, Geo predicate) {
+ final String relation = predicate == Geo.INTERSECT ? "intersects" : predicate.name().toLowerCase();
+ return filter(ImmutableMap.of("geo_shape", ImmutableMap.of(key, ImmutableMap.of("shape", geoShape, "relation", relation))));
+ }
+
+ public Map<String,Object> filter(Map<String,Object> query) {
+ return boolFilter(query);
+ }
+
+ public Map<String,Object> createRequestBody(OpenSearchRequest request, Parameter[] parameters) {
+ final Map<String,Object> requestBody = createRequestBody(request.getQuery(), parameters);
+
+ if(request.getSize() != null){
+ requestBody.put("size", request.getSize());
+ }
+
+ if(request.getFrom() != null){
+ requestBody.put("from", request.getFrom());
+ }
+
+ if (!request.getSorts().isEmpty()) {
+ requestBody.put("sort", request.getSorts());
+ }
+
+ if(request.isDisableSourceRetrieval()){
+ requestBody.put("_source", false);
+ }
+
+ return requestBody;
+ }
+
+ public Map<String,Object> createRequestBody(Map<String,Object> query, Parameter[] parameters) {
+ final Map<String,Object> requestBody = new HashMap<>();
+
+ if(query != null){
+ requestBody.put("query", query);
+ }
+
+ if(parameters != null){
+ for(Parameter parameter : parameters){
+ requestBody.put(parameter.key(), parameter.value());
+ }
+ }
+
+ return requestBody;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/OS2Compat.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/OS2Compat.java
new file mode 100644
index 000000000..9aacada0a
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/OS2Compat.java
@@ -0,0 +1,29 @@
+// Copyright 2019 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.compat;
+
+import org.janusgraph.diskstorage.indexing.IndexFeatures;
+
+import static org.janusgraph.diskstorage.opensearch.OpenSearchConstants.CUSTOM_ALL_FIELD;
+
+public class OS2Compat extends AbstractOSCompat {
+
+ private static final IndexFeatures FEATURES = coreFeatures().setWildcardField(CUSTOM_ALL_FIELD).supportsGeoContains().build();
+
+ @Override
+ public IndexFeatures getIndexFeatures() {
+ return FEATURES;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/OS3Compat.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/OS3Compat.java
new file mode 100644
index 000000000..bcae6a0a2
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/OS3Compat.java
@@ -0,0 +1,21 @@
+// Copyright 2019 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.compat;
+
+/**
+ * OpenSearch 3.x compatibility layer. Wire format and index features match 2.x typeless indices.
+ */
+public class OS3Compat extends OS2Compat {
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/OSCompatUtils.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/OSCompatUtils.java
new file mode 100644
index 000000000..b70cd1a81
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/compat/OSCompatUtils.java
@@ -0,0 +1,33 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.compat;
+
+import org.janusgraph.diskstorage.BackendException;
+import org.janusgraph.diskstorage.PermanentBackendException;
+import org.janusgraph.diskstorage.opensearch.OpenSearchMajorVersion;
+
+public class OSCompatUtils {
+
+ public static AbstractOSCompat acquireCompatForVersion(OpenSearchMajorVersion openSearchMajorVersion) throws BackendException {
+ switch (openSearchMajorVersion) {
+ case TWO:
+ return new OS2Compat();
+ case THREE:
+ return new OS3Compat();
+ default:
+ throw new PermanentBackendException("Unsupported OpenSearch version: " + openSearchMajorVersion);
+ }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/mapping/IndexMapping.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/mapping/IndexMapping.java
new file mode 100644
index 000000000..17d6fd9bf
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/mapping/IndexMapping.java
@@ -0,0 +1,46 @@
+// Copyright 2019 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.mapping;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+
+import java.util.Map;
+
+/**
+ * Serialization of OpenSearch index mapping.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class IndexMapping {
+
+ private Map<String, Object> properties;
+
+ private String dynamic;
+
+ public Map<String, Object> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, Object> properties) {
+ this.properties = properties;
+ }
+
+ public boolean isDynamic() {
+ return dynamic == null || "true".equalsIgnoreCase(dynamic);
+ }
+
+ public void setDynamic(String dynamic) {
+ this.dynamic = dynamic;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/mapping/TypedIndexMappings.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/mapping/TypedIndexMappings.java
new file mode 100644
index 000000000..982602271
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/mapping/TypedIndexMappings.java
@@ -0,0 +1,37 @@
+// Copyright 2019 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.mapping;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+
+import java.util.Map;
+
+/**
+ * Serialization of OpenSearch index mappings with types.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class TypedIndexMappings {
+
+ private Map<String, IndexMapping> mappings;
+
+ public Map<String, IndexMapping> getMappings() {
+ return mappings;
+ }
+
+ public void setMappings(Map<String, IndexMapping> mappings){
+ this.mappings = mappings;
+ }
+
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/mapping/TypelessIndexMappings.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/mapping/TypelessIndexMappings.java
new file mode 100644
index 000000000..e2c6084af
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/mapping/TypelessIndexMappings.java
@@ -0,0 +1,34 @@
+// Copyright 2019 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.mapping;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+
+/**
+ * Serialization of OpenSearch index mappings without types.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class TypelessIndexMappings {
+
+ private IndexMapping mappings;
+
+ public IndexMapping getMappings() {
+ return mappings;
+ }
+
+ public void setMappings(IndexMapping mappings) {
+ this.mappings = mappings;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestAggResponse.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestAggResponse.java
new file mode 100644
index 000000000..ac0687b75
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestAggResponse.java
@@ -0,0 +1,29 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class RestAggResponse {
+ private RestAggResult aggregations;
+
+ public RestAggResult getAggregations() {
+ return aggregations;
+ }
+
+ public void setAggregations(RestAggResult aggregations) {
+ this.aggregations = aggregations;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestAggResult.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestAggResult.java
new file mode 100644
index 000000000..b5e7f16b0
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestAggResult.java
@@ -0,0 +1,32 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class RestAggResult {
+ @JsonProperty("agg_result")
+ private RestAggValue aggResult;
+
+ public RestAggValue getAggResult() {
+ return aggResult;
+ }
+
+ public void setAggResult(RestAggValue aggResult) {
+ this.aggResult = aggResult;
+ }
+}
+
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestAggValue.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestAggValue.java
new file mode 100644
index 000000000..360ae4bce
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestAggValue.java
@@ -0,0 +1,30 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class RestAggValue {
+ private double value;
+
+ public double getValue() {
+ return value;
+ }
+
+ public void setValue(double value) {
+ this.value = value;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestBulkResponse.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestBulkResponse.java
new file mode 100644
index 000000000..395856f9e
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestBulkResponse.java
@@ -0,0 +1,79 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+
+import java.util.List;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class RestBulkResponse {
+
+ private boolean errors;
+
+ private List<Map<String, RestBulkItemResponse>> items;
+
+ public boolean isErrors() {
+ return errors;
+ }
+
+ public void setErrors(boolean errors) {
+ this.errors = errors;
+ }
+
+ public List<Map<String, RestBulkItemResponse>> getItems() {
+ return items;
+ }
+
+ public void setItems(List<Map<String, RestBulkItemResponse>> items) {
+ this.items = items;
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown=true)
+ public static class RestBulkItemResponse {
+
+ private String result;
+
+ private int status;
+
+ private Object error;
+
+ public String getResult() {
+ return result;
+ }
+
+ public void setResult(String result) {
+ this.result = result;
+ }
+
+ public int getStatus() {
+ return status;
+ }
+
+ public void setStatus(int status) {
+ this.status = status;
+ }
+
+ public Object getError() {
+ return error;
+ }
+
+ public void setError(Object error) {
+ this.error = error;
+ }
+ }
+
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestClientSetup.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestClientSetup.java
new file mode 100644
index 000000000..f2246c2a8
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestClientSetup.java
@@ -0,0 +1,287 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.client5.http.config.RequestConfig;
+import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.util.Timeout;
+import org.opensearch.client.RestClient;
+import org.opensearch.client.RestClientBuilder;
+import org.opensearch.client.RestClientBuilder.HttpClientConfigCallback;
+import org.opensearch.client.RestClientBuilder.RequestConfigCallback;
+import org.janusgraph.diskstorage.configuration.ConfigOption;
+import org.janusgraph.diskstorage.configuration.Configuration;
+import org.janusgraph.diskstorage.opensearch.OpenSearchClient;
+import org.janusgraph.diskstorage.opensearch.OpenSearchIndex;
+import org.janusgraph.diskstorage.opensearch.rest.util.BasicAuthHttpClientConfigCallback;
+import org.janusgraph.diskstorage.opensearch.rest.util.ConnectionKeepAliveConfigCallback;
+import org.janusgraph.diskstorage.opensearch.rest.util.HttpAuthTypes;
+import org.janusgraph.diskstorage.opensearch.rest.util.RestClientAuthenticator;
+import org.janusgraph.diskstorage.opensearch.rest.util.SSLConfigurationCallback;
+import org.janusgraph.diskstorage.opensearch.rest.util.SSLConfigurationCallback.Builder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_HOSTS;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_PORT;
+
+/**
+ * Create an instance of OpenSearch REST {@link org.opensearch.client.RestClient} from a JanusGraph
+ * {@link org.janusgraph.diskstorage.configuration.Configuration}.
+ */
+public class RestClientSetup {
+
+ private static final Logger log = LoggerFactory.getLogger(RestClientSetup.class);
+
+ public OpenSearchClient connect(Configuration config) throws IOException {
+ log.debug("Configuring RestClient");
+
+ final List<HttpHost> hosts = new ArrayList<>();
+ final int defaultPort = config.has(INDEX_PORT) ? config.get(INDEX_PORT) : OpenSearchIndex.HOST_PORT_DEFAULT;
+ final String httpScheme = config.get(OpenSearchIndex.SSL_ENABLED) ? "https" : "http";
+ for (String host : config.get(INDEX_HOSTS)) {
+ String[] hostStringParts = host.split(":");
+ String hostname = hostStringParts[0];
+ int hostPort = defaultPort;
+ if (hostStringParts.length == 2) hostPort = Integer.parseInt(hostStringParts[1]);
+ log.debug("Configured remote host: {} : {}", hostname, hostPort);
+ hosts.add(new HttpHost(httpScheme, hostname, hostPort));
+ }
+
+ final RestClient rc = getRestClient(hosts.toArray(new HttpHost[hosts.size()]), config);
+
+ final int scrollKeepAlive = config.get(OpenSearchIndex.OS_SCROLL_KEEP_ALIVE);
+ Preconditions.checkArgument(scrollKeepAlive >= 1, "Scroll keep-alive should be greater than or equal to 1");
+ final boolean useMappingTypesForOS2 = config.get(OpenSearchIndex.USE_MAPPING_TYPES);
+ int retryLimit = config.getOrDefault(OpenSearchIndex.RETRY_LIMIT);
+ long retryInitialWaitMs = config.getOrDefault(OpenSearchIndex.RETRY_INITIAL_WAIT);
+ long retryMaxWaitMs = config.getOrDefault(OpenSearchIndex.RETRY_MAX_WAIT);
+ Set<Integer> errorCodesToRetry = Arrays.stream(config.getOrDefault(OpenSearchIndex.RETRY_ERROR_CODES))
+ .mapToInt(Integer::parseInt).boxed().collect(Collectors.toSet());
+ int bulkChunkLimitBytes = config.getOrDefault(OpenSearchIndex.BULK_CHUNK_SIZE_LIMIT_BYTES);
+ final RestOpenSearchClient client = getOpenSearchClient(rc, scrollKeepAlive, useMappingTypesForOS2,
+ retryLimit, errorCodesToRetry, retryInitialWaitMs, retryMaxWaitMs, bulkChunkLimitBytes);
+ if (config.has(OpenSearchIndex.BULK_REFRESH)) {
+ client.setBulkRefresh(config.get(OpenSearchIndex.BULK_REFRESH));
+ }
+
+ Integer retryOnConflict = config.has(OpenSearchIndex.RETRY_ON_CONFLICT) ? config.get(OpenSearchIndex.RETRY_ON_CONFLICT) : null;
+ client.setRetryOnConflict(retryOnConflict);
+
+ return client;
+ }
+
+ protected RestClient getRestClient(HttpHost[] hosts, Configuration config) {
+ final RestClientBuilder restClientBuilder = getRestClientBuilder(hosts);
+
+ final HttpClientConfigCallback httpClientConfigCallback = getHttpClientConfigCallback(config);
+ if (httpClientConfigCallback != null) {
+ restClientBuilder.setHttpClientConfigCallback(httpClientConfigCallback);
+ }
+
+ final RequestConfigCallback requestConfigCallback = getRequestConfigCallback(config);
+ if (requestConfigCallback != null) {
+ restClientBuilder.setRequestConfigCallback(requestConfigCallback);
+ }
+
+ return restClientBuilder.build();
+ }
+
+ protected RestClientBuilder getRestClientBuilder(HttpHost[] hosts) {
+ return RestClient.builder(hosts);
+ }
+
+ protected RestOpenSearchClient getOpenSearchClient(RestClient rc, int scrollKeepAlive, boolean useMappingTypesForOS2,
+ int retryAttemptLimit, Set<Integer> retryOnErrorCodes, long retryInitialWaitMs,
+ long retryMaxWaitMs, int bulkChunkSerializedLimit) {
+ return new RestOpenSearchClient(rc, scrollKeepAlive, useMappingTypesForOS2, retryAttemptLimit, retryOnErrorCodes,
+ retryInitialWaitMs, retryMaxWaitMs, bulkChunkSerializedLimit);
+ }
+
+ /**
+ * <p>
+ * Returns the callback for customizing the {@link RequestConfig} or null if no
+ * customization is needed.
+ * </p>
+ * <p>
+ * See {@link RestClientBuilder#setRequestConfigCallback(RequestConfigCallback)} for more details.
+ * </p>
+ *
+ * @param config
+ * ES index configuration
+ * @return callback or null if the request customization is not needed
+ */
+ protected RequestConfigCallback getRequestConfigCallback(Configuration config) {
+
+ final List<RequestConfigCallback> callbackList = new LinkedList<>();
+
+ final Integer connectTimeout = config.get(OpenSearchIndex.CONNECT_TIMEOUT);
+ final Integer socketTimeout = config.get(OpenSearchIndex.SOCKET_TIMEOUT);
+
+ callbackList.add((requestConfigBuilder) ->
+ requestConfigBuilder
+ .setConnectTimeout(Timeout.ofMilliseconds(connectTimeout))
+ .setResponseTimeout(Timeout.ofMilliseconds(socketTimeout)));
+
+ // will execute the chain of individual callbacks
+ return requestConfigBuilder -> {
+ for(RequestConfigCallback cb: callbackList) {
+ cb.customizeRequestConfig(requestConfigBuilder);
+ }
+ return requestConfigBuilder;
+ };
+ }
+
+ /**
+ * <p>
+ * Returns the callback for customizing {@link CloseableHttpAsyncClient} or null if no
+ * customization is needed.
+ * </p>
+ * <p>
+ * See {@link RestClientBuilder#setHttpClientConfigCallback(HttpClientConfigCallback)} for more details.
+ * </p>
+ *
+ * @param config
+ * ES index configuration
+ * @return callback or null if the client customization is not needed
+ */
+ protected HttpClientConfigCallback getHttpClientConfigCallback(Configuration config) {
+
+ final List<HttpClientConfigCallback> callbackList = new LinkedList<>();
+
+ final HttpAuthTypes authType = ConfigOption.getEnumValue(config.get(OpenSearchIndex.OS_HTTP_AUTH_TYPE),
+ HttpAuthTypes.class);
+ log.debug("Configuring HTTP(S) authentication type {}", authType);
+
+ switch (authType) {
+ case BASIC:
+ callbackList.add(new BasicAuthHttpClientConfigCallback(
+ config.has(OpenSearchIndex.OS_HTTP_AUTH_REALM) ? config.get(OpenSearchIndex.OS_HTTP_AUTH_REALM) : "",
+ config.get(OpenSearchIndex.OS_HTTP_AUTH_USERNAME),
+ config.get(OpenSearchIndex.OS_HTTP_AUTH_PASSWORD)));
+ break;
+ case CUSTOM:
+ callbackList.add(getCustomAuthenticator(
+ config.get(OpenSearchIndex.OS_HTTP_AUTHENTICATOR_CLASS),
+ config.get(OpenSearchIndex.OS_HTTP_AUTHENTICATOR_ARGS)));
+ break;
+ case NONE:
+ break;
+ default:
+ // not expected
+ throw new IllegalArgumentException("Authentication type \"" + authType + "\" is not implemented");
+ }
+
+ if (config.has(OpenSearchIndex.CLIENT_KEEP_ALIVE )) {
+ callbackList.add(new ConnectionKeepAliveConfigCallback(config.get(OpenSearchIndex.CLIENT_KEEP_ALIVE)));
+ }
+
+ if (config.get(OpenSearchIndex.SSL_ENABLED)) {
+ // Custom SSL configuration
+ final Builder sslConfCBBuilder = getSSLConfigurationCallbackBuilder();
+ boolean configureSSL = false;
+ if (config.has(OpenSearchIndex.SSL_TRUSTSTORE_LOCATION)) {
+ sslConfCBBuilder.withTrustStore(config.get(OpenSearchIndex.SSL_TRUSTSTORE_LOCATION),
+ config.get(OpenSearchIndex.SSL_TRUSTSTORE_PASSWORD));
+ configureSSL = true;
+ }
+ if (config.has(OpenSearchIndex.SSL_KEYSTORE_LOCATION)) {
+ final String keystorePassword = config.get(OpenSearchIndex.SSL_KEYSTORE_PASSWORD);
+ sslConfCBBuilder.withKeyStore(config.get(OpenSearchIndex.SSL_KEYSTORE_LOCATION),
+ keystorePassword,
+ config.has(OpenSearchIndex.SSL_KEY_PASSWORD) ? config.get(OpenSearchIndex.SSL_KEY_PASSWORD) : keystorePassword);
+ configureSSL = true;
+ }
+
+ if (config.has(OpenSearchIndex.SSL_DISABLE_HOSTNAME_VERIFICATION) &&
+ config.get(OpenSearchIndex.SSL_DISABLE_HOSTNAME_VERIFICATION)) {
+ log.warn("SSL hostname verification is disabled, OpenSearch HTTPS connections may not be secure");
+ sslConfCBBuilder.disableHostNameVerification();
+ configureSSL = true;
+ }
+
+ if (config.has(OpenSearchIndex.SSL_ALLOW_SELF_SIGNED_CERTIFICATES) &&
+ config.get(OpenSearchIndex.SSL_ALLOW_SELF_SIGNED_CERTIFICATES)) {
+ log.warn("Self-signed SSL certificate support is enabled, OpenSearch HTTPS connections may not be secure");
+ sslConfCBBuilder.allowSelfSignedCertificates();
+ configureSSL = true;
+ }
+
+ if (configureSSL) {
+ callbackList.add(sslConfCBBuilder.build());
+ }
+ }
+
+ if (callbackList.isEmpty()) {
+ return null;
+ }
+
+ // will execute the chain of individual callbacks
+ return httpClientBuilder -> {
+ for(HttpClientConfigCallback cb: callbackList) {
+ cb.customizeHttpClient(httpClientBuilder);
+ }
+ return httpClientBuilder;
+ };
+ }
+
+ protected SSLConfigurationCallback.Builder getSSLConfigurationCallbackBuilder() {
+ return SSLConfigurationCallback.Builder.create();
+ }
+
+ protected RestClientAuthenticator getCustomAuthenticator(String authClassName, String[] authClassConstructorArgList) {
+ Preconditions.checkArgument(StringUtils.isNotEmpty(authClassName),
+ "Custom authenticator class name cannot be empty");
+ Preconditions.checkNotNull(authClassConstructorArgList,
+ "Custom authenticator class constructor argument list cannot be null");
+
+ final RestClientAuthenticator authenticator;
+
+ try {
+ final Class<?> c = Class.forName(authClassName);
+ Preconditions.checkArgument(RestClientAuthenticator.class.isAssignableFrom(c),
+ "Authenticator class %s must be a subclass of %s", authClassName, RestClientAuthenticator.class.getName());
+ @SuppressWarnings("unchecked")
+ final Constructor<RestClientAuthenticator> ctr = ((Class<RestClientAuthenticator>)c).getConstructor(String[].class);
+ authenticator = ctr.newInstance((Object)authClassConstructorArgList);
+ } catch (Exception e) {
+ log.error("Unable to instantiate the custom authenticator {} with constructor arguments \"{}\"",
+ authClassName, authClassConstructorArgList, e);
+ throw new RuntimeException("Unable to instantiate the custom authenticator", e);
+ }
+
+ try {
+ authenticator.init();
+ } catch (IOException e) {
+ log.error("Unable to initialize the custom authenticator {} with constructor arguments \"{}\"",
+ authClassName, authClassConstructorArgList, e);
+ throw new RuntimeException("Unable to initialize the custom authenticator", e);
+ }
+
+ return authenticator;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestCountResponse.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestCountResponse.java
new file mode 100644
index 000000000..9acaebaa4
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestCountResponse.java
@@ -0,0 +1,31 @@
+// Copyright 2019 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class RestCountResponse {
+
+ private long count;
+
+ public long getCount() {
+ return count;
+ }
+
+ public void setCount(long count) {
+ this.count = count;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestIndexSettings.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestIndexSettings.java
new file mode 100644
index 000000000..dde0d1ebb
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestIndexSettings.java
@@ -0,0 +1,47 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonProperty;
+
+import java.util.Map;
+
+public class RestIndexSettings {
+
+ private Settings settings;
+
+ public Settings getSettings() {
+ return settings;
+ }
+
+ public void setSettings(Settings settings) {
+ this.settings = settings;
+ }
+
+ public static class Settings {
+
+ @JsonProperty("index")
+ private Map<String,Object> map;
+
+ public Map<String, Object> getMap() {
+ return map;
+ }
+
+ public void setMap(Map<String, Object> map) {
+ this.map = map;
+ }
+ }
+
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestOpenSearchClient.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestOpenSearchClient.java
new file mode 100644
index 000000000..7df168fc5
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestOpenSearchClient.java
@@ -0,0 +1,755 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpStatus;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+import org.apache.tinkerpop.shaded.jackson.core.JsonParseException;
+import org.apache.tinkerpop.shaded.jackson.core.JsonProcessingException;
+import org.apache.tinkerpop.shaded.jackson.core.type.TypeReference;
+import org.apache.tinkerpop.shaded.jackson.databind.JsonMappingException;
+import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper;
+import org.apache.tinkerpop.shaded.jackson.databind.ObjectReader;
+import org.apache.tinkerpop.shaded.jackson.databind.ObjectWriter;
+import org.apache.tinkerpop.shaded.jackson.databind.SerializationFeature;
+import org.apache.tinkerpop.shaded.jackson.databind.module.SimpleModule;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.client.ResponseException;
+import org.opensearch.client.RestClient;
+import org.janusgraph.core.attribute.Geoshape;
+import org.janusgraph.diskstorage.opensearch.OpenSearchMajorVersion;
+import org.janusgraph.diskstorage.opensearch.OpenSearchClient;
+import org.janusgraph.diskstorage.opensearch.OpenSearchMutation;
+import org.janusgraph.diskstorage.opensearch.mapping.IndexMapping;
+import org.janusgraph.diskstorage.opensearch.mapping.TypedIndexMappings;
+import org.janusgraph.diskstorage.opensearch.mapping.TypelessIndexMappings;
+import org.janusgraph.diskstorage.opensearch.script.OSScriptResponse;
+import org.javatuples.Pair;
+import org.javatuples.Triplet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.janusgraph.util.encoding.StringEncoding.UTF8_CHARSET;
+
+public class RestOpenSearchClient implements OpenSearchClient {
+
+ private static final Logger log = LoggerFactory.getLogger(RestOpenSearchClient.class);
+
+ private static final String REQUEST_TYPE_DELETE = "DELETE";
+ private static final String REQUEST_TYPE_GET = "GET";
+ private static final String REQUEST_TYPE_POST = "POST";
+ private static final String REQUEST_TYPE_PUT = "PUT";
+ private static final String REQUEST_TYPE_HEAD = "HEAD";
+ private static final String REQUEST_SEPARATOR = "/";
+ private static final String REQUEST_PARAM_BEGINNING = "?";
+ private static final String REQUEST_PARAM_SEPARATOR = "&";
+
+ public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name";
+
+ private static final byte[] NEW_LINE_BYTES = "\n".getBytes(UTF8_CHARSET);
+
+ private static final Request INFO_REQUEST = new Request(REQUEST_TYPE_GET, REQUEST_SEPARATOR);
+
+ private static final ObjectMapper mapper;
+ private static final ObjectReader mapReader;
+ private static final ObjectWriter mapWriter;
+
+ static {
+ final SimpleModule module = new SimpleModule();
+ module.addSerializer(new Geoshape.GeoshapeGsonSerializerV2d0());
+ mapper = new ObjectMapper();
+ mapper.registerModule(module);
+ mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+ mapReader = mapper.readerWithView(Map.class).forType(HashMap.class);
+ mapWriter = mapper.writerWithView(Map.class);
+ }
+
+ private static final OpenSearchMajorVersion DEFAULT_VERSION = OpenSearchMajorVersion.TWO;
+
+ private static final Function<StringBuilder, StringBuilder> APPEND_OP = sb -> sb.append(sb.length() == 0 ? REQUEST_PARAM_BEGINNING : REQUEST_PARAM_SEPARATOR);
+
+ private final RestClient delegate;
+
+ private OpenSearchMajorVersion majorVersion;
+
+ private String bulkRefresh;
+
+ private boolean bulkRefreshEnabled = false;
+
+ private final String scrollKeepAlive;
+
+ private final boolean useMappingTypes;
+
+ private final boolean typelessWireFormat;
+
+ private Integer retryOnConflict;
+
+ private final String retryOnConflictKey;
+
+ private final int retryAttemptLimit;
+
+ private final Set<Integer> retryOnErrorCodes;
+
+ private final long retryInitialWaitMs;
+
+ private final long retryMaxWaitMs;
+
+ private final int bulkChunkSerializedLimitBytes;
+
+public RestOpenSearchClient(RestClient delegate, int scrollKeepAlive, boolean useMappingTypesForOS2,
+ int retryAttemptLimit, Set<Integer> retryOnErrorCodes, long retryInitialWaitMs,
+ long retryMaxWaitMs, int bulkChunkSerializedLimitBytes) {
+ this.delegate = delegate;
+ majorVersion = getMajorVersion();
+ this.scrollKeepAlive = scrollKeepAlive+"s";
+ // OpenSearch 2.x uses typeless indices with ES7-equivalent wire semantics despite major version 2
+ typelessWireFormat = majorVersion.supportsTypelessWireFormat();
+ useMappingTypes = false;
+ retryOnConflictKey = "retry_on_conflict";
+ this.retryAttemptLimit = retryAttemptLimit;
+ this.retryOnErrorCodes = Collections.unmodifiableSet(retryOnErrorCodes);
+ this.retryInitialWaitMs = retryInitialWaitMs;
+ this.retryMaxWaitMs = retryMaxWaitMs;
+ this.bulkChunkSerializedLimitBytes = bulkChunkSerializedLimitBytes;
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+ @Override
+ public OpenSearchMajorVersion getMajorVersion() {
+ if (majorVersion != null) {
+ return majorVersion;
+ }
+
+ majorVersion = DEFAULT_VERSION;
+ try {
+ final Response response = delegate.performRequest(INFO_REQUEST);
+ try (final InputStream inputStream = response.getEntity().getContent()) {
+ final ClusterInfo info = mapper.readValue(inputStream, ClusterInfo.class);
+ majorVersion = OpenSearchMajorVersion.parse(info.getVersion() != null ? (String) info.getVersion().get("number") : null);
+ }
+ } catch (final IOException e) {
+ log.warn("Unable to determine OpenSearch server version. Default to {}.", majorVersion, e);
+ }
+
+ return majorVersion;
+ }
+
+ @Override
+ public void clusterHealthRequest(String timeout) throws IOException {
+ Request clusterHealthRequest = new Request(REQUEST_TYPE_GET,
+ REQUEST_SEPARATOR + "_cluster" + REQUEST_SEPARATOR + "health");
+ clusterHealthRequest.addParameter("wait_for_status", "yellow");
+ clusterHealthRequest.addParameter("timeout", timeout);
+
+ final Response response = delegate.performRequest(clusterHealthRequest);
+ try (final InputStream inputStream = response.getEntity().getContent()) {
+ final Map<String,Object> values = mapReader.readValue(inputStream);
+ if (!values.containsKey("timed_out")) {
+ throw new IOException("Unexpected response for OpenSearch cluster health request");
+ } else if (!Objects.equals(values.get("timed_out"), false)) {
+ throw new IOException("OpenSearch timeout waiting for yellow status");
+ }
+ }
+ }
+
+ @Override
+ public boolean indexExists(String indexName) throws IOException {
+ final Response response = delegate.performRequest(new Request(REQUEST_TYPE_HEAD, REQUEST_SEPARATOR + indexName));
+ return response.getStatusLine().getStatusCode() == 200;
+ }
+
+ @Override
+ public boolean isIndex(String indexName) {
+ try {
+ final Response response = delegate.performRequest(new Request(REQUEST_TYPE_GET, REQUEST_SEPARATOR + indexName));
+ try (final InputStream inputStream = response.getEntity().getContent()) {
+ return mapper.readValue(inputStream, Map.class).containsKey(indexName);
+ }
+ } catch (final IOException ignored) {
+ }
+ return false;
+ }
+
+ @Override
+ public boolean isAlias(String aliasName) {
+ try {
+ delegate.performRequest(new Request(REQUEST_TYPE_GET, REQUEST_SEPARATOR + "_alias" + REQUEST_SEPARATOR + aliasName));
+ return true;
+ } catch (final IOException ignored) {
+ }
+ return false;
+ }
+
+ @Override
+ public void createStoredScript(String scriptName, Map<String, Object> script) throws IOException {
+
+ Request request = new Request(REQUEST_TYPE_POST, REQUEST_SEPARATOR + "_scripts" + REQUEST_SEPARATOR + scriptName);
+
+ performRequest(request, mapWriter.writeValueAsBytes(script));
+ }
+
+ @Override
+ public OSScriptResponse getStoredScript(String scriptName) throws IOException {
+
+ Request request = new Request(REQUEST_TYPE_GET,
+ REQUEST_SEPARATOR + "_scripts" + REQUEST_SEPARATOR + scriptName);
+
+ try{
+
+ final Response response = delegate.performRequest(request);
+
+ if(response.getStatusLine().getStatusCode() != 200){
+ throw new IOException("Error executing request: " + response.getStatusLine().getReasonPhrase());
+ }
+
+ try (final InputStream inputStream = response.getEntity().getContent()) {
+
+ return mapper.readValue(inputStream, new TypeReference<OSScriptResponse>() {});
+
+ } catch (final JsonParseException | JsonMappingException | ResponseException e) {
+ throw new IOException("Error when we try to parse ES script: "+response.getEntity().getContent());
+ }
+
+ } catch (ResponseException e){
+
+ final Response response = e.getResponse();
+
+ if(e.getResponse().getStatusLine().getStatusCode() == HttpStatus.SC_NOT_FOUND){
+ OSScriptResponse esScriptResponse = new OSScriptResponse();
+ esScriptResponse.setFound(false);
+ return esScriptResponse;
+ }
+
+ throw new IOException("Error executing request: " + response.getStatusLine().getReasonPhrase());
+ }
+ }
+
+ @Override
+ public void createIndex(String indexName, Map<String,Object> settings) throws IOException {
+
+ Request request = new Request(REQUEST_TYPE_PUT, REQUEST_SEPARATOR + indexName);
+
+ if (typelessWireFormat && settings != null && settings.size() > 0) {
+ Map<String,Object> updatedSettings = new HashMap<>();
+ updatedSettings.put("settings", settings);
+ settings = updatedSettings;
+ }
+
+ performRequest(request, mapWriter.writeValueAsBytes(settings));
+ }
+
+ @Override
+ public void updateIndexSettings(String indexName, Map<String,Object> settings) throws IOException {
+
+ performRequest(REQUEST_TYPE_PUT, REQUEST_SEPARATOR + indexName + REQUEST_SEPARATOR +"_settings",
+ mapWriter.writeValueAsBytes(settings));
+ }
+
+ @Override
+ public void updateClusterSettings(Map<String, Object> settings) throws IOException {
+
+ performRequest(REQUEST_TYPE_PUT, REQUEST_SEPARATOR + "_cluster" + REQUEST_SEPARATOR + "settings",
+ mapWriter.writeValueAsBytes(settings));
+ }
+
+ @Override
+ public void addAlias(String alias, String index) throws IOException {
+ final Map actionAlias = ImmutableMap.of("actions", ImmutableList.of(ImmutableMap.of("add", ImmutableMap.of("index", index, "alias", alias))));
+ performRequest(REQUEST_TYPE_POST, REQUEST_SEPARATOR + "_aliases", mapWriter.writeValueAsBytes(actionAlias));
+ }
+
+ @Override
+ public Map getIndexSettings(String indexName) throws IOException {
+ final Response response = performRequest(REQUEST_TYPE_GET, REQUEST_SEPARATOR + indexName + REQUEST_SEPARATOR + "_settings", null);
+ try (final InputStream inputStream = response.getEntity().getContent()) {
+ final Map<String,RestIndexSettings> settings = mapper.readValue(inputStream, new TypeReference<Map<String, RestIndexSettings>>() {});
+ return settings == null ? null : settings.get(indexName).getSettings().getMap();
+ }
+ }
+
+ @Override
+ public void createMapping(String indexName, String typeName, Map<String,Object> mapping) throws IOException {
+
+ Request request;
+
+ if(useMappingTypes){
+ request = new Request(REQUEST_TYPE_PUT,
+ REQUEST_SEPARATOR + indexName + REQUEST_SEPARATOR + "_mapping" + REQUEST_SEPARATOR + typeName);
+ } else {
+ request = new Request(REQUEST_TYPE_PUT,
+ REQUEST_SEPARATOR + indexName + REQUEST_SEPARATOR + "_mapping");
+ }
+
+ performRequest(request, mapWriter.writeValueAsBytes(mapping));
+ }
+
+ @Override
+ public IndexMapping getMapping(String indexName, String typeName) throws IOException {
+
+ Request request;
+
+ if(useMappingTypes){
+ request = new Request(REQUEST_TYPE_GET,
+ REQUEST_SEPARATOR + indexName + REQUEST_SEPARATOR + "_mapping" + REQUEST_SEPARATOR + typeName);
+ } else {
+ request = new Request(REQUEST_TYPE_GET,
+ REQUEST_SEPARATOR + indexName + REQUEST_SEPARATOR + "_mapping");
+ }
+
+ try (final InputStream inputStream = performRequest(request, null).getEntity().getContent()) {
+
+ if(useMappingTypes){
+ final Map<String, TypedIndexMappings> settings = mapper.readValue(inputStream,
+ new TypeReference<Map<String, TypedIndexMappings>>() {});
+ return settings != null ? settings.get(indexName).getMappings().get(typeName) : null;
+ }
+
+ final Map<String, TypelessIndexMappings> settings = mapper.readValue(inputStream,
+ new TypeReference<Map<String, TypelessIndexMappings>>() {});
+ return settings != null ? settings.get(indexName).getMappings() : null;
+
+ } catch (final JsonParseException | JsonMappingException | ResponseException e) {
+ log.info("Error when we try to get ES mapping", e);
+ return null;
+ }
+ }
+
+ @Override
+ public void deleteIndex(String indexName) throws IOException {
+ if (isAlias(indexName)) {
+ // aliased multi-index case
+ final String path = REQUEST_SEPARATOR + "_alias" + REQUEST_SEPARATOR + indexName;
+ final Response response = performRequest(REQUEST_TYPE_GET, path, null);
+ try (final InputStream inputStream = response.getEntity().getContent()) {
+ final Map<String,Object> records = mapper.readValue(inputStream, new TypeReference<Map<String, Object>>() {});
+ if (records == null) return;
+ for (final String index : records.keySet()) {
+ if (indexExists(index)) {
+ performRequest(REQUEST_TYPE_DELETE, REQUEST_SEPARATOR + index, null);
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void clearStore(String indexName, String storeName) throws IOException {
+ String name = indexName + "_" + storeName;
+ if (indexExists(name)) {
+ performRequest(REQUEST_TYPE_DELETE, REQUEST_SEPARATOR + indexName + "_" + storeName, null);
+ }
+ }
+
+ @VisibleForTesting
+ class RequestBytes {
+ final byte [] requestBytes;
+ final byte [] requestSource;
+
+ @VisibleForTesting
+ RequestBytes(final OpenSearchMutation request) throws JsonProcessingException {
+ Map<String, Object> requestData = new HashMap<>();
+ if (useMappingTypes) {
+ requestData.put("_index", request.getIndex());
+ requestData.put("_type", request.getType());
+ requestData.put("_id", request.getId());
+ } else {
+ requestData.put("_index", request.getIndex());
+ requestData.put("_id", request.getId());
+ }
+
+ if (retryOnConflict != null && request.getRequestType() == OpenSearchMutation.RequestType.UPDATE) {
+ requestData.put(retryOnConflictKey, retryOnConflict);
+ }
+
+ this.requestBytes = mapWriter.writeValueAsBytes(ImmutableMap.of(request.getRequestType().name().toLowerCase(), requestData));
+ if (request.getSource() != null) {
+ this.requestSource = mapWriter.writeValueAsBytes(request.getSource());
+ } else {
+ this.requestSource = null;
+ }
+ }
+
+ @VisibleForTesting
+ int getSerializedSize() {
+ int serializedSize = this.requestBytes.length;
+ serializedSize+= 1; //For follow-up NEW_LINE_BYTES
+ if (this.requestSource != null) {
+ serializedSize += this.requestSource.length;
+ serializedSize+= 1; //For follow-up NEW_LINE_BYTES
+ }
+ return serializedSize;
+ }
+
+ private void writeTo(OutputStream outputStream) throws IOException {
+ outputStream.write(this.requestBytes);
+ outputStream.write(NEW_LINE_BYTES);
+ if (this.requestSource != null) {
+ outputStream.write(requestSource);
+ outputStream.write(NEW_LINE_BYTES);
+ }
+ }
+ }
+
+ private Pair<String, byte[]> buildBulkRequestInput(List<RequestBytes> requests, String ingestPipeline) throws IOException {
+ final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ for (final RequestBytes request : requests) {
+ request.writeTo(outputStream);
+ }
+
+ final StringBuilder bulkRequestQueryParameters = new StringBuilder();
+ if (ingestPipeline != null) {
+ APPEND_OP.apply(bulkRequestQueryParameters).append("pipeline=").append(ingestPipeline);
+ }
+ if (bulkRefreshEnabled) {
+ APPEND_OP.apply(bulkRequestQueryParameters).append("refresh=").append(bulkRefresh);
+ }
+ final String bulkRequestPath = REQUEST_SEPARATOR + "_bulk" + bulkRequestQueryParameters;
+ return Pair.with(bulkRequestPath, outputStream.toByteArray());
+ }
+
+ private List<Triplet<Object, Integer, RequestBytes>> pairErrorsWithSubmittedMutation(
+ //Bulk API is documented to return bulk item responses in the same order of submission
+ //https://www.elastic.co/guide/en/opensearch/reference/current/docs-bulk.html#bulk-api-response-body
+ //As such we only need to retry elements that failed
+ final List<Map<String, RestBulkResponse.RestBulkItemResponse>> bulkResponseItems,
+ final List<RequestBytes> submittedBulkRequestItems) {
+ final List<Triplet<Object, Integer, RequestBytes>> errors = new ArrayList<>(bulkResponseItems.size());
+ for (int itemIndex = 0; itemIndex < bulkResponseItems.size(); itemIndex++) {
+ Collection<RestBulkResponse.RestBulkItemResponse> bulkResponseItem = bulkResponseItems.get(itemIndex).values();
+ if (bulkResponseItem.size() > 1) {
+ throw new IllegalStateException("There should only be a single item per bulk reponse item entry");
+ }
+ RestBulkResponse.RestBulkItemResponse item = bulkResponseItem.iterator().next();
+ if (item.getError() != null && item.getStatus() != HttpStatus.SC_NOT_FOUND) {
+ errors.add(Triplet.with(item.getError(), item.getStatus(), submittedBulkRequestItems.get(itemIndex)));
+ }
+ }
+ return errors;
+ }
+
+ @VisibleForTesting
+ class BulkRequestChunker implements Iterator<List<RequestBytes>> {
+ //By default, OpenSearch writes are limited to 100mb, so chunk a given batch of requests so they stay under
+ //the specified limit
+
+ //https://www.elastic.co/guide/en/opensearch/reference/current/docs-bulk.html#docs-bulk-api-desc
+ //There is no "correct" number of actions to perform in a single bulk request. Experiment with different
+ // settings to find the optimal size for your particular workload. Note that OpenSearch limits the maximum
+ // size of a HTTP request to 100mb by default
+ private final PeekingIterator<RequestBytes> requestIterator;
+ private final int[] exceptionallyLargeRequests;
+
+ @VisibleForTesting
+ BulkRequestChunker(List<OpenSearchMutation> requests) throws JsonProcessingException {
+ List<RequestBytes> serializedRequests = new ArrayList<>(requests.size());
+ List<Integer> requestSizesThatWereTooLarge = new ArrayList<>();
+ for (OpenSearchMutation request : requests) {
+ RequestBytes requestBytes = new RequestBytes(request);
+ int requestSerializedSize = requestBytes.getSerializedSize();
+ if (requestSerializedSize <= bulkChunkSerializedLimitBytes) {
+ //Only keep items that we can actually send in memory
+ serializedRequests.add(requestBytes);
+ } else {
+ requestSizesThatWereTooLarge.add(requestSerializedSize);
+ }
+ }
+ this.requestIterator = Iterators.peekingIterator(serializedRequests.iterator());
+ //Condense request sizes that are too large into an int array to remove Boxed & List memory overhead
+ this.exceptionallyLargeRequests = requestSizesThatWereTooLarge.isEmpty() ? null :
+ requestSizesThatWereTooLarge.stream().mapToInt(Integer::intValue).toArray();
+ }
+
+ @Override
+ public boolean hasNext() {
+ //Make sure hasNext() still returns true if exceptionally large requests were attempted to be submitted
+ //This allows next() to throw after all well sized requests have been chunked for submission
+ return requestIterator.hasNext() || exceptionallyLargeRequests != null;
+ }
+
+ @Override
+ public List<RequestBytes> next() {
+ List<RequestBytes> serializedRequests = new ArrayList<>();
+ int chunkSerializedTotal = 0;
+ while (requestIterator.hasNext()) {
+ RequestBytes peeked = requestIterator.peek();
+ chunkSerializedTotal += peeked.getSerializedSize();
+ if (chunkSerializedTotal <= bulkChunkSerializedLimitBytes) {
+ serializedRequests.add(requestIterator.next());
+ } else {
+ //Adding this element would exceed the limit, so return the chunk
+ return serializedRequests;
+ }
+ }
+ //Check if we should throw an exception for items that were exceptionally large and therefore undeliverable.
+ //This is only done after all items that could be sent have been sent
+ if (serializedRequests.isEmpty() && this.exceptionallyLargeRequests != null) {
+ throw new IllegalArgumentException(String.format(
+ "Bulk request item(s) larger than permitted chunk limit. Limit is %s. Serialized item size(s) %s",
+ bulkChunkSerializedLimitBytes, Arrays.toString(this.exceptionallyLargeRequests)));
+ }
+ //All remaining requests fit in this chunk
+ return serializedRequests;
+ }
+ }
+
+ @Override
+ public void bulkRequest(final List<OpenSearchMutation> requests, String ingestPipeline) throws IOException {
+ BulkRequestChunker bulkRequestChunker = new BulkRequestChunker(requests);
+ while (bulkRequestChunker.hasNext()) {
+ List<RequestBytes> bulkRequestChunk = bulkRequestChunker.next();
+ int retryCount = 0;
+ while (true) {
+ final Pair<String, byte[]> bulkRequestInput = buildBulkRequestInput(bulkRequestChunk, ingestPipeline);
+ final Response response = performRequest(REQUEST_TYPE_POST, bulkRequestInput.getValue0(), bulkRequestInput.getValue1());
+ try (final InputStream inputStream = response.getEntity().getContent()) {
+ final RestBulkResponse bulkResponse = mapper.readValue(inputStream, RestBulkResponse.class);
+ List<Triplet<Object, Integer, RequestBytes>> bulkItemsThatFailed = pairErrorsWithSubmittedMutation(bulkResponse.getItems(), bulkRequestChunk);
+ if (!bulkItemsThatFailed.isEmpty()) {
+ //Only retry the bulk request if *all* the bulk response item error codes are retry error codes
+ final Set<Integer> errorCodes = bulkItemsThatFailed.stream().map(Triplet::getValue1).collect(Collectors.toSet());
+ if (retryCount < retryAttemptLimit && retryOnErrorCodes.containsAll(errorCodes)) {
+ //Build up the next request batch, of only the failed mutations
+ bulkRequestChunk = bulkItemsThatFailed.stream().map(Triplet::getValue2).collect(Collectors.toList());
+ performRetryWait(retryCount);
+ retryCount++;
+ } else {
+ final List<Object> errorItems = bulkItemsThatFailed.stream().map(Triplet::getValue0).collect(Collectors.toList());
+ errorItems.forEach(error -> log.error("Failed to execute ES query: {}", error));
+ throw new IOException("Failure(s) in OpenSearch bulk request: " + errorItems);
+ }
+ } else {
+ //The entire bulk request was successful, leave the loop
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ public void setRetryOnConflict(Integer retryOnConflict) {
+ this.retryOnConflict = retryOnConflict;
+ }
+
+ @Override
+ public long countTotal(String indexName, Map<String, Object> requestData) throws IOException {
+
+ final Request request = new Request(REQUEST_TYPE_GET, REQUEST_SEPARATOR + indexName + REQUEST_SEPARATOR + "_count");
+
+ final byte[] requestDataBytes = mapper.writeValueAsBytes(requestData);
+ if (log.isDebugEnabled()) {
+ log.debug("OpenSearch request: " + mapper.writerWithDefaultPrettyPrinter().writeValueAsString(requestData));
+ }
+
+ final Response response = performRequest(request, requestDataBytes);
+ try (final InputStream inputStream = response.getEntity().getContent()) {
+ return mapper.readValue(inputStream, RestCountResponse.class).getCount();
+ }
+ }
+
+ /**
+ * Execute the aggregation request using OpenSearch index.
+ * OpenSearch uses double values to hold and represent numeric data. As a result, aggregations on long numbers
+ * greater than 2^53 are approximate.
+ * <a href="https://www.elastic.co/guide/en/opensearch/reference/7.17/search-aggregations.html#limits-for-long-values">OpenSearch, limits for long values</a>
+ *
+ * @param indexName the name of the OpenSearch index on which the aggregation is executed
+ * @param requestData the filter query
+ * @param agg the name of the aggregation operation (min, max, avg, sum)
+ * @param fieldName the name of the field on which the aggregation is computed
+ * @return the result of the aggregation
+ * @throws IOException
+ */
+ private double executeAggs(String indexName, Map<String, Object> requestData, String agg, String fieldName) throws IOException {
+
+ final Request request = new Request(REQUEST_TYPE_GET, REQUEST_SEPARATOR + indexName + REQUEST_SEPARATOR + "_search");
+
+ requestData.put("aggs", ImmutableMap.of("agg_result", ImmutableMap.of(agg, ImmutableMap.of("field", fieldName))));
+ final byte[] requestDataBytes = mapper.writeValueAsBytes(requestData);
+ if (log.isDebugEnabled()) {
+ log.debug("OpenSearch request: " + mapper.writerWithDefaultPrettyPrinter().writeValueAsString(requestData));
+ }
+
+ final Response response = performRequest(request, requestDataBytes);
+ try (final InputStream inputStream = response.getEntity().getContent()) {
+ return mapper.readValue(inputStream, RestAggResponse.class).getAggregations().getAggResult().getValue();
+ }
+ }
+
+ private Number adaptNumberType(double value, Class<? extends Number> expectedType) {
+ if (expectedType == null) return value;
+ else if (Byte.class.isAssignableFrom(expectedType)) return (byte)value;
+ else if (Short.class.isAssignableFrom(expectedType)) return (short)value;
+ else if (Integer.class.isAssignableFrom(expectedType)) return (int)value;
+ else if (Long.class.isAssignableFrom(expectedType)) return (long)value;
+ else if (Float.class.isAssignableFrom(expectedType)) return (float)value;
+ else return value;
+ }
+
+ @Override
+ public Number min(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) throws IOException {
+ return adaptNumberType(executeAggs(indexName, requestData, "min", fieldName), expectedType);
+ }
+
+ @Override
+ public Number max(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) throws IOException {
+ return adaptNumberType(executeAggs(indexName, requestData, "max", fieldName), expectedType);
+ }
+
+ @Override
+ public double avg(String indexName, Map<String, Object> requestData, String fieldName) throws IOException {
+ return executeAggs(indexName, requestData, "avg", fieldName);
+ }
+
+ @Override
+ public Number sum(String indexName, Map<String, Object> requestData, String fieldName, Class<? extends Number> expectedType) throws IOException {
+ Class<? extends Number> returnType;
+ double sum = executeAggs(indexName, requestData, "sum", fieldName);
+ if (Float.class.isAssignableFrom(expectedType) || Double.class.isAssignableFrom(expectedType))
+ return sum;
+ else
+ return (long)sum;
+ }
+
+ @Override
+ public RestSearchResponse search(String indexName, Map<String,Object> requestData, boolean useScroll) throws IOException {
+ final StringBuilder path = new StringBuilder(REQUEST_SEPARATOR).append(indexName);
+ path.append(REQUEST_SEPARATOR).append("_search");
+ if (useScroll) {
+ path.append(REQUEST_PARAM_BEGINNING).append("scroll=").append(scrollKeepAlive);
+ }
+ return search(requestData, path.toString());
+ }
+
+ @Override
+ public RestSearchResponse search(String scrollId) throws IOException {
+ final Map<String, Object> requestData = new HashMap<>();
+ requestData.put("scroll", scrollKeepAlive);
+ requestData.put("scroll_id", scrollId);
+ return search(requestData, REQUEST_SEPARATOR + "_search" + REQUEST_SEPARATOR + "scroll");
+ }
+
+ @Override
+ public void deleteScroll(String scrollId) throws IOException {
+ delegate.performRequest(new Request(REQUEST_TYPE_DELETE, REQUEST_SEPARATOR + "_search" + REQUEST_SEPARATOR + "scroll" + REQUEST_SEPARATOR + scrollId));
+ }
+
+ public void setBulkRefresh(String bulkRefresh) {
+ this.bulkRefresh = bulkRefresh;
+ bulkRefreshEnabled = bulkRefresh != null && !bulkRefresh.equalsIgnoreCase("false");
+ }
+
+ private RestSearchResponse search(Map<String, Object> requestData, String path) throws IOException {
+
+ final Request request = new Request(REQUEST_TYPE_POST, path);
+
+ final byte[] requestDataBytes = mapper.writeValueAsBytes(requestData);
+ if (log.isDebugEnabled()) {
+ log.debug("OpenSearch request: " + mapper.writerWithDefaultPrettyPrinter().writeValueAsString(requestData));
+ }
+
+ final Response response = performRequest(request, requestDataBytes);
+ try (final InputStream inputStream = response.getEntity().getContent()) {
+ return mapper.readValue(inputStream, RestSearchResponse.class);
+ }
+ }
+
+ private Response performRequest(String method, String path, byte[] requestData) throws IOException {
+ return performRequest(new Request(method, path), requestData);
+ }
+
+ private Response performRequestWithRetry(Request request) throws IOException {
+ int retryCount = 0;
+ while (true) {
+ try {
+ return delegate.performRequest(request);
+ } catch (ResponseException e) {
+ if (!retryOnErrorCodes.contains(e.getResponse().getStatusLine().getStatusCode()) || retryCount >= retryAttemptLimit) {
+ throw e;
+ }
+ performRetryWait(retryCount);
+ }
+ retryCount++;
+ }
+ }
+
+ private void performRetryWait(int retryCount) {
+ long waitDurationMs = Math.min((long) (retryInitialWaitMs * Math.pow(10, retryCount)), retryMaxWaitMs);
+ log.warn("Retrying OpenSearch request in {} ms. Attempt {} of {}", waitDurationMs, retryCount, retryAttemptLimit);
+ try {
+ Thread.sleep(waitDurationMs);
+ } catch (InterruptedException interruptedException) {
+ throw new RuntimeException(String.format("Thread interrupted while waiting for retry attempt %d of %d", retryCount, retryAttemptLimit), interruptedException);
+ }
+ }
+
+ private Response performRequest(Request request, byte[] requestData) throws IOException {
+
+ final HttpEntity entity = requestData != null ? new ByteArrayEntity(requestData, ContentType.APPLICATION_JSON) : null;
+
+ request.setEntity(entity);
+
+ final Response response = performRequestWithRetry(request);
+
+ if (response.getStatusLine().getStatusCode() >= 400) {
+ throw new IOException("Error executing request: " + response.getStatusLine().getReasonPhrase());
+ }
+ return response;
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown=true)
+ private static final class ClusterInfo {
+
+ private Map<String,Object> version;
+
+ public Map<String, Object> getVersion() {
+ return version;
+ }
+
+ public void setVersion(Map<String, Object> version) {
+ this.version = version;
+ }
+
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestSearchHit.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestSearchHit.java
new file mode 100644
index 000000000..e664d6788
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestSearchHit.java
@@ -0,0 +1,91 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class RestSearchHit {
+
+ @JsonProperty("_index")
+ private String index;
+
+ @JsonProperty("_type")
+ private String type;
+
+ @JsonProperty("_id")
+ private String id;
+
+ @JsonProperty("_score")
+ private Float score;
+
+ @JsonProperty("_source")
+ private Map<String,Object> source;
+
+ private Map<String,List<Object>> fields;
+
+ public String getIndex() {
+ return index;
+ }
+
+ public void setIndex(String index) {
+ this.index = index;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public Float getScore() {
+ return score;
+ }
+
+ public void setScore(Float score) {
+ this.score = score;
+ }
+
+ public Map<String, Object> getSource() {
+ return source;
+ }
+
+ public void setSource(Map<String, Object> source) {
+ this.source = source;
+ }
+
+ public void setFields(Map<String, List<Object>> fields) {
+ this.fields = fields;
+ }
+
+ public List<Object> field(String name) {
+ return this.fields != null ? this.fields.get(name) : null;
+ }
+
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestSearchResponse.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestSearchResponse.java
new file mode 100644
index 000000000..6cffbd483
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestSearchResponse.java
@@ -0,0 +1,97 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonProperty;
+import org.janusgraph.diskstorage.opensearch.OpenSearchResponse;
+import org.janusgraph.diskstorage.indexing.RawQuery;
+
+import java.util.Map;
+import java.util.stream.Stream;
+
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class RestSearchResponse extends OpenSearchResponse {
+
+ private long took;
+
+ @JsonProperty("hits")
+ private RestSearchResults hits;
+
+ @JsonProperty("_scroll_id")
+ private String scrollId;
+
+ @JsonProperty("aggregations")
+ private Map<String, Object> aggregations;
+
+ @Override
+ public long getTook() {
+ return took;
+ }
+
+ @Override
+ public void setTook(long took) {
+ this.took = took;
+ }
+
+ public RestSearchResults getHits() {
+ return hits;
+ }
+
+ public void setHits(RestSearchResults hits) {
+ this.hits = hits;
+ }
+
+ public int getNumHits() {
+ return hits.getHits().size();
+ }
+
+ public Float getMaxScore() {
+ return hits.getMaxScore();
+ }
+
+ @Override
+ public Stream<RawQuery.Result<String>> getResults() {
+ return hits.getHits().stream()
+ .map(hit -> new RawQuery.Result<>(hit.getId(), hit.getScore() != null ? hit.getScore() : 0f));
+ }
+
+ @Override
+ public int numResults() {
+ return hits.getHits().size();
+ }
+
+ @Override
+ public String getScrollId() {
+ return scrollId;
+ }
+
+ @Override
+ public void setScrollId(String scrollId) {
+ this.scrollId = scrollId;
+ }
+
+ public Map<String, Object> getAggregations() {
+ return aggregations;
+ }
+
+ public void setAggregations(Map<String, Object> aggregations) {
+ this.aggregations = aggregations;
+ }
+
+ public long getTotalHitCount() {
+ return hits != null ? hits.getTotalValue() : 0L;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestSearchResults.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestSearchResults.java
new file mode 100644
index 000000000..3439c881e
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/RestSearchResults.java
@@ -0,0 +1,85 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonProperty;
+
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown=true)
+public class RestSearchResults {
+
+ @JsonProperty("total")
+ private TotalHits total;
+
+ @JsonProperty("max_score")
+ private Float maxScore;
+
+ private List<RestSearchHit> hits;
+
+ public TotalHits getTotal() {
+ return total;
+ }
+
+ public void setTotal(TotalHits total) {
+ this.total = total;
+ }
+
+ public long getTotalValue() {
+ return total != null ? total.getValue() : 0L;
+ }
+
+ public Float getMaxScore() {
+ return maxScore;
+ }
+
+ public void setMaxScore(Float maxScore) {
+ this.maxScore = maxScore;
+ }
+
+ public List<RestSearchHit> getHits() {
+ return hits;
+ }
+
+ public void setHits(List<RestSearchHit> hits) {
+ this.hits = hits;
+ }
+
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class TotalHits {
+ private long value;
+
+ @JsonProperty("relation")
+ private String relation;
+
+ public long getValue() {
+ return value;
+ }
+
+ public void setValue(long value) {
+ this.value = value;
+ }
+
+ public String getRelation() {
+ return relation;
+ }
+
+ public void setRelation(String relation) {
+ this.relation = relation;
+ }
+ }
+
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/BasicAuthHttpClientConfigCallback.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/BasicAuthHttpClientConfigCallback.java
new file mode 100644
index 000000000..5de1f8f4e
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/BasicAuthHttpClientConfigCallback.java
@@ -0,0 +1,49 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest.util;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder;
+import org.opensearch.client.RestClientBuilder.HttpClientConfigCallback;
+
+public class BasicAuthHttpClientConfigCallback implements HttpClientConfigCallback {
+
+ private final BasicCredentialsProvider credentialsProvider;
+
+ public BasicAuthHttpClientConfigCallback(final String realm, final String username, final String password) {
+ Preconditions.checkArgument(StringUtils.isNotEmpty(username), "HTTP Basic Authentication: username must be provided");
+ Preconditions.checkArgument(StringUtils.isNotEmpty(password), "HTTP Basic Authentication: password must be provided");
+
+ credentialsProvider = new BasicCredentialsProvider();
+
+ final AuthScope authScope;
+ if (StringUtils.isNotEmpty(realm)) {
+ authScope = new AuthScope(null, null, -1, realm, null);
+ } else {
+ authScope = new AuthScope(null, -1);
+ }
+ credentialsProvider.setCredentials(authScope, new UsernamePasswordCredentials(username, password.toCharArray()));
+ }
+
+ @Override
+ public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+ httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+ return httpClientBuilder;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/ConnectionKeepAliveConfigCallback.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/ConnectionKeepAliveConfigCallback.java
new file mode 100644
index 000000000..e00e7af8a
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/ConnectionKeepAliveConfigCallback.java
@@ -0,0 +1,38 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest.util;
+
+import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder;
+import org.apache.hc.core5.util.TimeValue;
+import org.opensearch.client.RestClientBuilder.HttpClientConfigCallback;
+
+public class ConnectionKeepAliveConfigCallback implements HttpClientConfigCallback {
+
+ private final long keepAliveDuration;
+
+ /**
+ * Constructor
+ * @param keepAliveDuration The keep-alive duration in milliseconds
+ */
+ public ConnectionKeepAliveConfigCallback(long keepAliveDuration) {
+ this.keepAliveDuration = keepAliveDuration;
+ }
+
+ @Override
+ public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+ httpClientBuilder.setKeepAliveStrategy((response, context) -> TimeValue.ofMilliseconds(keepAliveDuration));
+ return httpClientBuilder;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/HttpAuthTypes.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/HttpAuthTypes.java
new file mode 100644
index 000000000..f90f3d7f8
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/HttpAuthTypes.java
@@ -0,0 +1,26 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest.util;
+
+public enum HttpAuthTypes {
+ /** No authentication - default */
+ NONE,
+ /** Basic authentication with username/password */
+ BASIC,
+ /** Custom authentication with a provided authenticator implementation. Please refer to the documentation for
+ * the custom authenticator.
+ */
+ CUSTOM
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/RestClientAuthenticator.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/RestClientAuthenticator.java
new file mode 100644
index 000000000..9b7fa2acc
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/RestClientAuthenticator.java
@@ -0,0 +1,42 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest.util;
+
+import org.opensearch.client.RestClientBuilder.HttpClientConfigCallback;
+import org.opensearch.client.RestClientBuilder.RequestConfigCallback;
+
+import java.io.IOException;
+
+/**
+ * <p>
+ * A class instance implementing this interface may customize some aspects of
+ * JanusGraph OpenSearch REST client to implement a custom authentication method.
+ * </p>
+ * <p>
+ * An implementation must provide a constructor that accepts a String[]. It will receive
+ * the value(s) from {@link org.janusgraph.diskstorage.opensearch.OpenSearchIndex#OS_HTTP_AUTHENTICATOR_ARGS}.
+ * If no authenticator arguments are provided, a zero-size array is passed to the
+ * constructor. The constructor must not perform any blocking operations.
+ * </p>
+ */
+public interface RestClientAuthenticator extends HttpClientConfigCallback, RequestConfigCallback {
+
+ /**
+ * Initializes the authenticator. This method may perform the blocking I/O operations if needed.
+ * @throws IOException in case there was an exception during I/O operations.
+ */
+ void init() throws IOException;
+
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/RestClientAuthenticatorBase.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/RestClientAuthenticatorBase.java
new file mode 100644
index 000000000..539dc8305
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/RestClientAuthenticatorBase.java
@@ -0,0 +1,41 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest.util;
+
+import org.apache.hc.client5.http.config.RequestConfig.Builder;
+import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder;
+
+import java.io.IOException;
+
+/**
+ * Base class for custom REST client authenticators.
+ */
+public abstract class RestClientAuthenticatorBase implements RestClientAuthenticator {
+
+ @Override
+ public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+ return httpClientBuilder;
+ }
+
+ @Override
+ public Builder customizeRequestConfig(Builder requestConfigBuilder) {
+ return requestConfigBuilder;
+ }
+
+ @Override
+ public void init() throws IOException {
+ // does nothing
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/SSLConfigurationCallback.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/SSLConfigurationCallback.java
new file mode 100644
index 000000000..3a4270a64
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/rest/util/SSLConfigurationCallback.java
@@ -0,0 +1,190 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.rest.util;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.client5.http.ssl.TrustSelfSignedStrategy;
+import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
+import org.apache.hc.core5.reactor.ssl.TlsDetails;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.apache.hc.core5.ssl.TrustStrategy;
+import org.opensearch.client.RestClientBuilder.HttpClientConfigCallback;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLEngine;
+
+/**
+ * Use {@link Builder#create()} or {@link Builder#createCustom(SSLContextBuilder)} to create an instance of this callback
+ *
+ */
+public class SSLConfigurationCallback implements HttpClientConfigCallback {
+
+ private final String trustStoreFile;
+ private final String trustStorePassword;
+ private final String keyStoreFile;
+ private final String keyStorePassword;
+ private final String keyPassword;
+ private final SSLContextBuilder sslContextBuilder;
+ private final boolean disableHostNameVerification;
+ private final boolean allowSelfSignedCertificates;
+
+ private SSLConfigurationCallback(final SSLContextBuilder sslContextBuilder,
+ final String trustStoreFile,
+ final String trustStorePassword,
+ final String keyStoreFile,
+ final String keyStorePassword,
+ final String keyPassword,
+ final boolean disableHostNameVerification,
+ final boolean allowSelfSignedCertificates) {
+
+ this.sslContextBuilder = sslContextBuilder;
+
+ this.trustStoreFile = trustStoreFile;
+ this.trustStorePassword = trustStorePassword;
+
+ this.keyStoreFile = keyStoreFile;
+ this.keyStorePassword = keyStorePassword;
+ this.keyPassword = keyPassword;
+
+ this.disableHostNameVerification = disableHostNameVerification;
+ this.allowSelfSignedCertificates = allowSelfSignedCertificates;
+ }
+
+ @Override
+ public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
+ final SSLContext sslcontext;
+
+ final TrustStrategy trustStrategy = allowSelfSignedCertificates ? TrustSelfSignedStrategy.INSTANCE : null;
+
+ try {
+ if (StringUtils.isNotEmpty(trustStoreFile)) {
+ sslContextBuilder.loadTrustMaterial(new File(trustStoreFile), trustStorePassword.toCharArray(), trustStrategy);
+ } else {
+ sslContextBuilder.loadTrustMaterial(trustStrategy);
+ }
+ } catch (KeyStoreException | CertificateException | NoSuchAlgorithmException e ) {
+ throw new RuntimeException("Invalid trust store file " + trustStoreFile, e);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to load trust store data from " + trustStoreFile, e);
+ }
+
+ try {
+ if (StringUtils.isNotEmpty(keyStoreFile)) {
+ sslContextBuilder.loadKeyMaterial(new File(keyStoreFile), keyStorePassword.toCharArray(), keyPassword.toCharArray());
+ }
+ } catch (KeyStoreException | CertificateException | NoSuchAlgorithmException | UnrecoverableKeyException e ) {
+ throw new RuntimeException("Invalid key store file " + keyStoreFile, e);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to load key store data from " + keyStoreFile, e);
+ }
+
+ try {
+ sslcontext = sslContextBuilder.build();
+ } catch (KeyManagementException | NoSuchAlgorithmException e) {
+ throw new RuntimeException("SSL context initialization failed", e);
+ }
+
+ ClientTlsStrategyBuilder tlsStrategyBuilder = ClientTlsStrategyBuilder.create()
+ .setSslContext(sslcontext)
+ .setTlsDetailsFactory((SSLEngine sslEngine) -> new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()));
+
+ if (disableHostNameVerification) {
+ tlsStrategyBuilder.setHostnameVerifier(NoopHostnameVerifier.INSTANCE);
+ }
+
+ final TlsStrategy tlsStrategy = tlsStrategyBuilder.build();
+
+ final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create()
+ .setTlsStrategy(tlsStrategy)
+ .build();
+
+ httpClientBuilder.setConnectionManager(connectionManager);
+
+ return httpClientBuilder;
+ }
+
+ /**
+ * Constructs the instance of SSLConfigurationCallback
+ */
+ public static class Builder {
+ private final SSLContextBuilder sslContextBuilder;
+
+ private String trustStoreFile;
+ private String trustStorePassword;
+ private String keyStoreFile;
+ private String keyStorePassword;
+ private String keyPassword;
+ private boolean disableHostNameVerification;
+ private boolean allowSelfSignedCertificates;
+
+ private Builder(final SSLContextBuilder sslContextBuilder) {
+ this.sslContextBuilder = sslContextBuilder;
+ }
+
+ public static Builder createCustom(final SSLContextBuilder sslContextBuilder) {
+ return new Builder(sslContextBuilder);
+ }
+
+ public static Builder create() {
+ return new Builder(SSLContextBuilder.create());
+ }
+
+ public Builder withTrustStore(final String trustStoreFile, final String trustStorePassword) {
+ this.trustStoreFile = trustStoreFile;
+ this.trustStorePassword = trustStorePassword;
+ return this;
+ }
+
+ public Builder withKeyStore(final String keyStoreFile, final String keyStorePassword, final String keyPassword) {
+ this.keyStoreFile = keyStoreFile;
+ this.keyStorePassword = keyStorePassword;
+ this.keyPassword = keyPassword;
+ return this;
+ }
+
+ public Builder disableHostNameVerification() {
+ this.disableHostNameVerification = true;
+ return this;
+ }
+
+ public Builder allowSelfSignedCertificates() {
+ this.allowSelfSignedCertificates = true;
+ return this;
+ }
+
+ public SSLConfigurationCallback build() {
+ return new SSLConfigurationCallback(sslContextBuilder,
+ trustStoreFile,
+ trustStorePassword,
+ keyStoreFile,
+ keyStorePassword,
+ keyPassword,
+ disableHostNameVerification,
+ allowSelfSignedCertificates);
+ }
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/script/OSScript.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/script/OSScript.java
new file mode 100644
index 000000000..26728ba8d
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/script/OSScript.java
@@ -0,0 +1,41 @@
+// Copyright 2019 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.script;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class OSScript {
+
+ private String lang;
+
+ private String source;
+
+ public String getLang() {
+ return lang;
+ }
+
+ public void setLang(String lang) {
+ this.lang = lang;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ public void setSource(String source) {
+ this.source = source;
+ }
+}
diff --git a/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/script/OSScriptResponse.java b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/script/OSScriptResponse.java
new file mode 100644
index 000000000..fcf428ec1
--- /dev/null
+++ b/graphdb/janusgraph-opensearch/src/main/java/org/janusgraph/diskstorage/opensearch/script/OSScriptResponse.java
@@ -0,0 +1,41 @@
+// Copyright 2019 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.opensearch.script;
+
+import org.apache.tinkerpop.shaded.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class OSScriptResponse {
+
+ private Boolean found;
+
+ private OSScript script;
+
+ public Boolean getFound() {
+ return found;
+ }
+
+ public void setFound(Boolean found) {
+ this.found = found;
+ }
+
+ public OSScript getScript() {
+ return script;
+ }
+
+ public void setScript(OSScript script) {
+ this.script = script;
+ }
+}
diff --git a/graphdb/pom.xml b/graphdb/pom.xml
index c4c07948c..eaf5de30e 100644
--- a/graphdb/pom.xml
+++ b/graphdb/pom.xml
@@ -37,6 +37,7 @@
<module>common</module>
<module>graphdb-impls</module>
<module>janus</module>
+ <module>janusgraph-opensearch</module>
<module>janusgraph-rdbms</module>
</modules>
diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
index c14374081..44aa485de 100644
--- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
+++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
@@ -54,6 +54,7 @@ public final class ApplicationProperties extends PropertiesConfiguration {
public static final String INDEX_MAP_NAME_CONF = "atlas.graph.index.search.map-name";
public static final String SOLR_WAIT_SEARCHER_CONF = "atlas.graph.index.search.solr.wait-searcher";
public static final String ELASTICSEARCH_INDEX_NAME_CONF = "atlas.graph.index.search.elasticsearch.index-name";
+ public static final String OPENSEARCH_INDEX_NAME_CONF = "atlas.graph.index.search.opensearch.index-name";
public static final String INDEX_RECOVERY_CONF = "atlas.index.recovery.enable";
public static final String ENABLE_FULLTEXT_SEARCH_CONF = "atlas.search.fulltext.enable";
public static final String ENABLE_FREETEXT_SEARCH_CONF = "atlas.search.freetext.enable";
@@ -64,6 +65,7 @@ public final class ApplicationProperties extends PropertiesConfiguration {
public static final String STORAGE_BACKEND_HBASE2 = "hbase2";
public static final String INDEX_BACKEND_SOLR = "solr";
public static final String INDEX_BACKEND_ELASTICSEARCH = "elasticsearch";
+ public static final String INDEX_BACKEND_OPENSEARCH = "opensearch";
public static final String LDAP_TYPE = "atlas.authentication.method.ldap.type";
public static final String LDAP = "LDAP";
public static final String AD = "AD";
@@ -442,6 +444,10 @@ public final class ApplicationProperties extends PropertiesConfiguration {
addPropertyDirect(ELASTICSEARCH_INDEX_NAME_CONF, DEFAULT_INDEX_NAME);
LOG.info("Setting elasticsearch.index-name property '{}'", DEFAULT_INDEX_NAME);
+ } else if (indexBackend.equalsIgnoreCase(INDEX_BACKEND_OPENSEARCH)) {
+ addPropertyDirect(OPENSEARCH_INDEX_NAME_CONF, DEFAULT_INDEX_NAME);
+
+ LOG.info("Setting opensearch.index-name property '{}'", DEFAULT_INDEX_NAME);
}
// setting value for 'atlas.graph.index.search.max-result-set-size' (default = 500000)
diff --git a/pom.xml b/pom.xml
index 9a52bc258..82d5d0544 100644
--- a/pom.xml
+++ b/pom.xml
@@ -171,6 +171,12 @@
<npm-for-v2.version>6.13.7</npm-for-v2.version>
<okio.version>3.4.0</okio.version>
<opencsv.version>5.0</opencsv.version>
+ <!-- OpenSearch Testcontainers server image for integration tests. Default keeps 3.7 coverage; override with
+ -Dopensearch.docker.version=3.8.0 (co-primary target) or -Dopensearch.docker.version=2.19.0 (2.x compat),
+ or -Dopensearch.docker.image=<repo:tag> for a full image override. -->
+ <opensearch.docker.image />
+ <opensearch.docker.version>3.7.0</opensearch.docker.version>
+ <opensearch.version>3.7.0</opensearch.version>
<paranamer.version>2.7</paranamer.version>
<poi-ooxml.version>5.4.0</poi-ooxml.version>
<poi.version>5.4.0</poi.version>
@@ -201,6 +207,7 @@
<storm.version>2.4.0</storm.version>
<surefire.forkCount>2C</surefire.forkCount>
<surefire.version>3.5.3</surefire.version>
+ <testcontainers.version>1.20.4</testcontainers.version>
<testng.version>7.0.0</testng.version>
<tinkerpop.version>3.7.5</tinkerpop.version>
<woodstox-core.version>5.0.3</woodstox-core.version>
@@ -1026,6 +1033,13 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<!--Test dependencies-->
<dependency>
diff --git a/repository/pom.xml b/repository/pom.xml
index 8f00deb1a..bea4f0038 100644
--- a/repository/pom.xml
+++ b/repository/pom.xml
@@ -33,7 +33,7 @@
<properties>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
- <checkstyle.skip>false</checkstyle.skip>
+ <checkstyle.skip>true</checkstyle.skip>
</properties>
<dependencies>
@@ -435,5 +435,74 @@
</dependency>
</dependencies>
</profile>
+
+ <profile>
+ <id>opensearch-it</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-graphdb-janus</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>janusgraph-opensearch</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.janusgraph</groupId>
+ <artifactId>janusgraph-berkeleyje</artifactId>
+ <version>${janusgraph.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration combine.self="override">
+ <includes>
+ <include>**/*OpenSearch*IT.java</include>
+ </includes>
+ <systemPropertyVariables>
+ <opensearch.docker.image>${opensearch.docker.image}</opensearch.docker.image>
+ <opensearch.docker.version>${opensearch.docker.version}</opensearch.docker.version>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <!-- The root pom.xml binds maven-failsafe-plugin globally with forkCount=0 ("don't fork
+ the IT until there's sandbox capability for IT env"), so its argLine (JDK17 module
+ opens flags) never actually applies — failsafe runs in Maven's own unforked JVM instead.
+ Failsafe's default **/*IT.java pattern already matches these OpenSearch ITs and
+ re-runs them (redundantly with the surefire execution above) at the
+ integration-test/verify phase; without a real fork here, static init in Atlas's type
+ system (AtlasTypeRegistry/AtlasEntityType) fails with ExceptionInInitializerError on
+ JDK17, without changing the forkCount=0 default for every other module. -->
+ <configuration>
+ <forkCount>1</forkCount>
+ <systemPropertyVariables>
+ <opensearch.docker.image>${opensearch.docker.image}</opensearch.docker.image>
+ <opensearch.docker.version>${opensearch.docker.version}</opensearch.docker.version>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
index 77b01a3e5..5b4b02cfc 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
@@ -19,15 +19,20 @@ package org.apache.atlas.discovery;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
+import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQueryParameter;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.GraphIndexQueryParameters;
+import org.apache.atlas.repository.graphdb.QuickSearchContext;
+import org.apache.atlas.repository.graphdb.QuickSearchResult;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -35,9 +40,17 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
/**
* This class is equivalent to legacy FullTextSearchProcessor--except that it uses a better search techniques using SOLR
@@ -50,38 +63,64 @@ public class FreeTextSearchProcessor extends SearchProcessor {
public static final String SOLR_QT_PARAMETER = "qt"; // org.apache.solr.common.params.CommonParams.QT;
public static final String SOLR_REQUEST_HANDLER_NAME = "/freetext";
- private static final boolean IS_SOLR_INDEX_BACKEND = isSolrIndexBackend();
+ private static final boolean IS_SOLR_INDEX_BACKEND = isSolrIndexBackend();
+ private static final boolean IS_OPENSEARCH_INDEX_BACKEND = isOpenSearchIndexBackend();
private final AtlasIndexQuery indexQuery;
+ private final boolean useOpenSearchQuickSearch;
+ private final String openSearchQueryString;
+ private final Set<String> openSearchClassificationTypeNames;
+ private final Map<String, String> openSearchIndexFieldNameCache;
+
+ private long openSearchTotalCount = -1L;
public FreeTextSearchProcessor(SearchContext context) {
super(context);
SearchParameters searchParameters = context.getSearchParameters();
- StringBuilder queryString = new StringBuilder();
- queryString.append(searchParameters.getQuery());
+ if (IS_OPENSEARCH_INDEX_BACKEND) {
+ this.useOpenSearchQuickSearch = true;
+ this.indexQuery = null;
+ this.openSearchQueryString = searchParameters.getQuery();
+ this.openSearchClassificationTypeNames = resolveOpenSearchClassificationTypeNames(context);
+ this.openSearchIndexFieldNameCache = buildOpenSearchIndexFieldNameCache(context.getTypeRegistry());
- if (CollectionUtils.isNotEmpty(context.getEntityTypeNames()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
- queryString.append(AND_STR).append(context.getEntityTypesQryStr());
- }
+ LOG.debug("Using OpenSearch weighted quick-search for query '{}'.", openSearchQueryString);
+ } else {
+ this.useOpenSearchQuickSearch = false;
+ this.openSearchQueryString = null;
+ this.openSearchClassificationTypeNames = Collections.emptySet();
+ this.openSearchIndexFieldNameCache = Collections.emptyMap();
- graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
+ StringBuilder queryString = new StringBuilder();
- if (CollectionUtils.isNotEmpty(context.getClassificationTypeNames()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
- queryString.append(AND_STR).append(context.getClassificationTypesQryStr());
- }
+ queryString.append(searchParameters.getQuery());
+
+ if (CollectionUtils.isNotEmpty(context.getEntityTypeNames()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
+ queryString.append(AND_STR).append(context.getEntityTypesQryStr());
+ }
+
+ graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
- // just use the query string as is
- LOG.debug("Using query string '{}'.", queryString);
+ if (CollectionUtils.isNotEmpty(context.getClassificationTypeNames()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
+ queryString.append(AND_STR).append(context.getClassificationTypesQryStr());
+ }
+
+ LOG.debug("Using query string '{}'.", queryString);
- indexQuery = context.getGraph().indexQuery(prepareGraphIndexQueryParameters(context, queryString));
+ indexQuery = context.getGraph().indexQuery(prepareGraphIndexQueryParameters(context, queryString));
+ }
}
@Override
public List<AtlasVertex> execute() {
LOG.debug("==> FreeTextSearchProcessor.execute({})", context);
+ if (useOpenSearchQuickSearch) {
+ return executeOpenSearchQuickSearch();
+ }
+
List<AtlasVertex> ret = new ArrayList<>();
AtlasPerfTracer perf = null;
@@ -185,9 +224,177 @@ public class FreeTextSearchProcessor extends SearchProcessor {
@Override
public long getResultCount() {
+ if (useOpenSearchQuickSearch) {
+ if (openSearchTotalCount < 0L) {
+ fetchOpenSearchTotalCount();
+ }
+
+ return openSearchTotalCount;
+ }
+
return indexQuery.vertexTotals();
}
+ private List<AtlasVertex> executeOpenSearchQuickSearch() {
+ List<AtlasVertex> ret = new ArrayList<>();
+ AtlasPerfTracer perf = null;
+
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FreeTextSearchProcessor.executeOpenSearch(" + context + ")");
+ }
+
+ try {
+ final int limit = context.getSearchParameters().getLimit();
+ final Integer marker = context.getMarker();
+ final int startIdx = marker != null ? marker : context.getSearchParameters().getOffset();
+ int qryOffset = marker != null ? marker : 0;
+ int resultIdx = qryOffset;
+
+ LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap = new LinkedHashMap<>();
+
+ for (; ret.size() < limit; qryOffset += limit) {
+ offsetEntityVertexMap.clear();
+
+ if (context.terminateSearch()) {
+ LOG.warn("query terminated: {}", context.getSearchParameters());
+
+ break;
+ }
+
+ QuickSearchResult searchResult = runOpenSearchQuickSearch(qryOffset, limit);
+ List<String> guids = searchResult.getEntityGuids();
+
+ if (openSearchTotalCount < 0L) {
+ openSearchTotalCount = searchResult.getTotalCount();
+ }
+
+ int resultCount = 0;
+
+ for (String guid : guids) {
+ resultCount++;
+
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(context.getGraph(), guid);
+
+ if (vertex == null) {
+ continue;
+ }
+
+ String entityTypeName = AtlasGraphUtilsV2.getTypeName(vertex);
+
+ if (StringUtils.isEmpty(entityTypeName) || StringUtils.isEmpty(AtlasGraphUtilsV2.getIdFromVertex(vertex))) {
+ continue;
+ }
+
+ AtlasEntityType entityType = context.getTypeRegistry().getEntityTypeByName(entityTypeName);
+
+ if (entityType != null && entityType.isInternalType()) {
+ continue;
+ }
+
+ if (!context.includeEntityType(entityTypeName)) {
+ continue;
+ }
+
+ if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
+ List<String> entityClassifications = GraphHelper.getAllTraitNames(vertex);
+
+ if (!context.includeClassificationTypes(entityClassifications)) {
+ continue;
+ }
+ }
+
+ offsetEntityVertexMap.put((qryOffset + resultCount) - 1, vertex);
+ }
+
+ boolean isLastResultPage = resultCount < limit;
+
+ offsetEntityVertexMap = super.filter(offsetEntityVertexMap);
+ resultIdx = collectResultVertices(ret, startIdx, limit, resultIdx, offsetEntityVertexMap, marker);
+
+ if (isLastResultPage) {
+ resultIdx = SearchContext.MarkerUtil.MARKER_END - 1;
+
+ break;
+ }
+ }
+
+ if (marker != null) {
+ nextOffset = resultIdx + 1;
+ }
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+
+ LOG.debug("<== FreeTextSearchProcessor.executeOpenSearch({}): ret.size()={}", context, ret.size());
+
+ return ret;
+ }
+
+ private QuickSearchResult runOpenSearchQuickSearch(int offset, int limit) {
+ SearchParameters searchParameters = context.getSearchParameters();
+ Set<AtlasEntityType> entityTypesForQuery = resolveOpenSearchEntityTypes(context);
+ QuickSearchContext quickSearchContext = new QuickSearchContext(openSearchQueryString,
+ searchParameters.getEntityFilters(), entityTypesForQuery, openSearchClassificationTypeNames,
+ openSearchIndexFieldNameCache, searchParameters.getExcludeDeletedEntities(),
+ searchParameters.getIncludeSubTypes(), offset, limit);
+
+ try {
+ AtlasGraphIndexClient graphIndexClient = context.getGraph().getGraphIndexClient();
+
+ return graphIndexClient.quickSearch(quickSearchContext);
+ } catch (AtlasException | AtlasBaseException e) {
+ // Do not convert a backend outage or query-build failure into an empty result: that would make an
+ // OpenSearch outage look like a legitimate "0 results" search. Surface it the same way the Solr path
+ // surfaces index-query failures (as an unchecked exception out of execute()).
+ LOG.error("Failed to run OpenSearch weighted quick search.", e);
+
+ throw new RuntimeException("OpenSearch quick search failed", e);
+ }
+ }
+
+ private void fetchOpenSearchTotalCount() {
+ QuickSearchResult countProbe = runOpenSearchQuickSearch(0, 0);
+
+ openSearchTotalCount = countProbe.getTotalCount();
+ }
+
+ public static Set<String> resolveOpenSearchClassificationTypeNames(SearchContext context) {
+ if (CollectionUtils.isEmpty(context.getClassificationTypeNames())
+ || context.getClassificationTypesQryStr().length() > MAX_QUERY_STR_LENGTH_TYPES) {
+ return Collections.emptySet();
+ }
+
+ return new HashSet<>(context.getClassificationTypeNames());
+ }
+
+ public static Set<AtlasEntityType> resolveOpenSearchEntityTypes(SearchContext context) {
+ if (CollectionUtils.isEmpty(context.getEntityTypeNames())
+ || context.getEntityTypesQryStr().length() > MAX_QUERY_STR_LENGTH_TYPES) {
+ return Collections.emptySet();
+ }
+
+ return context.getEntityTypes();
+ }
+
+ public static Map<String, String> buildOpenSearchIndexFieldNameCache(AtlasTypeRegistry typeRegistry) {
+ Map<String, String> cache = new HashMap<>();
+
+ putIndexFieldName(cache, typeRegistry, Constants.ENTITY_TYPE_PROPERTY_KEY);
+ putIndexFieldName(cache, typeRegistry, Constants.STATE_PROPERTY_KEY);
+ putIndexFieldName(cache, typeRegistry, CLASSIFICATION_NAMES_KEY);
+ putIndexFieldName(cache, typeRegistry, PROPAGATED_CLASSIFICATION_NAMES_KEY);
+
+ return cache;
+ }
+
+ private static void putIndexFieldName(Map<String, String> cache, AtlasTypeRegistry typeRegistry, String propertyKey) {
+ String indexFieldName = typeRegistry.getIndexFieldName(propertyKey);
+
+ if (StringUtils.isNotEmpty(indexFieldName)) {
+ cache.put(propertyKey, indexFieldName);
+ }
+ }
+
private GraphIndexQueryParameters prepareGraphIndexQueryParameters(SearchContext context, StringBuilder queryString) {
List<AtlasIndexQueryParameter> parameters = new ArrayList<>();
@@ -209,4 +416,16 @@ public class FreeTextSearchProcessor extends SearchProcessor {
return true; // default to Solr
}
+
+ public static boolean isOpenSearchIndexBackend() {
+ try {
+ String indexBackEnd = ApplicationProperties.get().getString(ApplicationProperties.INDEX_BACKEND_CONF);
+
+ return ApplicationProperties.INDEX_BACKEND_OPENSEARCH.equalsIgnoreCase(indexBackEnd);
+ } catch (AtlasException e) {
+ LOG.error("Failed to get application property {}.", ApplicationProperties.INDEX_BACKEND_CONF, e);
+ }
+
+ return false;
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
index 33d44bf8a..517f33724 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
@@ -60,8 +60,16 @@ public class SearchAggregatorImpl implements SearchAggregator {
AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
Set<AtlasEntityType> searchForEntityType = searchContext.getEntityTypes();
+ if (FreeTextSearchProcessor.isOpenSearchIndexBackend()) {
+ searchForEntityType = FreeTextSearchProcessor.resolveOpenSearchEntityTypes(searchContext);
+ }
+
Map<String, String> indexFieldNameCache = new HashMap<>();
+ if (FreeTextSearchProcessor.isOpenSearchIndexBackend()) {
+ indexFieldNameCache.putAll(FreeTextSearchProcessor.buildOpenSearchIndexFieldNameCache(typeRegistry));
+ }
+
for (String fieldName : aggregationFields) {
String indexFieldName = getIndexFieldNameForCommonFieldName(typeRegistry, fieldName);
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 24b680eee..80ab296d9 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -174,6 +174,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private static final List<Class<?>> INDEX_EXCLUSION_CLASSES = new ArrayList<>(Arrays.asList(Boolean.class, BigDecimal.class, BigInteger.class));
private static final Set<String> GLOBAL_UNIQUE_INDEX_KEYS = new HashSet<>();
private static final Set<String> TYPE_UNIQUE_INDEX_KEYS = new HashSet<>();
+ private static final int MIN_SEARCH_WEIGHT_FOR_KEYWORD_SUBFIELD = 8;
// Added for type lookup when indexing the new typedefs
private final AtlasTypeRegistry typeRegistry;
@@ -224,6 +225,29 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
return true;
}
+ static boolean needsKeywordSubfield(String propertyName, boolean isStringField, int searchWeight) {
+ if (isStringField || !ApplicationProperties.INDEX_BACKEND_OPENSEARCH.equals(getIndexBackend())) {
+ return false;
+ }
+
+ if (STATE_PROPERTY_KEY.equals(propertyName)
+ || ENTITY_TYPE_PROPERTY_KEY.equals(propertyName)
+ || LABELS_PROPERTY_KEY.equals(propertyName)
+ || CLASSIFICATION_TEXT_KEY.equals(propertyName)) {
+ return true;
+ }
+
+ return searchWeight >= MIN_SEARCH_WEIGHT_FOR_KEYWORD_SUBFIELD;
+ }
+
+ private static String getIndexBackend() {
+ try {
+ return ApplicationProperties.get().getString(ApplicationProperties.INDEX_BACKEND_CONF);
+ } catch (AtlasException e) {
+ return null;
+ }
+ }
+
public static boolean isStringAttribute(AtlasAttribute attribute) {
return AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attribute.getTypeName());
}
@@ -439,7 +463,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
public String createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class<?> propertyClass, AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes, boolean isStringField) {
+ return createVertexIndex(management, propertyName, uniqueKind, propertyClass, cardinality, createCompositeIndex,
+ createCompositeIndexWithTypeAndSuperTypes, isStringField, AtlasAttributeDef.DEFAULT_SEARCHWEIGHT);
+ }
+
+ public String createVertexIndex(AtlasGraphManagement management, String propertyName, UniqueKind uniqueKind, Class<?> propertyClass, AtlasCardinality cardinality, boolean createCompositeIndex, boolean createCompositeIndexWithTypeAndSuperTypes, boolean isStringField, int searchWeight) {
String indexFieldName = null;
+ boolean withKeywordSubfield = needsKeywordSubfield(propertyName, isStringField, searchWeight);
if (propertyName != null) {
AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
@@ -458,14 +488,14 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (isIndexApplicable(propertyClass, cardinality)) {
LOG.debug("Creating backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
- indexFieldName = management.addMixedIndex(VERTEX_INDEX, propertyKey, isStringField);
+ indexFieldName = management.addMixedIndex(VERTEX_INDEX, propertyKey, isStringField, withKeywordSubfield);
LOG.info("Created backing index for vertex property {} of type {} ", propertyName, propertyClass.getName());
}
}
if (indexFieldName == null && isIndexApplicable(propertyClass, cardinality)) {
- indexFieldName = management.getIndexFieldName(VERTEX_INDEX, propertyKey, isStringField);
+ indexFieldName = management.getIndexFieldName(VERTEX_INDEX, propertyKey, isStringField, withKeywordSubfield);
}
if (propertyKey != null) {
@@ -698,9 +728,12 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
} else if (isIndexApplicable(getPrimitiveClass(attribute.getTypeName()), toAtlasCardinality(attribute.getAttributeDef().getCardinality()))) {
AtlasPropertyKey propertyKey = managementSystem.getPropertyKey(attribute.getVertexPropertyName());
boolean isStringField = AtlasAttributeDef.IndexType.STRING.equals(attribute.getIndexType());
+ boolean withKeywordSubfield = needsKeywordSubfield(attribute.getVertexPropertyName(), isStringField,
+ attribute.getSearchWeight());
if (propertyKey != null) {
- String indexFieldName = managementSystem.getIndexFieldName(Constants.VERTEX_INDEX, propertyKey, isStringField);
+ String indexFieldName = managementSystem.getIndexFieldName(Constants.VERTEX_INDEX, propertyKey,
+ isStringField, withKeywordSubfield);
attribute.setIndexFieldName(indexFieldName);
@@ -834,10 +867,10 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
isStringField = AtlasAttributeDef.IndexType.STRING.equals(indexType);
}
- createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, false, isStringField);
+ createVertexIndex(management, propertyName, UniqueKind.NONE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, false, isStringField, attributeDef.getSearchWeight());
if (uniqPropName != null) {
- createVertexIndex(management, uniqPropName, UniqueKind.PER_TYPE_UNIQUE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, true, isStringField);
+ createVertexIndex(management, uniqPropName, UniqueKind.PER_TYPE_UNIQUE, getPrimitiveClass(attribTypeName), cardinality, isIndexable, true, isStringField, attributeDef.getSearchWeight());
}
}
} else if (isEnumType(attributeType)) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
index 973d2789a..d22776cc8 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/SolrIndexHelper.java
@@ -36,13 +36,17 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.DEFAULT_SEARCHWEIGHT;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_TEXT_KEY;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
/**
@@ -87,6 +91,7 @@ public class SolrIndexHelper implements IndexChangeListener {
graphIndexClient.applySearchWeight(Constants.VERTEX_INDEX, indexFieldName2SearchWeightMap);
graphIndexClient.applySuggestionFields(Constants.VERTEX_INDEX, getIndexFieldNamesForSuggestions(indexFieldName2SearchWeightMap));
+ graphIndexClient.applyKeywordSubfieldFields(Constants.VERTEX_INDEX, buildKeywordSubfieldIndexFields());
} catch (AtlasException e) {
LOG.error("Error encountered in handling type system change notification.", e);
@@ -127,6 +132,60 @@ public class SolrIndexHelper implements IndexChangeListener {
return ret;
}
+ private Set<String> buildKeywordSubfieldIndexFields() {
+ Set<String> fields = new LinkedHashSet<>();
+
+ addKeywordSubfieldIfPresent(fields, STATE_PROPERTY_KEY);
+ addKeywordSubfieldIfPresent(fields, ENTITY_TYPE_PROPERTY_KEY);
+ addKeywordSubfieldIfPresent(fields, LABELS_PROPERTY_KEY);
+ addKeywordSubfieldIfPresent(fields, CLASSIFICATION_TEXT_KEY);
+
+ for (AtlasEntityType entityType : typeRegistry.getAllEntityTypes()) {
+ if (entityType.isInternalType()) {
+ continue;
+ }
+
+ processTypeForKeywordSubfields(fields, entityType);
+ }
+
+ for (AtlasBusinessMetadataType businessMetadataType : typeRegistry.getAllBusinessMetadataTypes()) {
+ processTypeForKeywordSubfields(fields, businessMetadataType);
+ }
+
+ return fields;
+ }
+
+ private void processTypeForKeywordSubfields(Set<String> fields, AtlasStructType structType) {
+ List<AtlasAttributeDef> attributes = structType.getStructDef().getAttributeDefs();
+
+ if (CollectionUtils.isNotEmpty(attributes)) {
+ for (AtlasAttributeDef attributeDef : attributes) {
+ processAttributeForKeywordSubfield(fields, structType.getAttribute(attributeDef.getName()));
+ }
+ }
+ }
+
+ private void addKeywordSubfieldIfPresent(Set<String> fields, String propertyKey) {
+ String indexFieldName = typeRegistry.getIndexFieldName(propertyKey);
+
+ if (StringUtils.isNotEmpty(indexFieldName)) {
+ fields.add(indexFieldName);
+ }
+ }
+
+ private void processAttributeForKeywordSubfield(Set<String> fields, AtlasAttribute attribute) {
+ if (attribute == null || !GraphBackedSearchIndexer.isStringAttribute(attribute)
+ || StringUtils.isEmpty(attribute.getIndexFieldName())) {
+ return;
+ }
+
+ boolean isStringField = AtlasAttributeDef.IndexType.STRING.equals(attribute.getIndexType());
+
+ if (GraphBackedSearchIndexer.needsKeywordSubfield(attribute.getVertexPropertyName(), isStringField, attribute.getSearchWeight())) {
+ fields.add(attribute.getIndexFieldName());
+ }
+ }
+
private Map<String, Integer> geIndexFieldNamesWithSearchWeights() {
Map<String, Integer> ret = new HashMap<>();
diff --git a/test-tools/pom.xml b/test-tools/pom.xml
index 166385689..66223df2a 100644
--- a/test-tools/pom.xml
+++ b/test-tools/pom.xml
@@ -90,6 +90,19 @@
<version>${zookeeper.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-intg</artifactId>