You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by pi...@apache.org on 2026/01/27 06:46:38 UTC
(atlas) 01/01: ATLAS-5195: Support to run ITs against postgres backend
This is an automated email from the ASF dual-hosted git repository.
pinal pushed a commit to branch ATLAS-5195
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 274ce06ad3f6b351dfb43b762afb0fc66bf410b1
Author: Pinal Shah <pi...@cloudera.com>
AuthorDate: Tue Jan 27 12:16:12 2026 +0530
ATLAS-5195: Support to run ITs against postgres backend
---
build-tools/it/it-rdbms-start-postgres.sh | 48 ++++++++
build-tools/it/it-rdbms-stop-postgres.sh | 25 ++++
build-tools/it/it-rdbms-wait-atlas-active.sh | 35 ++++++
dev-support/atlas-docker/scripts/atlas-build.sh | 2 +
pom.xml | 28 +++++
webapp/pom.xml | 100 +++++++++++++++
.../resources-rdbms/atlas-application.properties | 135 +++++++++++++++++++++
7 files changed, 373 insertions(+)
diff --git a/build-tools/it/it-rdbms-start-postgres.sh b/build-tools/it/it-rdbms-start-postgres.sh
new file mode 100644
index 000000000..dafb1c3c7
--- /dev/null
+++ b/build-tools/it/it-rdbms-start-postgres.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euo pipefail
+
+CONTAINER_NAME="${ATLAS_IT_RDBMS_CONTAINER_NAME:-atlas-it-postgres}"
+IMAGE="${ATLAS_IT_RDBMS_IMAGE:-postgres:13.21}"
+HOST_PORT="${ATLAS_IT_RDBMS_HOSTPORT:-15432}"
+USERNAME="${ATLAS_IT_RDBMS_USERNAME:-atlas}"
+PASSWORD="${ATLAS_IT_RDBMS_PASSWORD:-atlasR0cks!}"
+DB_NAME="${ATLAS_IT_RDBMS_DBNAME:-atlas}"
+MAX_CONNECTIONS="${ATLAS_IT_RDBMS_MAX_CONNECTIONS:-300}"
+
+docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true
+
+docker run -d --name "${CONTAINER_NAME}" \
+ -e "POSTGRES_DB=${DB_NAME}" \
+ -e "POSTGRES_USER=${USERNAME}" \
+ -e "POSTGRES_PASSWORD=${PASSWORD}" \
+ -p "${HOST_PORT}:5432" \
+ "${IMAGE}" -c "max_connections=${MAX_CONNECTIONS}" >/dev/null
+
+for i in $(seq 1 60); do
+ docker exec "${CONTAINER_NAME}" pg_isready -U "${USERNAME}" -d "${DB_NAME}" >/dev/null 2>&1 && exit 0
+ sleep 1
+done
+
+echo "Postgres container did not become ready in time" >&2
+docker logs "${CONTAINER_NAME}" || true
+exit 1
+
+
diff --git a/build-tools/it/it-rdbms-stop-postgres.sh b/build-tools/it/it-rdbms-stop-postgres.sh
new file mode 100644
index 000000000..8e70992fa
--- /dev/null
+++ b/build-tools/it/it-rdbms-stop-postgres.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euo pipefail
+
+CONTAINER_NAME="${ATLAS_IT_RDBMS_CONTAINER_NAME:-atlas-it-postgres}"
+docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true
+
+
diff --git a/build-tools/it/it-rdbms-wait-atlas-active.sh b/build-tools/it/it-rdbms-wait-atlas-active.sh
new file mode 100644
index 000000000..676a60e40
--- /dev/null
+++ b/build-tools/it/it-rdbms-wait-atlas-active.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euo pipefail
+
+STATUS_URL="${ATLAS_IT_STATUS_URL:-http://localhost:31000/api/atlas/admin/status}"
+MAX_SECONDS="${ATLAS_IT_WAIT_ACTIVE_SECONDS:-180}"
+
+for i in $(seq 1 "${MAX_SECONDS}"); do
+ out="$(curl -fsS "${STATUS_URL}" 2>/dev/null || true)"
+ echo "${out}" | grep -q ACTIVE && exit 0
+ sleep 1
+done
+
+echo "Atlas did not become ACTIVE in time" >&2
+curl -v "${STATUS_URL}" || true
+exit 1
+
+
diff --git a/dev-support/atlas-docker/scripts/atlas-build.sh b/dev-support/atlas-docker/scripts/atlas-build.sh
index ff60538e8..bb21f5685 100755
--- a/dev-support/atlas-docker/scripts/atlas-build.sh
+++ b/dev-support/atlas-docker/scripts/atlas-build.sh
@@ -87,6 +87,8 @@ else
fi
mvn ${ARG_PROFILES} ${ARG_SKIPTESTS} -DskipDocs clean package
+mvn -Pit-rdbms -DskipUTs=true -DskipTests=false -DskipITs=false verify
+mvn -Pdist -DskipUTs=true -DskipTests=false -DskipITs=false verify
mv -f distro/target/apache-atlas-${ATLAS_VERSION}-server.tar.gz /home/atlas/dist/
mv -f distro/target/apache-atlas-${ATLAS_VERSION}-hive-hook.tar.gz /home/atlas/dist/
diff --git a/pom.xml b/pom.xml
index 9e0a0999a..8f586846d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2396,5 +2396,33 @@
</properties>
</profile>
+ <profile>
+ <id>it-rdbms</id>
+ <properties>
+ <!-- Enunciate uses javac internals and fails on newer JDKs without module exports; not needed for IT runs -->
+ <skipEnunciate>true</skipEnunciate>
+ <skipITs>false</skipITs>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>integration-test</id>
+ <configuration>
+ <includes>
+ <include>**/*IT.java</include>
+ </includes>
+ <excludes combine.self="override" />
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
</profiles>
</project>
diff --git a/webapp/pom.xml b/webapp/pom.xml
index c314e5857..8bb0f1387 100755
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -866,6 +866,106 @@
</properties>
</profile>
+ <!--
+ Profile for running Postgres/RDBMS-backed integration tests.
+ This profile overlays an alternate atlas-application.properties onto target/test-classes
+ (atlas.conf is already set to ${project.build.directory}/test-classes for Jetty ITs).
+ -->
+ <profile>
+ <id>it-rdbms</id>
+ <properties>
+ <it.rdbms.containerName>atlas-it-postgres</it.rdbms.containerName>
+ <it.rdbms.dbName>atlas</it.rdbms.dbName>
+ <it.rdbms.docker.skip>false</it.rdbms.docker.skip>
+ <it.rdbms.hostPort>15432</it.rdbms.hostPort>
+ <it.rdbms.image>postgres:13.21</it.rdbms.image>
+ <it.rdbms.jdbcUrl>jdbc:postgresql://localhost:${it.rdbms.hostPort}/atlas</it.rdbms.jdbcUrl>
+ <it.rdbms.maxConnections>300</it.rdbms.maxConnections>
+ <it.rdbms.password>itpwd</it.rdbms.password>
+ <it.rdbms.scriptsDir>${projectBaseDir}/build-tools/it</it.rdbms.scriptsDir>
+ <it.rdbms.username>atlas</it.rdbms.username>
+ <it.rdbms.waitActiveSeconds>180</it.rdbms.waitActiveSeconds>
+ </properties>
+ <build>
+ <testResources>
+ <testResource>
+ <filtering>true</filtering>
+ <directory>src/test/resources</directory>
+ </testResource>
+ <testResource>
+ <filtering>true</filtering>
+ <directory>src/test/resources-rdbms</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>3.5.0</version>
+ <executions>
+ <execution>
+ <id>it-rdbms-start-postgres</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>initialize</phase>
+ <configuration>
+ <skip>${it.rdbms.docker.skip}</skip>
+ <executable>bash</executable>
+ <environmentVariables>
+ <ATLAS_IT_RDBMS_CONTAINER_NAME>${it.rdbms.containerName}</ATLAS_IT_RDBMS_CONTAINER_NAME>
+ <ATLAS_IT_RDBMS_IMAGE>${it.rdbms.image}</ATLAS_IT_RDBMS_IMAGE>
+ <ATLAS_IT_RDBMS_HOSTPORT>${it.rdbms.hostPort}</ATLAS_IT_RDBMS_HOSTPORT>
+ <ATLAS_IT_RDBMS_USERNAME>${it.rdbms.username}</ATLAS_IT_RDBMS_USERNAME>
+ <ATLAS_IT_RDBMS_PASSWORD>${it.rdbms.password}</ATLAS_IT_RDBMS_PASSWORD>
+ <ATLAS_IT_RDBMS_DBNAME>${it.rdbms.dbName}</ATLAS_IT_RDBMS_DBNAME>
+ <ATLAS_IT_RDBMS_MAX_CONNECTIONS>${it.rdbms.maxConnections}</ATLAS_IT_RDBMS_MAX_CONNECTIONS>
+ </environmentVariables>
+ <arguments>
+ <argument>${it.rdbms.scriptsDir}/it-rdbms-start-postgres.sh</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>it-rdbms-wait-atlas-active</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>pre-integration-test</phase>
+ <configuration>
+ <executable>bash</executable>
+ <environmentVariables>
+ <ATLAS_IT_STATUS_URL>http://localhost:31000/api/atlas/admin/status</ATLAS_IT_STATUS_URL>
+ <ATLAS_IT_WAIT_ACTIVE_SECONDS>${it.rdbms.waitActiveSeconds}</ATLAS_IT_WAIT_ACTIVE_SECONDS>
+ </environmentVariables>
+ <arguments>
+ <argument>${it.rdbms.scriptsDir}/it-rdbms-wait-atlas-active.sh</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>it-rdbms-stop-postgres</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>post-integration-test</phase>
+ <configuration>
+ <skip>${it.rdbms.docker.skip}</skip>
+ <executable>bash</executable>
+ <environmentVariables>
+ <ATLAS_IT_RDBMS_CONTAINER_NAME>${it.rdbms.containerName}</ATLAS_IT_RDBMS_CONTAINER_NAME>
+ </environmentVariables>
+ <arguments>
+ <argument>${it.rdbms.scriptsDir}/it-rdbms-stop-postgres.sh</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<profile>
<id>Windows</id>
<activation>
diff --git a/webapp/src/test/resources-rdbms/atlas-application.properties b/webapp/src/test/resources-rdbms/atlas-application.properties
new file mode 100644
index 000000000..6f07c2d1c
--- /dev/null
+++ b/webapp/src/test/resources-rdbms/atlas-application.properties
@@ -0,0 +1,135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+######### Atlas Server Configs #########
+atlas.rest.address=http://localhost:31000
+
+######### Graph Database Configs #########
+atlas.graph.index.search.solr.wait-searcher=true
+
+# Graph database implementation. Value inserted by maven.
+atlas.graphdb.backend=org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase
+
+#
+# RDBMS (Postgres) backend for JanusGraph
+#
+atlas.graph.storage.backend=rdbms
+
+# JDBC/Hikari settings (override these in CI if needed)
+atlas.graph.storage.rdbms.jpa.hikari.driverClassName=org.postgresql.Driver
+atlas.graph.storage.rdbms.jpa.hikari.jdbcUrl=${it.rdbms.jdbcUrl}
+atlas.graph.storage.rdbms.jpa.hikari.username=${it.rdbms.username}
+atlas.graph.storage.rdbms.jpa.hikari.password=${it.rdbms.password}
+atlas.graph.storage.rdbms.jpa.hikari.maximumPoolSize=40
+atlas.graph.storage.rdbms.jpa.hikari.minimumIdle=5
+atlas.graph.storage.rdbms.jpa.hikari.connectionTestQuery=select 1
+atlas.graph.storage.rdbms.jpa.hikari.connectionTimeout=60000
+atlas.graph.storage.rdbms.jpa.hikari.idleTimeout=300000
+atlas.graph.storage.rdbms.jpa.hikari.maxLifetime=1800000
+atlas.graph.storage.rdbms.jpa.hikari.leakDetectionThreshold=60000
+
+# JPA schema init for JanusGraph RDBMS store
+atlas.graph.storage.rdbms.jpa.javax.persistence.jdbc.dialect=org.eclipse.persistence.platform.database.PostgreSQLPlatform
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.database.action=create
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-database-schemas=true
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-source=script
+atlas.graph.storage.rdbms.jpa.javax.persistence.schema-generation.create-script-source=META-INF/postgres/create_schema.sql
+
+# Entity audit repository for ITs (avoid HBase audit dependency)
+# NOTE: RdbmsBasedAuditRepository currently doesn't implement V1 audit event writes used by ITs.
+atlas.EntityAuditRepository.impl=org.apache.atlas.repository.audit.InMemoryEntityAuditRepository
+
+# Graph Search Index Backend
+atlas.graph.index.search.backend=solr
+
+# Solr cloud mode properties
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.zookeeper-url=${solr.zk.address}
+atlas.graph.index.search.solr.embedded=true
+atlas.graph.index.search.max-result-set-size=150
+
+######### Notification Configs #########
+atlas.notification.embedded=true
+atlas.notification.entity.version=v1
+
+atlas.kafka.zookeeper.connect=localhost:19026
+atlas.kafka.bootstrap.servers=localhost:19027
+atlas.kafka.data=${sys:atlas.data}/kafka
+atlas.kafka.zookeeper.session.timeout.ms=4000
+atlas.kafka.zookeeper.sync.time.ms=20
+atlas.kafka.consumer.timeout.ms=4000
+atlas.kafka.auto.commit.interval.ms=100
+atlas.kafka.hook.group.id=atlas
+atlas.kafka.entities.group.id=atlas_entities
+
+atlas.kafka.enable.auto.commit=false
+atlas.kafka.auto.offset.reset=earliest
+atlas.kafka.session.timeout.ms=30000
+atlas.kafka.offsets.topic.replication.factor=1
+
+######### Entity Audit Configs #########
+atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS
+atlas.audit.zookeeper.session.timeout.ms=1000
+atlas.audit.hbase.zookeeper.quorum=localhost
+atlas.audit.hbase.zookeeper.property.clientPort=19026
+
+######### Security Properties #########
+
+# SSL config
+atlas.enableTLS=false
+atlas.server.https.port=31443
+
+######### Security Properties #########
+
+hbase.security.authentication=simple
+
+atlas.hook.falcon.synchronous=true
+
+######### JAAS Configuration ########
+
+atlas.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+atlas.jaas.KafkaClient.loginModuleControlFlag = required
+atlas.jaas.KafkaClient.option.useKeyTab = true
+atlas.jaas.KafkaClient.option.storeKey = true
+atlas.jaas.KafkaClient.option.serviceName = kafka
+atlas.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/atlas.service.keytab
+atlas.jaas.KafkaClient.option.principal = atlas/_HOST@EXAMPLE.COM
+
+######### High Availability Configuration ########
+atlas.server.ha.enabled=false
+
+######### Atlas Authorization #########
+atlas.authorizer.impl=none
+
+######### Atlas Authentication #########
+atlas.authentication.method.file=true
+atlas.authentication.method.ldap.type=none
+atlas.authentication.method.kerberos=false
+
+######### Gremlin Search Configuration #########
+# Set to false to disable gremlin search.
+atlas.search.gremlin.enable=true
+
+######### Configure use of Tasks #########
+atlas.tasks.enabled=false
+atlas.debug.metrics.enabled=true
+
+######### Configure on-demand lineage #########
+atlas.lineage.on.demand.enabled=true
+
+