You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by di...@apache.org on 2026/07/15 15:54:23 UTC
(polaris) branch main updated: Avoid fetching full entity rows in lookupEntityVersions (#5038)
This is an automated email from the ASF dual-hosted git repository.
dimas-b pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 970cffbc40 Avoid fetching full entity rows in lookupEntityVersions (#5038)
970cffbc40 is described below
commit 970cffbc40d85662b96b65c1f024bbbf55d45be3
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Wed Jul 15 21:24:18 2026 +0530
Avoid fetching full entity rows in lookupEntityVersions (#5038)
---
.../relational/jdbc/JdbcBasePersistenceImpl.java | 36 ++++-----
.../relational/jdbc/QueryGenerator.java | 23 +++++-
.../jdbc/models/EntityVersionConverter.java | 48 ++++++++++++
.../relational/jdbc/models/ModelEntity.java | 4 +
...yTest.java => JdbcBasePersistenceImplTest.java} | 89 +++++++++++++++++++++-
.../relational/jdbc/QueryGeneratorTest.java | 35 +++++++++
6 files changed, 212 insertions(+), 23 deletions(-)
diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java
index cc711cc646..1dd254b687 100644
--- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java
+++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java
@@ -29,7 +29,6 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
@@ -70,6 +69,7 @@ import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
import org.apache.polaris.core.storage.PolarisStorageIntegration;
import org.apache.polaris.core.storage.StorageLocation;
import org.apache.polaris.persistence.relational.jdbc.models.EntityNameLookupRecordConverter;
+import org.apache.polaris.persistence.relational.jdbc.models.EntityVersionConverter;
import org.apache.polaris.persistence.relational.jdbc.models.ModelCommitMetricsReport;
import org.apache.polaris.persistence.relational.jdbc.models.ModelEntity;
import org.apache.polaris.persistence.relational.jdbc.models.ModelEvent;
@@ -494,23 +494,23 @@ public class JdbcBasePersistenceImpl
@Override
public List<PolarisChangeTrackingVersions> lookupEntityVersions(
@NonNull PolarisCallContext callCtx, List<PolarisEntityId> entityIds) {
- Map<PolarisEntityId, ModelEntity> idToEntityMap =
- lookupEntities(callCtx, entityIds).stream()
- .filter(Objects::nonNull)
- .collect(
- Collectors.toMap(
- entry -> new PolarisEntityId(entry.getCatalogId(), entry.getId()),
- entry -> ModelEntity.fromEntity(entry, schemaVersion)));
- return entityIds.stream()
- .map(
- entityId -> {
- ModelEntity entity = idToEntityMap.getOrDefault(entityId, null);
- return entity == null
- ? null
- : new PolarisChangeTrackingVersions(
- entity.getEntityVersion(), entity.getGrantRecordsVersion());
- })
- .collect(Collectors.toList());
+ if (entityIds == null || entityIds.isEmpty()) {
+ return new ArrayList<>();
+ }
+ PreparedQuery query =
+ QueryGenerator.generateSelectQueryWithEntityIdsVersionOnly(realmId, entityIds);
+ Map<PolarisEntityId, PolarisChangeTrackingVersions> idToVersions;
+ try {
+ idToVersions =
+ datasourceOperations.executeSelect(query, new EntityVersionConverter()).stream()
+ .collect(
+ Collectors.toMap(
+ EntityVersionConverter.EntityVersionRow::entityId,
+ EntityVersionConverter.EntityVersionRow::versions));
+ } catch (SQLException e) {
+ throw new RuntimeException("Failed to retrieve entity versions: " + e.getMessage(), e);
+ }
+ return entityIds.stream().map(idToVersions::get).collect(Collectors.toList());
}
private PreparedQuery buildEntityQuery(
diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java
index 033835f644..d35773a81c 100644
--- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java
+++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java
@@ -120,6 +120,24 @@ public class QueryGenerator {
*/
public static PreparedQuery generateSelectQueryWithEntityIds(
@NonNull String realmId, int schemaVersion, @NonNull List<PolarisEntityId> entityIds) {
+ return generateSelectQueryWithEntityIds(
+ realmId, ModelEntity.getAllColumnNames(schemaVersion), entityIds);
+ }
+
+ /**
+ * Like {@link #generateSelectQueryWithEntityIds(String, int, List)} but selects only {@link
+ * ModelEntity#VERSION_COLUMNS}. Used by {@code lookupEntityVersions} to avoid fetching large JSON
+ * property blobs on the cache-validation hot path.
+ */
+ public static PreparedQuery generateSelectQueryWithEntityIdsVersionOnly(
+ @NonNull String realmId, @NonNull List<PolarisEntityId> entityIds) {
+ return generateSelectQueryWithEntityIds(realmId, ModelEntity.VERSION_COLUMNS, entityIds);
+ }
+
+ private static PreparedQuery generateSelectQueryWithEntityIds(
+ @NonNull String realmId,
+ @NonNull List<String> columns,
+ @NonNull List<PolarisEntityId> entityIds) {
if (entityIds.isEmpty()) {
throw new IllegalArgumentException("Empty entity ids");
}
@@ -132,10 +150,7 @@ public class QueryGenerator {
params.add(realmId);
String where = " WHERE (catalog_id, id) IN (" + placeholders + ") AND realm_id = ?";
return new PreparedQuery(
- generateSelectQuery(
- ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, where, null)
- .sql(),
- params);
+ generateSelectQuery(columns, ModelEntity.TABLE_NAME, where, null).sql(), params);
}
/**
diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityVersionConverter.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityVersionConverter.java
new file mode 100644
index 0000000000..63944befc1
--- /dev/null
+++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/EntityVersionConverter.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.persistence.relational.jdbc.models;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Map;
+import org.apache.polaris.core.entity.PolarisChangeTrackingVersions;
+import org.apache.polaris.core.entity.PolarisEntityId;
+import org.apache.polaris.persistence.relational.jdbc.DatabaseType;
+
+/** Read-only converter for {@link ModelEntity#VERSION_COLUMNS} projection queries. */
+public class EntityVersionConverter implements Converter<EntityVersionConverter.EntityVersionRow> {
+
+ public record EntityVersionRow(
+ PolarisEntityId entityId, PolarisChangeTrackingVersions versions) {}
+
+ @Override
+ public EntityVersionRow fromResultSet(ResultSet rs) throws SQLException {
+ return new EntityVersionRow(
+ new PolarisEntityId(rs.getLong("catalog_id"), rs.getLong("id")),
+ new PolarisChangeTrackingVersions(
+ rs.getInt("entity_version"), rs.getInt("grant_records_version")));
+ }
+
+ @Override
+ public Map<String, Object> toMap(DatabaseType databaseType) {
+ throw new UnsupportedOperationException(
+ "EntityVersionConverter is read-only and does not support toMap");
+ }
+}
diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java
index ef39529174..e8075db4c5 100644
--- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java
+++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java
@@ -83,6 +83,10 @@ public class ModelEntity implements Converter<PolarisBaseEntity> {
public static final List<String> ENTITY_LOOKUP_COLUMNS =
List.of("id", "catalog_id", "parent_id", "type_code", "name", "sub_type_code");
+ /** Minimal projection for version-check queries; avoids fetching large JSON property blobs. */
+ public static final List<String> VERSION_COLUMNS =
+ List.of("id", "catalog_id", "entity_version", "grant_records_version");
+
// the id of the catalog associated to that entity. use 0 if this entity is top-level
// like a catalog
private long catalogId;
diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImplTest.java
similarity index 60%
rename from persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java
rename to persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImplTest.java
index 6d01f2d9cd..e520368dc5 100644
--- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java
+++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImplTest.java
@@ -19,27 +19,36 @@
package org.apache.polaris.persistence.relational.jdbc;
import static org.apache.polaris.core.persistence.PrincipalSecretsGenerator.RANDOM_SECRETS;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
+import java.util.List;
import java.util.Optional;
import org.apache.polaris.core.PolarisCallContext;
import org.apache.polaris.core.PolarisDefaultDiagServiceImpl;
import org.apache.polaris.core.context.RealmContext;
+import org.apache.polaris.core.entity.PolarisBaseEntity;
+import org.apache.polaris.core.entity.PolarisChangeTrackingVersions;
+import org.apache.polaris.core.entity.PolarisEntityId;
+import org.apache.polaris.core.entity.PolarisEntitySubType;
+import org.apache.polaris.core.entity.PolarisEntityType;
import org.apache.polaris.core.entity.PolarisGrantRecord;
import org.h2.jdbcx.JdbcConnectionPool;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
-class JdbcGrantRecordsIdempotencyTest {
+class JdbcBasePersistenceImplTest {
private static final RealmContext REALM_CONTEXT = () -> "REALM";
@@ -130,6 +139,84 @@ class JdbcGrantRecordsIdempotencyTest {
.withCause(nonUniqueViolation);
}
+ @ParameterizedTest
+ @ValueSource(ints = {1, 2})
+ void lookupEntityVersionsPreservesOrderAndNullsForMissing(int schemaVersion)
+ throws SQLException, IOException {
+ JdbcConnectionPool dataSource =
+ JdbcConnectionPool.create(
+ "jdbc:h2:mem:lookup_versions_v"
+ + schemaVersion
+ + "_"
+ + System.nanoTime()
+ + ";DB_CLOSE_DELAY=-1",
+ "sa",
+ "");
+ DatasourceOperations real = new DatasourceOperations(dataSource, new TestJdbcConfiguration());
+ try (InputStream script = DatabaseType.H2.openInitScriptResource(schemaVersion)) {
+ real.executeScript(script);
+ }
+ // Spy so we can assert on the SQL actually issued, without blocking the real call.
+ DatasourceOperations spy = Mockito.spy(real);
+ doCallRealMethod().when(spy).executeSelect(any(), any());
+
+ JdbcBasePersistenceImpl impl =
+ new JdbcBasePersistenceImpl(
+ new PolarisDefaultDiagServiceImpl(),
+ spy,
+ RANDOM_SECRETS,
+ REALM_CONTEXT.getRealmIdentifier(),
+ schemaVersion);
+ PolarisCallContext callCtx = new PolarisCallContext(REALM_CONTEXT, impl);
+
+ PolarisBaseEntity e1 =
+ new PolarisBaseEntity.Builder()
+ .id(101L)
+ .catalogId(0L)
+ .parentId(0L)
+ .typeCode(PolarisEntityType.PRINCIPAL.getCode())
+ .subTypeCode(PolarisEntitySubType.NULL_SUBTYPE.getCode())
+ .name("e1")
+ .entityVersion(1)
+ .grantRecordsVersion(2)
+ .createTimestamp(System.currentTimeMillis())
+ .build();
+ PolarisBaseEntity e2 =
+ new PolarisBaseEntity.Builder()
+ .id(102L)
+ .catalogId(0L)
+ .parentId(0L)
+ .typeCode(PolarisEntityType.PRINCIPAL.getCode())
+ .subTypeCode(PolarisEntitySubType.NULL_SUBTYPE.getCode())
+ .name("e2")
+ .entityVersion(3)
+ .grantRecordsVersion(4)
+ .createTimestamp(System.currentTimeMillis())
+ .build();
+ impl.writeEntity(callCtx, e1, false, null);
+ impl.writeEntity(callCtx, e2, false, null);
+
+ PolarisEntityId id1 = new PolarisEntityId(0L, 101L);
+ PolarisEntityId missing = new PolarisEntityId(0L, 999L);
+ PolarisEntityId id2 = new PolarisEntityId(0L, 102L);
+ List<PolarisChangeTrackingVersions> result =
+ impl.lookupEntityVersions(callCtx, List.of(id2, missing, id1));
+
+ // Behavioural: order preserved, null for missing.
+ assertThat(result).hasSize(3);
+ assertThat(result.get(0).entityVersion()).isEqualTo(e2.getEntityVersion());
+ assertThat(result.get(0).grantRecordsVersion()).isEqualTo(e2.getGrantRecordsVersion());
+ assertThat(result.get(1)).isNull();
+ assertThat(result.get(2).entityVersion()).isEqualTo(e1.getEntityVersion());
+ assertThat(result.get(2).grantRecordsVersion()).isEqualTo(e1.getGrantRecordsVersion());
+
+ // Perf: the SQL that actually executed must not fetch the large JSON property columns.
+ ArgumentCaptor<QueryGenerator.PreparedQuery> captor =
+ ArgumentCaptor.forClass(QueryGenerator.PreparedQuery.class);
+ verify(spy).executeSelect(captor.capture(), any());
+ assertThat(captor.getValue().sql()).doesNotContain("properties");
+ }
+
private static final class TestJdbcConfiguration implements RelationalJdbcConfiguration {
@Override
public Optional<Integer> maxRetries() {
diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java
index 38a06f2b5f..04ae231a88 100644
--- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java
+++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java
@@ -99,6 +99,41 @@ public class QueryGeneratorTest {
() -> QueryGenerator.generateSelectQueryWithEntityIds(REALM_ID, 2, entityIds).sql());
}
+ @Test
+ void testGenerateSelectQueryWithEntityIdsVersionOnly_selectsOnlyVersionColumns() {
+ List<PolarisEntityId> entityIds = Collections.singletonList(new PolarisEntityId(123L, 1L));
+ String expectedQuery =
+ "SELECT id, catalog_id, entity_version, grant_records_version"
+ + " FROM POLARIS_SCHEMA.ENTITIES"
+ + " WHERE (catalog_id, id) IN ((?, ?)) AND realm_id = ?";
+ Assertions.assertThat(
+ QueryGenerator.generateSelectQueryWithEntityIdsVersionOnly(REALM_ID, entityIds).sql())
+ .isEqualTo(expectedQuery);
+ }
+
+ @Test
+ void testGenerateSelectQueryWithEntityIdsVersionOnly_multipleIds() {
+ List<PolarisEntityId> entityIds =
+ Arrays.asList(new PolarisEntityId(10L, 1L), new PolarisEntityId(20L, 2L));
+ String sql =
+ QueryGenerator.generateSelectQueryWithEntityIdsVersionOnly(REALM_ID, entityIds).sql();
+ // Must NOT contain large property columns on the hot path.
+ Assertions.assertThat(sql).doesNotContain("properties");
+ Assertions.assertThat(sql).doesNotContain("internal_properties");
+ Assertions.assertThat(sql).contains("entity_version");
+ Assertions.assertThat(sql).contains("grant_records_version");
+ Assertions.assertThat(sql).contains("(catalog_id, id) IN ((?, ?), (?, ?))");
+ }
+
+ @Test
+ void testGenerateSelectQueryWithEntityIdsVersionOnly_emptyList() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ QueryGenerator.generateSelectQueryWithEntityIdsVersionOnly(
+ REALM_ID, Collections.emptyList()));
+ }
+
@Test
void testGenerateInsertQuery_nonNullFields() {
ModelEntity entity = ModelEntity.builder().name("test").entityVersion(1).build();