You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by yu...@apache.org on 2024/09/16 20:07:30 UTC

(polaris) branch main updated: Update PolarisEclipseLinkStore to work with more databases (#247)

This is an automated email from the ASF dual-hosted git repository.

yufei pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git


The following commit(s) were added to refs/heads/main by this push:
     new 537eee65 Update PolarisEclipseLinkStore to work with more databases (#247)
537eee65 is described below

commit 537eee6578afbea8dede47958edf0b6168326252
Author: Eric Maynard <er...@snowflake.com>
AuthorDate: Mon Sep 16 16:07:26 2024 -0400

    Update PolarisEclipseLinkStore to work with more databases (#247)
---
 .../PolarisEclipseLinkMetaStoreSessionImpl.java    |   5 +-
 .../impl/eclipselink/PolarisEclipseLinkStore.java  |  44 +++++++-
 .../impl/eclipselink/PolarisSequenceUtil.java      | 120 +++++++++++++++++++++
 .../core/persistence/models/ModelEntity.java       |   4 +-
 .../core/persistence/models/ModelSequenceId.java   |  17 +--
 5 files changed, 178 insertions(+), 12 deletions(-)

diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java
index da6d9005..e7c813a6 100644
--- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java
+++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java
@@ -117,6 +117,9 @@ public class PolarisEclipseLinkMetaStoreSessionImpl implements PolarisMetaStoreS
 
     // init store
     this.store = store;
+    try (EntityManager session = emf.createEntityManager()) {
+      this.store.initialize(session);
+    }
     this.storageIntegrationProvider = storageIntegrationProvider;
   }
 
@@ -281,8 +284,8 @@ public class PolarisEclipseLinkMetaStoreSessionImpl implements PolarisMetaStoreS
           LOGGER.debug("transaction committed");
         }
       } catch (Exception e) {
+        LOGGER.debug("Rolling back transaction due to an error", e);
         tr.rollback();
-        LOGGER.debug("transaction rolled back", e);
 
         if (e instanceof OptimisticLockException
             || e.getCause() instanceof OptimisticLockException) {
diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java
index 02461fa4..2958f7cc 100644
--- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java
+++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java
@@ -22,6 +22,7 @@ import jakarta.persistence.EntityManager;
 import jakarta.persistence.TypedQuery;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 import org.apache.polaris.core.PolarisDiagnostics;
 import org.apache.polaris.core.entity.PolarisBaseEntity;
@@ -53,6 +54,9 @@ public class PolarisEclipseLinkStore {
   // diagnostic services
   private final PolarisDiagnostics diagnosticServices;
 
+  // Used to track when the store is initialized
+  private final AtomicBoolean initialized = new AtomicBoolean(false);
+
   /**
    * Constructor, allocate everything at once
    *
@@ -62,14 +66,22 @@ public class PolarisEclipseLinkStore {
     this.diagnosticServices = diagnostics;
   }
 
+  /** Initialize the store. This should be called before other methods. */
+  public void initialize(EntityManager session) {
+    PolarisSequenceUtil.initialize(session);
+    initialized.set(true);
+  }
+
   long getNextSequence(EntityManager session) {
     diagnosticServices.check(session != null, "session_is_null");
-    // implement with a sequence table POLARIS_SEQUENCE
-    return (long) session.createNativeQuery("SELECT NEXTVAL('POLARIS_SEQ')").getSingleResult();
+    checkInitialized();
+
+    return PolarisSequenceUtil.getNewId(session);
   }
 
   void writeToEntities(EntityManager session, PolarisBaseEntity entity) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     ModelEntity model = lookupEntity(session, entity.getCatalogId(), entity.getId());
     if (model != null) {
@@ -84,6 +96,7 @@ public class PolarisEclipseLinkStore {
 
   void writeToEntitiesActive(EntityManager session, PolarisBaseEntity entity) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     ModelEntityActive model = lookupEntityActive(session, new PolarisEntitiesActiveKey(entity));
     if (model == null) {
@@ -93,6 +106,7 @@ public class PolarisEclipseLinkStore {
 
   void writeToEntitiesDropped(EntityManager session, PolarisBaseEntity entity) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     ModelEntityDropped entityDropped =
         lookupEntityDropped(session, entity.getCatalogId(), entity.getId());
@@ -103,6 +117,7 @@ public class PolarisEclipseLinkStore {
 
   void writeToEntitiesChangeTracking(EntityManager session, PolarisBaseEntity entity) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     // Update the existing change tracking if a record with the same ids exists; otherwise, persist
     // a new one
@@ -119,12 +134,14 @@ public class PolarisEclipseLinkStore {
 
   void writeToGrantRecords(EntityManager session, PolarisGrantRecord grantRec) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     session.persist(ModelGrantRecord.fromGrantRecord(grantRec));
   }
 
   void deleteFromEntities(EntityManager session, long catalogId, long entityId) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     ModelEntity model = lookupEntity(session, catalogId, entityId);
     diagnosticServices.check(model != null, "entity_not_found");
@@ -134,6 +151,7 @@ public class PolarisEclipseLinkStore {
 
   void deleteFromEntitiesActive(EntityManager session, PolarisEntitiesActiveKey key) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     ModelEntityActive entity = lookupEntityActive(session, key);
     diagnosticServices.check(entity != null, "active_entity_not_found");
@@ -142,6 +160,7 @@ public class PolarisEclipseLinkStore {
 
   void deleteFromEntitiesDropped(EntityManager session, long catalogId, long entityId) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     ModelEntityDropped entity = lookupEntityDropped(session, catalogId, entityId);
     diagnosticServices.check(entity != null, "dropped_entity_not_found");
@@ -151,6 +170,7 @@ public class PolarisEclipseLinkStore {
 
   void deleteFromEntitiesChangeTracking(EntityManager session, PolarisEntityCore entity) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     ModelEntityChangeTracking entityChangeTracking =
         lookupEntityChangeTracking(session, entity.getCatalogId(), entity.getId());
@@ -161,6 +181,7 @@ public class PolarisEclipseLinkStore {
 
   void deleteFromGrantRecords(EntityManager session, PolarisGrantRecord grantRec) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     ModelGrantRecord lookupGrantRecord =
         lookupGrantRecord(
@@ -178,6 +199,7 @@ public class PolarisEclipseLinkStore {
 
   void deleteAllEntityGrantRecords(EntityManager session, PolarisEntityCore entity) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     // Delete grant records from grantRecords tables
     lookupAllGrantRecordsOnSecurable(session, entity.getCatalogId(), entity.getId())
@@ -190,6 +212,7 @@ public class PolarisEclipseLinkStore {
 
   void deleteAll(EntityManager session) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     session.createQuery("DELETE from ModelEntity").executeUpdate();
     session.createQuery("DELETE from ModelEntityActive").executeUpdate();
@@ -203,6 +226,7 @@ public class PolarisEclipseLinkStore {
 
   ModelEntity lookupEntity(EntityManager session, long catalogId, long entityId) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     return session
         .createQuery(
@@ -218,6 +242,7 @@ public class PolarisEclipseLinkStore {
   @SuppressWarnings("unchecked")
   List<ModelEntity> lookupEntities(EntityManager session, List<PolarisEntityId> entityIds) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     if (entityIds == null || entityIds.isEmpty()) return new ArrayList<>();
 
@@ -234,6 +259,7 @@ public class PolarisEclipseLinkStore {
   ModelEntityActive lookupEntityActive(
       EntityManager session, PolarisEntitiesActiveKey entityActiveKey) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     return session
         .createQuery(
@@ -254,6 +280,7 @@ public class PolarisEclipseLinkStore {
       long parentId,
       @Nullable PolarisEntityType entityType) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     String hql =
         "SELECT COUNT(m) from ModelEntityActive m where m.catalogId=:catalogId and m.parentId=:parentId";
@@ -276,6 +303,7 @@ public class PolarisEclipseLinkStore {
   List<ModelEntity> lookupFullEntitiesActive(
       EntityManager session, long catalogId, long parentId, @NotNull PolarisEntityType entityType) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     // Currently check against ENTITIES not joining with ENTITIES_ACTIVE
     String hql =
@@ -293,6 +321,7 @@ public class PolarisEclipseLinkStore {
 
   ModelEntityDropped lookupEntityDropped(EntityManager session, long catalogId, long entityId) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     return session
         .createQuery(
@@ -308,6 +337,7 @@ public class PolarisEclipseLinkStore {
   ModelEntityChangeTracking lookupEntityChangeTracking(
       EntityManager session, long catalogId, long entityId) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     return session
         .createQuery(
@@ -328,6 +358,7 @@ public class PolarisEclipseLinkStore {
       long granteeId,
       int privilegeCode) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     return session
         .createQuery(
@@ -350,6 +381,7 @@ public class PolarisEclipseLinkStore {
   List<ModelGrantRecord> lookupAllGrantRecordsOnSecurable(
       EntityManager session, long securableCatalogId, long securableId) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     return session
         .createQuery(
@@ -365,6 +397,7 @@ public class PolarisEclipseLinkStore {
   List<ModelGrantRecord> lookupGrantRecordsOnGrantee(
       EntityManager session, long granteeCatalogId, long granteeId) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     return session
         .createQuery(
@@ -379,6 +412,7 @@ public class PolarisEclipseLinkStore {
 
   ModelPrincipalSecrets lookupPrincipalSecrets(EntityManager session, String clientId) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     return session
         .createQuery(
@@ -392,6 +426,7 @@ public class PolarisEclipseLinkStore {
 
   void writePrincipalSecrets(EntityManager session, PolarisPrincipalSecrets principalSecrets) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     ModelPrincipalSecrets modelPrincipalSecrets =
         lookupPrincipalSecrets(session, principalSecrets.getPrincipalClientId());
@@ -406,10 +441,15 @@ public class PolarisEclipseLinkStore {
 
   void deletePrincipalSecrets(EntityManager session, String clientId) {
     diagnosticServices.check(session != null, "session_is_null");
+    checkInitialized();
 
     ModelPrincipalSecrets modelPrincipalSecrets = lookupPrincipalSecrets(session, clientId);
     diagnosticServices.check(modelPrincipalSecrets != null, "principal_secretes_not_found");
 
     session.remove(modelPrincipalSecrets);
   }
+
+  private void checkInitialized() {
+    diagnosticServices.check(this.initialized.get(), "store_not_initialized");
+  }
 }
diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisSequenceUtil.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisSequenceUtil.java
new file mode 100644
index 00000000..609eec52
--- /dev/null
+++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisSequenceUtil.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.extension.persistence.impl.eclipselink;
+
+import jakarta.persistence.*;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.polaris.core.persistence.models.ModelSequenceId;
+import org.eclipse.persistence.internal.jpa.EntityManagerImpl;
+import org.eclipse.persistence.platform.database.DatabasePlatform;
+import org.eclipse.persistence.platform.database.PostgreSQLPlatform;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Used to generate sequence IDs for Polaris entities. If the legacy `POLARIS_SEQ` generator is
+ * available it will be used then cleaned up. In all other cases the `POLARIS_SEQUENCE` table is
+ * used directly.
+ */
+class PolarisSequenceUtil {
+  private static final Logger LOGGER = LoggerFactory.getLogger(PolarisSequenceUtil.class);
+
+  private static final AtomicBoolean initialized = new AtomicBoolean(false);
+
+  private PolarisSequenceUtil() {}
+
+  /* If `initialize` was never called, throw an exception */
+  private static void throwIfNotInitialized() {
+    if (!initialized.get()) {
+      throw new IllegalStateException("Sequence util has not been initialized");
+    }
+  }
+
+  /* Get the database platform associated with the `EntityManager` */
+  private static DatabasePlatform getDatabasePlatform(EntityManager session) {
+    EntityManagerImpl entityManagerImpl = session.unwrap(EntityManagerImpl.class);
+    return entityManagerImpl.getDatabaseSession().getPlatform();
+  }
+
+  private static void removeSequence(EntityManager session) {
+    LOGGER.info("Renaming legacy sequence `POLARIS_SEQ` to `POLARIS_SEQ_UNUSED`");
+    String renameSequenceQuery = "ALTER SEQUENCE POLARIS_SEQ RENAME TO POLARIS_SEQ_UNUSED";
+    session.createNativeQuery(renameSequenceQuery).executeUpdate();
+  }
+
+  /**
+   * Prepare the `PolarisSequenceUtil` to generate IDs. This may run a failing query, so it should
+   * be called for the first time outside the context of a transaction. This method should be called
+   * before any other methods. TODO: after a sufficient this can eventually be removed or altered
+   */
+  public static void initialize(EntityManager session) {
+    // Trigger cleanup of the POLARIS_SEQ if it is present
+    DatabasePlatform databasePlatform = getDatabasePlatform(session);
+    if (!initialized.get()) {
+      if (databasePlatform instanceof PostgreSQLPlatform) {
+        Optional<Long> result = Optional.empty();
+        LOGGER.info("Checking if the sequence POLARIS_SEQ exists");
+        String checkSequenceQuery =
+            "SELECT COUNT(*) FROM information_schema.sequences WHERE sequence_name IN "
+                + "('polaris_seq', 'POLARIS_SEQ')";
+        int sequenceExists =
+            ((Number) session.createNativeQuery(checkSequenceQuery).getSingleResult()).intValue();
+
+        if (sequenceExists > 0) {
+          LOGGER.info("POLARIS_SEQ exists, calling NEXTVAL");
+          long queryResult =
+              (long) session.createNativeQuery("SELECT NEXTVAL('POLARIS_SEQ')").getSingleResult();
+          result = Optional.of(queryResult);
+        } else {
+          LOGGER.info("POLARIS_SEQ does not exist, skipping NEXTVAL");
+        }
+        result.ifPresent(
+            r -> {
+              ModelSequenceId modelSequenceId = new ModelSequenceId();
+              modelSequenceId.setId(r);
+
+              // Persist the new ID:
+              session.persist(modelSequenceId);
+              session.flush();
+
+              // Clean the sequence:
+              removeSequence(session);
+            });
+      }
+      initialized.set(true);
+    }
+  }
+
+  /**
+   * Generates a new ID from `POLARIS_SEQUENCE` or `POLARIS_SEQ` depending on availability.
+   * `initialize` should be called before this method.
+   */
+  public static Long getNewId(EntityManager session) {
+    throwIfNotInitialized();
+
+    ModelSequenceId modelSequenceId = new ModelSequenceId();
+
+    // Persist the new ID:
+    session.persist(modelSequenceId);
+    session.flush();
+
+    return modelSequenceId.getId();
+  }
+}
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/models/ModelEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/models/ModelEntity.java
index 0f2994f6..5a594b04 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/models/ModelEntity.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/models/ModelEntity.java
@@ -73,11 +73,11 @@ public class ModelEntity {
   private long lastUpdateTimestamp;
 
   // properties, serialized as a JSON string
-  @Column(length = 65535)
+  @Column(columnDefinition = "TEXT")
   private String properties;
 
   // internal properties, serialized as a JSON string
-  @Column(length = 65535)
+  @Column(columnDefinition = "TEXT")
   private String internalProperties;
 
   // current version for that entity, will be monotonically incremented
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/models/ModelSequenceId.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/models/ModelSequenceId.java
index 8a5e3b7b..04fddfa7 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/models/ModelSequenceId.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/models/ModelSequenceId.java
@@ -22,18 +22,21 @@ import jakarta.persistence.Entity;
 import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
-import jakarta.persistence.SequenceGenerator;
 import jakarta.persistence.Table;
 
+/** Used to manage unique IDs in Polaris */
 @Entity
 @Table(name = "POLARIS_SEQUENCE")
 public class ModelSequenceId {
   @Id
-  @SequenceGenerator(
-      name = "sequenceGen",
-      sequenceName = "POLARIS_SEQ",
-      initialValue = 1000,
-      allocationSize = 25)
-  @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "sequenceGen")
+  @GeneratedValue(strategy = GenerationType.AUTO)
   private Long id;
+
+  public void setId(Long id) {
+    this.id = id;
+  }
+
+  public Long getId() {
+    return id;
+  }
 }