You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vp...@apache.org on 2024/12/05 07:55:51 UTC

(ignite-3) branch main updated: IGNITE-23525 Prolong all leases in batch if at least one of them needs to be prolonged (#4798)

This is an automated email from the ASF dual-hosted git repository.

vpyatkov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git


The following commit(s) were added to refs/heads/main by this push:
     new c8b0f784bd IGNITE-23525 Prolong all leases in batch if at least one of them needs to be prolonged (#4798)
c8b0f784bd is described below

commit c8b0f784bd9463b9c0a9fedd1a83ee2d3dc91842
Author: Denis Chudov <mo...@gmail.com>
AuthorDate: Thu Dec 5 10:55:45 2024 +0300

    IGNITE-23525 Prolong all leases in batch if at least one of them needs to be prolonged (#4798)
---
 .../apache/ignite/internal/hlc/ClockService.java   |  2 +-
 .../internal/placementdriver/LeaseUpdater.java     | 81 +++++++++++--------
 .../internal/placementdriver/ActiveActorTest.java  |  3 -
 .../placementdriver/LeaseNegotiationTest.java      | 91 +++++++++++++++++++---
 4 files changed, 131 insertions(+), 46 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/hlc/ClockService.java b/modules/core/src/main/java/org/apache/ignite/internal/hlc/ClockService.java
index c173f8b221..b538ab51aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/hlc/ClockService.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/hlc/ClockService.java
@@ -79,7 +79,7 @@ public interface ClockService {
     long maxClockSkewMillis();
 
     /**
-     * Defines whether this timestamp is strictly before the given one, taking the clock skew into account.
+     * Defines whether the first timestamp is strictly before the another one, taking the clock skew into account.
      *
      * @param firstTimestamp First timestamp.
      * @param anotherTimestamp Another timestamp.
diff --git a/modules/placement-driver/src/main/java/org/apache/ignite/internal/placementdriver/LeaseUpdater.java b/modules/placement-driver/src/main/java/org/apache/ignite/internal/placementdriver/LeaseUpdater.java
index a483ff4357..5fe5cc462e 100644
--- a/modules/placement-driver/src/main/java/org/apache/ignite/internal/placementdriver/LeaseUpdater.java
+++ b/modules/placement-driver/src/main/java/org/apache/ignite/internal/placementdriver/LeaseUpdater.java
@@ -33,12 +33,13 @@ import static org.apache.ignite.internal.util.CollectionUtils.union;
 import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
@@ -398,17 +399,19 @@ public class LeaseUpdater {
 
         /** Updates leases in Meta storage. This method is supposed to be used in the busy lock. */
         private void updateLeaseBatchInternal() {
-            HybridTimestamp now = clockService.now();
+            HybridTimestamp currentTime = clockService.current();
 
             leaseUpdateStatistics = new LeaseStats();
 
             long leaseExpirationInterval = replicationConfiguration.leaseExpirationInterval().value();
 
-            long outdatedLeaseThreshold = now.getPhysical() + leaseExpirationInterval / 2;
+            long outdatedLeaseThreshold = currentTime.getPhysical() + leaseExpirationInterval / 2;
+
+            HybridTimestamp newExpirationTimestamp = new HybridTimestamp(currentTime.getPhysical() + leaseExpirationInterval, 0);
 
             Leases leasesCurrent = leaseTracker.leasesCurrent();
             Map<ReplicationGroupId, LeaseAgreement> toBeNegotiated = new HashMap<>();
-            Map<ReplicationGroupId, Lease> renewedLeases = new HashMap<>(leasesCurrent.leaseByGroupId());
+            Map<ReplicationGroupId, Lease> renewedLeases = new HashMap<>(leasesCurrent.leaseByGroupId().size());
 
             Map<ReplicationGroupId, TokenizedAssignments> tokenizedStableAssignmentsMap = assignmentsTracker.stableAssignments();
             Map<ReplicationGroupId, TokenizedAssignments> tokenizedPendingAssignmentsMap = assignmentsTracker.pendingAssignments();
@@ -418,11 +421,6 @@ public class LeaseUpdater {
                     tokenizedStableAssignmentsMap.keySet()
             );
 
-            // Remove all expired leases that are no longer present in assignments.
-            renewedLeases.entrySet().removeIf(e -> clockService.before(e.getValue().getExpirationTime(), now)
-                    && !groupsAmongCurrentStableAndPendingAssignments.contains(e.getKey()));
-
-            //
             Map<ReplicationGroupId, Pair<Set<Assignment>, Set<Assignment>>> aggregatedStableAndPendingAssignmentsByGroups = new HashMap<>();
 
             for (ReplicationGroupId grpId : groupsAmongCurrentStableAndPendingAssignments) {
@@ -437,6 +435,8 @@ public class LeaseUpdater {
             int currentPendingAssignmentsSize = tokenizedPendingAssignmentsMap.size();
             int activeLeasesCount = 0;
 
+            Set<ReplicationGroupId> prolongableLeaseGroupIds = new HashSet<>();
+
             for (Map.Entry<ReplicationGroupId, Pair<Set<Assignment>, Set<Assignment>>> entry
                     : aggregatedStableAndPendingAssignmentsByGroups.entrySet()
             ) {
@@ -487,14 +487,20 @@ public class LeaseUpdater {
                     }
                 }
 
-                // The lease is expired or close to this.
-                if (lease.getExpirationTime().getPhysical() < outdatedLeaseThreshold) {
-                    String proposedLeaseholder = lease.isProlongable()
-                            ? lease.getLeaseholder()
-                            : lease.proposedCandidate();
+                // Calculate candidate for any accepted lease to define whether it can be prolonged (prolongation is possible only
+                // if the candidate is the same as the current leaseholder).
+                String proposedLeaseholder = lease.isProlongable()
+                        ? lease.getLeaseholder()
+                        : lease.proposedCandidate();
+
+                ClusterNode candidate = nextLeaseHolder(stableAssignments, pendingAssignments, grpId, proposedLeaseholder);
 
-                    ClusterNode candidate = nextLeaseHolder(stableAssignments, pendingAssignments, grpId, proposedLeaseholder);
+                boolean canBeProlonged = lease.isAccepted()
+                        && lease.isProlongable()
+                        && candidate != null && candidate.id().equals(lease.getLeaseholderId());
 
+                // The lease is expired or close to this.
+                if (lease.getExpirationTime().getPhysical() < outdatedLeaseThreshold) {
                     // If we couldn't find a candidate neither stable nor pending assignments set, so update stats and skip iteration
                     if (candidate == null) {
                         leaseUpdateStatistics.onLeaseWithoutCandidate();
@@ -512,15 +518,15 @@ public class LeaseUpdater {
                         boolean force = !lease.isProlongable() && lease.proposedCandidate() != null;
 
                         toBeNegotiated.put(grpId, new LeaseAgreement(newLease, force));
-                    } else if (lease.isProlongable() && candidate.id().equals(lease.getLeaseholderId())) {
+                    } else if (canBeProlonged) {
                         // Old lease is renewed.
-                        prolongLease(grpId, lease, renewedLeases, leaseExpirationInterval);
+                        renewedLeases.put(grpId, prolongLease(lease, newExpirationTimestamp));
                     }
+                } else if (canBeProlonged) {
+                    prolongableLeaseGroupIds.add(grpId);
                 }
             }
 
-            byte[] renewedValue = new LeaseBatch(renewedLeases.values()).bytes();
-
             ByteArray key = PLACEMENTDRIVER_LEASES_KEY;
 
             if (shouldLogLeaseStatistics()) {
@@ -535,11 +541,29 @@ public class LeaseUpdater {
                 );
             }
 
-            if (Arrays.equals(leasesCurrent.leasesBytes(), renewedValue)) {
+            boolean emptyAssignments = aggregatedStableAndPendingAssignmentsByGroups.isEmpty();
+            if (renewedLeases.isEmpty() && (!emptyAssignments || leasesCurrent.leaseByGroupId().isEmpty())) {
                 LOG.debug("No leases to update found.");
                 return;
             }
 
+            leasesCurrent.leaseByGroupId().forEach(renewedLeases::putIfAbsent);
+
+            for (Iterator<Entry<ReplicationGroupId, Lease>> iter = renewedLeases.entrySet().iterator(); iter.hasNext(); ) {
+                Map.Entry<ReplicationGroupId, Lease> entry = iter.next();
+                ReplicationGroupId groupId = entry.getKey();
+                Lease lease = entry.getValue();
+
+                if (clockService.before(lease.getExpirationTime(), currentTime)
+                        && !groupsAmongCurrentStableAndPendingAssignments.contains(groupId)) {
+                    iter.remove();
+                } else if (prolongableLeaseGroupIds.contains(groupId)) {
+                    entry.setValue(prolongLease(lease, newExpirationTimestamp));
+                }
+            }
+
+            byte[] renewedValue = new LeaseBatch(renewedLeases.values()).bytes();
+
             msManager.invoke(
                     or(notExists(key), value(key).eq(leasesCurrent.leasesBytes())),
                     put(key, renewedValue),
@@ -629,22 +653,17 @@ public class LeaseUpdater {
         /**
          * Prolongs the lease.
          *
-         * @param grpId Replication group id.
          * @param lease Lease to prolong.
+         * @param newExpirationTimestamp New expiration timestamp.
+         * @return Prolonged lease.
          */
-        private void prolongLease(
-                ReplicationGroupId grpId,
+        private Lease prolongLease(
                 Lease lease,
-                Map<ReplicationGroupId, Lease> renewedLeases,
-                long leaseExpirationInterval
+                HybridTimestamp newExpirationTimestamp
         ) {
-            var newTs = new HybridTimestamp(clockService.now().getPhysical() + leaseExpirationInterval, 0);
-
-            Lease renewedLease = lease.prolongLease(newTs);
-
-            renewedLeases.put(grpId, renewedLease);
-
             leaseUpdateStatistics.onLeaseProlong();
+
+            return lease.prolongLease(newExpirationTimestamp);
         }
 
         /**
diff --git a/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/ActiveActorTest.java b/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/ActiveActorTest.java
index c73a3241cd..d53e281460 100644
--- a/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/ActiveActorTest.java
+++ b/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/ActiveActorTest.java
@@ -21,7 +21,6 @@ import static java.util.concurrent.CompletableFuture.completedFuture;
 import static java.util.stream.Collectors.toSet;
 import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
 import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
-import static org.apache.ignite.internal.util.CompletableFutures.trueCompletedFuture;
 import static org.apache.ignite.internal.util.CursorUtils.emptyCursor;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -43,7 +42,6 @@ import org.apache.ignite.internal.manager.ComponentContext;
 import org.apache.ignite.internal.metastorage.Entry;
 import org.apache.ignite.internal.metastorage.MetaStorageManager;
 import org.apache.ignite.internal.metastorage.Revisions;
-import org.apache.ignite.internal.metastorage.dsl.Operation;
 import org.apache.ignite.internal.network.ClusterService;
 import org.apache.ignite.internal.raft.Loza;
 import org.apache.ignite.internal.raft.Peer;
@@ -74,7 +72,6 @@ public class ActiveActorTest extends AbstractTopologyAwareGroupServiceTest {
     @BeforeEach
     public void setUp() {
         when(msm.recoveryFinishedFuture()).thenReturn(completedFuture(new Revisions(0, -1)));
-        when(msm.invoke(any(), any(Operation.class), any(Operation.class))).thenReturn(trueCompletedFuture());
         when(msm.getLocally(any(), anyLong())).then(invocation -> emptyMetastoreEntry());
         when(msm.prefixLocally(any(), anyLong())).then(invocation -> emptyCursor());
     }
diff --git a/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/LeaseNegotiationTest.java b/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/LeaseNegotiationTest.java
index c3fa9b19f1..acced68515 100644
--- a/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/LeaseNegotiationTest.java
+++ b/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/LeaseNegotiationTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.placementdriver;
 
+import static java.util.Collections.emptyList;
 import static java.util.UUID.randomUUID;
 import static java.util.concurrent.CompletableFuture.completedFuture;
 import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.stablePartAssignmentsKey;
@@ -38,6 +39,7 @@ import static org.mockito.Mockito.when;
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.util.Collection;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
@@ -342,29 +344,96 @@ public class LeaseNegotiationTest extends BaseIgniteAbstractTest {
         assertEquals(negotiatedLeaseStartTime.get(), lease.getStartTime().longValue());
     }
 
+    @Test
+    public void testAllLeasesAreProlongedIfOneIs() throws InterruptedException {
+        leaseGrantedMessageHandler = (n, lgm) -> completedFuture(createLeaseGrantedMessageResponse(true));
+
+        TablePartitionId groupId0 = new TablePartitionId(1, 0);
+        TablePartitionId groupId1 = new TablePartitionId(1, 1);
+
+        metaStorageManager.put(stablePartAssignmentsKey(groupId0), Assignments.toBytes(Set.of(forPeer(NODE_0_NAME)), assignmentsTimestamp));
+
+        waitForAcceptedLease();
+
+        metaStorageManager.put(stablePartAssignmentsKey(groupId1), Assignments.toBytes(Set.of(forPeer(NODE_0_NAME)), assignmentsTimestamp));
+
+        try {
+            assertTrue(waitForCondition(() -> {
+                Collection<Lease> leases = getAllLeasesFromMs();
+
+                Lease lease0 = leases.stream().filter(l -> l.replicationGroupId().equals(groupId0)).findAny().orElse(null);
+                Lease lease1 = leases.stream().filter(l -> l.replicationGroupId().equals(groupId1)).findAny().orElse(null);
+
+                // Start time of the leases should be different, but their expiration time should be finally the same.
+                return lease0 != null
+                        && lease1 != null
+                        && lease0.isAccepted()
+                        && lease1.isAccepted()
+                        && lease1.getStartTime().compareTo(lease0.getStartTime()) > 0
+                        && lease1.getExpirationTime().equals(lease0.getExpirationTime());
+            }, 9000));
+        } catch (AssertionError e) {
+            Collection<Lease> leases = getAllLeasesFromMs();
+            log.warn("Leases from meta storage:");
+            leases.forEach(lease -> log.warn("    " + lease));
+            throw e;
+        }
+    }
+
+    @Test
+    public void testLeasesCleanup() throws InterruptedException {
+        leaseGrantedMessageHandler = (n, lgm) -> completedFuture(createLeaseGrantedMessageResponse(true));
+
+        metaStorageManager.put(stablePartAssignmentsKey(GROUP_ID), Assignments.toBytes(Set.of(forPeer(NODE_0_NAME)), assignmentsTimestamp));
+
+        waitForAcceptedLease();
+
+        metaStorageManager.remove(stablePartAssignmentsKey(GROUP_ID));
+
+        assertTrue(waitForCondition(() -> getAllLeasesFromMs().isEmpty(), 20_000));
+    }
+
+    @Test
+    public void testLeasesCleanupOfOneGroupFromMultiple() throws InterruptedException {
+        leaseGrantedMessageHandler = (n, lgm) -> completedFuture(createLeaseGrantedMessageResponse(true));
+
+        TablePartitionId groupId0 = new TablePartitionId(0, 0);
+        TablePartitionId groupId1 = new TablePartitionId(0, 1);
+
+        metaStorageManager.put(stablePartAssignmentsKey(groupId0), Assignments.toBytes(Set.of(forPeer(NODE_0_NAME)), assignmentsTimestamp));
+        metaStorageManager.put(stablePartAssignmentsKey(groupId1), Assignments.toBytes(Set.of(forPeer(NODE_1_NAME)), assignmentsTimestamp));
+
+        waitForAcceptedLease();
+        assertTrue(waitForCondition(() -> getAllLeasesFromMs().stream().allMatch(Lease::isAccepted), 3_000));
+
+        metaStorageManager.remove(stablePartAssignmentsKey(groupId0));
+
+        assertTrue(waitForCondition(() -> {
+            Collection<Lease> leases = getAllLeasesFromMs();
+            return leases.size() == 1 && leases.stream()
+                    .allMatch(lease -> lease.replicationGroupId().equals(groupId1) && lease.isAccepted());
+        }, 20_000));
+    }
+
     @Nullable
     private Lease getLeaseFromMs() {
+        return getAllLeasesFromMs().stream().findFirst().orElse(null);
+    }
+
+    private Collection<Lease> getAllLeasesFromMs() {
         CompletableFuture<Entry> f = metaStorageManager.get(PLACEMENTDRIVER_LEASES_KEY);
 
         assertThat(f, willSucceedFast());
 
         Entry e = f.join();
 
-        if (e.empty() || e.tombstone()) {
-            return null;
+        if (e.empty()) {
+            return emptyList();
         }
 
         LeaseBatch leases = LeaseBatch.fromBytes(ByteBuffer.wrap(e.value()).order(ByteOrder.LITTLE_ENDIAN));
 
-        return leases.leases().stream().findFirst().orElse(null);
-    }
-
-    private void waitForLease() throws InterruptedException {
-        assertTrue(waitForCondition(() -> {
-            Lease lease = getLeaseFromMs();
-
-            return lease != null;
-        }, 10_000));
+        return leases.leases();
     }
 
     private void waitForAcceptedLease() throws InterruptedException {