You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vp...@apache.org on 2024/12/05 07:55:51 UTC
(ignite-3) branch main updated: IGNITE-23525 Prolong all leases in batch if at least one of them needs to be prolonged (#4798)
This is an automated email from the ASF dual-hosted git repository.
vpyatkov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/main by this push:
new c8b0f784bd IGNITE-23525 Prolong all leases in batch if at least one of them needs to be prolonged (#4798)
c8b0f784bd is described below
commit c8b0f784bd9463b9c0a9fedd1a83ee2d3dc91842
Author: Denis Chudov <mo...@gmail.com>
AuthorDate: Thu Dec 5 10:55:45 2024 +0300
IGNITE-23525 Prolong all leases in batch if at least one of them needs to be prolonged (#4798)
---
.../apache/ignite/internal/hlc/ClockService.java | 2 +-
.../internal/placementdriver/LeaseUpdater.java | 81 +++++++++++--------
.../internal/placementdriver/ActiveActorTest.java | 3 -
.../placementdriver/LeaseNegotiationTest.java | 91 +++++++++++++++++++---
4 files changed, 131 insertions(+), 46 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/hlc/ClockService.java b/modules/core/src/main/java/org/apache/ignite/internal/hlc/ClockService.java
index c173f8b221..b538ab51aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/hlc/ClockService.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/hlc/ClockService.java
@@ -79,7 +79,7 @@ public interface ClockService {
long maxClockSkewMillis();
/**
- * Defines whether this timestamp is strictly before the given one, taking the clock skew into account.
+ * Defines whether the first timestamp is strictly before the another one, taking the clock skew into account.
*
* @param firstTimestamp First timestamp.
* @param anotherTimestamp Another timestamp.
diff --git a/modules/placement-driver/src/main/java/org/apache/ignite/internal/placementdriver/LeaseUpdater.java b/modules/placement-driver/src/main/java/org/apache/ignite/internal/placementdriver/LeaseUpdater.java
index a483ff4357..5fe5cc462e 100644
--- a/modules/placement-driver/src/main/java/org/apache/ignite/internal/placementdriver/LeaseUpdater.java
+++ b/modules/placement-driver/src/main/java/org/apache/ignite/internal/placementdriver/LeaseUpdater.java
@@ -33,12 +33,13 @@ import static org.apache.ignite.internal.util.CollectionUtils.union;
import static org.apache.ignite.internal.util.CompletableFutures.nullCompletedFuture;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
@@ -398,17 +399,19 @@ public class LeaseUpdater {
/** Updates leases in Meta storage. This method is supposed to be used in the busy lock. */
private void updateLeaseBatchInternal() {
- HybridTimestamp now = clockService.now();
+ HybridTimestamp currentTime = clockService.current();
leaseUpdateStatistics = new LeaseStats();
long leaseExpirationInterval = replicationConfiguration.leaseExpirationInterval().value();
- long outdatedLeaseThreshold = now.getPhysical() + leaseExpirationInterval / 2;
+ long outdatedLeaseThreshold = currentTime.getPhysical() + leaseExpirationInterval / 2;
+
+ HybridTimestamp newExpirationTimestamp = new HybridTimestamp(currentTime.getPhysical() + leaseExpirationInterval, 0);
Leases leasesCurrent = leaseTracker.leasesCurrent();
Map<ReplicationGroupId, LeaseAgreement> toBeNegotiated = new HashMap<>();
- Map<ReplicationGroupId, Lease> renewedLeases = new HashMap<>(leasesCurrent.leaseByGroupId());
+ Map<ReplicationGroupId, Lease> renewedLeases = new HashMap<>(leasesCurrent.leaseByGroupId().size());
Map<ReplicationGroupId, TokenizedAssignments> tokenizedStableAssignmentsMap = assignmentsTracker.stableAssignments();
Map<ReplicationGroupId, TokenizedAssignments> tokenizedPendingAssignmentsMap = assignmentsTracker.pendingAssignments();
@@ -418,11 +421,6 @@ public class LeaseUpdater {
tokenizedStableAssignmentsMap.keySet()
);
- // Remove all expired leases that are no longer present in assignments.
- renewedLeases.entrySet().removeIf(e -> clockService.before(e.getValue().getExpirationTime(), now)
- && !groupsAmongCurrentStableAndPendingAssignments.contains(e.getKey()));
-
- //
Map<ReplicationGroupId, Pair<Set<Assignment>, Set<Assignment>>> aggregatedStableAndPendingAssignmentsByGroups = new HashMap<>();
for (ReplicationGroupId grpId : groupsAmongCurrentStableAndPendingAssignments) {
@@ -437,6 +435,8 @@ public class LeaseUpdater {
int currentPendingAssignmentsSize = tokenizedPendingAssignmentsMap.size();
int activeLeasesCount = 0;
+ Set<ReplicationGroupId> prolongableLeaseGroupIds = new HashSet<>();
+
for (Map.Entry<ReplicationGroupId, Pair<Set<Assignment>, Set<Assignment>>> entry
: aggregatedStableAndPendingAssignmentsByGroups.entrySet()
) {
@@ -487,14 +487,20 @@ public class LeaseUpdater {
}
}
- // The lease is expired or close to this.
- if (lease.getExpirationTime().getPhysical() < outdatedLeaseThreshold) {
- String proposedLeaseholder = lease.isProlongable()
- ? lease.getLeaseholder()
- : lease.proposedCandidate();
+ // Calculate candidate for any accepted lease to define whether it can be prolonged (prolongation is possible only
+ // if the candidate is the same as the current leaseholder).
+ String proposedLeaseholder = lease.isProlongable()
+ ? lease.getLeaseholder()
+ : lease.proposedCandidate();
+
+ ClusterNode candidate = nextLeaseHolder(stableAssignments, pendingAssignments, grpId, proposedLeaseholder);
- ClusterNode candidate = nextLeaseHolder(stableAssignments, pendingAssignments, grpId, proposedLeaseholder);
+ boolean canBeProlonged = lease.isAccepted()
+ && lease.isProlongable()
+ && candidate != null && candidate.id().equals(lease.getLeaseholderId());
+ // The lease is expired or close to this.
+ if (lease.getExpirationTime().getPhysical() < outdatedLeaseThreshold) {
// If we couldn't find a candidate neither stable nor pending assignments set, so update stats and skip iteration
if (candidate == null) {
leaseUpdateStatistics.onLeaseWithoutCandidate();
@@ -512,15 +518,15 @@ public class LeaseUpdater {
boolean force = !lease.isProlongable() && lease.proposedCandidate() != null;
toBeNegotiated.put(grpId, new LeaseAgreement(newLease, force));
- } else if (lease.isProlongable() && candidate.id().equals(lease.getLeaseholderId())) {
+ } else if (canBeProlonged) {
// Old lease is renewed.
- prolongLease(grpId, lease, renewedLeases, leaseExpirationInterval);
+ renewedLeases.put(grpId, prolongLease(lease, newExpirationTimestamp));
}
+ } else if (canBeProlonged) {
+ prolongableLeaseGroupIds.add(grpId);
}
}
- byte[] renewedValue = new LeaseBatch(renewedLeases.values()).bytes();
-
ByteArray key = PLACEMENTDRIVER_LEASES_KEY;
if (shouldLogLeaseStatistics()) {
@@ -535,11 +541,29 @@ public class LeaseUpdater {
);
}
- if (Arrays.equals(leasesCurrent.leasesBytes(), renewedValue)) {
+ boolean emptyAssignments = aggregatedStableAndPendingAssignmentsByGroups.isEmpty();
+ if (renewedLeases.isEmpty() && (!emptyAssignments || leasesCurrent.leaseByGroupId().isEmpty())) {
LOG.debug("No leases to update found.");
return;
}
+ leasesCurrent.leaseByGroupId().forEach(renewedLeases::putIfAbsent);
+
+ for (Iterator<Entry<ReplicationGroupId, Lease>> iter = renewedLeases.entrySet().iterator(); iter.hasNext(); ) {
+ Map.Entry<ReplicationGroupId, Lease> entry = iter.next();
+ ReplicationGroupId groupId = entry.getKey();
+ Lease lease = entry.getValue();
+
+ if (clockService.before(lease.getExpirationTime(), currentTime)
+ && !groupsAmongCurrentStableAndPendingAssignments.contains(groupId)) {
+ iter.remove();
+ } else if (prolongableLeaseGroupIds.contains(groupId)) {
+ entry.setValue(prolongLease(lease, newExpirationTimestamp));
+ }
+ }
+
+ byte[] renewedValue = new LeaseBatch(renewedLeases.values()).bytes();
+
msManager.invoke(
or(notExists(key), value(key).eq(leasesCurrent.leasesBytes())),
put(key, renewedValue),
@@ -629,22 +653,17 @@ public class LeaseUpdater {
/**
* Prolongs the lease.
*
- * @param grpId Replication group id.
* @param lease Lease to prolong.
+ * @param newExpirationTimestamp New expiration timestamp.
+ * @return Prolonged lease.
*/
- private void prolongLease(
- ReplicationGroupId grpId,
+ private Lease prolongLease(
Lease lease,
- Map<ReplicationGroupId, Lease> renewedLeases,
- long leaseExpirationInterval
+ HybridTimestamp newExpirationTimestamp
) {
- var newTs = new HybridTimestamp(clockService.now().getPhysical() + leaseExpirationInterval, 0);
-
- Lease renewedLease = lease.prolongLease(newTs);
-
- renewedLeases.put(grpId, renewedLease);
-
leaseUpdateStatistics.onLeaseProlong();
+
+ return lease.prolongLease(newExpirationTimestamp);
}
/**
diff --git a/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/ActiveActorTest.java b/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/ActiveActorTest.java
index c73a3241cd..d53e281460 100644
--- a/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/ActiveActorTest.java
+++ b/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/ActiveActorTest.java
@@ -21,7 +21,6 @@ import static java.util.concurrent.CompletableFuture.completedFuture;
import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.internal.testframework.IgniteTestUtils.waitForCondition;
import static org.apache.ignite.internal.testframework.matchers.CompletableFutureMatcher.willCompleteSuccessfully;
-import static org.apache.ignite.internal.util.CompletableFutures.trueCompletedFuture;
import static org.apache.ignite.internal.util.CursorUtils.emptyCursor;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -43,7 +42,6 @@ import org.apache.ignite.internal.manager.ComponentContext;
import org.apache.ignite.internal.metastorage.Entry;
import org.apache.ignite.internal.metastorage.MetaStorageManager;
import org.apache.ignite.internal.metastorage.Revisions;
-import org.apache.ignite.internal.metastorage.dsl.Operation;
import org.apache.ignite.internal.network.ClusterService;
import org.apache.ignite.internal.raft.Loza;
import org.apache.ignite.internal.raft.Peer;
@@ -74,7 +72,6 @@ public class ActiveActorTest extends AbstractTopologyAwareGroupServiceTest {
@BeforeEach
public void setUp() {
when(msm.recoveryFinishedFuture()).thenReturn(completedFuture(new Revisions(0, -1)));
- when(msm.invoke(any(), any(Operation.class), any(Operation.class))).thenReturn(trueCompletedFuture());
when(msm.getLocally(any(), anyLong())).then(invocation -> emptyMetastoreEntry());
when(msm.prefixLocally(any(), anyLong())).then(invocation -> emptyCursor());
}
diff --git a/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/LeaseNegotiationTest.java b/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/LeaseNegotiationTest.java
index c3fa9b19f1..acced68515 100644
--- a/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/LeaseNegotiationTest.java
+++ b/modules/placement-driver/src/test/java/org/apache/ignite/internal/placementdriver/LeaseNegotiationTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.placementdriver;
+import static java.util.Collections.emptyList;
import static java.util.UUID.randomUUID;
import static java.util.concurrent.CompletableFuture.completedFuture;
import static org.apache.ignite.internal.distributionzones.rebalance.RebalanceUtil.stablePartAssignmentsKey;
@@ -38,6 +39,7 @@ import static org.mockito.Mockito.when;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.util.Collection;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@@ -342,29 +344,96 @@ public class LeaseNegotiationTest extends BaseIgniteAbstractTest {
assertEquals(negotiatedLeaseStartTime.get(), lease.getStartTime().longValue());
}
+ @Test
+ public void testAllLeasesAreProlongedIfOneIs() throws InterruptedException {
+ leaseGrantedMessageHandler = (n, lgm) -> completedFuture(createLeaseGrantedMessageResponse(true));
+
+ TablePartitionId groupId0 = new TablePartitionId(1, 0);
+ TablePartitionId groupId1 = new TablePartitionId(1, 1);
+
+ metaStorageManager.put(stablePartAssignmentsKey(groupId0), Assignments.toBytes(Set.of(forPeer(NODE_0_NAME)), assignmentsTimestamp));
+
+ waitForAcceptedLease();
+
+ metaStorageManager.put(stablePartAssignmentsKey(groupId1), Assignments.toBytes(Set.of(forPeer(NODE_0_NAME)), assignmentsTimestamp));
+
+ try {
+ assertTrue(waitForCondition(() -> {
+ Collection<Lease> leases = getAllLeasesFromMs();
+
+ Lease lease0 = leases.stream().filter(l -> l.replicationGroupId().equals(groupId0)).findAny().orElse(null);
+ Lease lease1 = leases.stream().filter(l -> l.replicationGroupId().equals(groupId1)).findAny().orElse(null);
+
+ // Start time of the leases should be different, but their expiration time should be finally the same.
+ return lease0 != null
+ && lease1 != null
+ && lease0.isAccepted()
+ && lease1.isAccepted()
+ && lease1.getStartTime().compareTo(lease0.getStartTime()) > 0
+ && lease1.getExpirationTime().equals(lease0.getExpirationTime());
+ }, 9000));
+ } catch (AssertionError e) {
+ Collection<Lease> leases = getAllLeasesFromMs();
+ log.warn("Leases from meta storage:");
+ leases.forEach(lease -> log.warn(" " + lease));
+ throw e;
+ }
+ }
+
+ @Test
+ public void testLeasesCleanup() throws InterruptedException {
+ leaseGrantedMessageHandler = (n, lgm) -> completedFuture(createLeaseGrantedMessageResponse(true));
+
+ metaStorageManager.put(stablePartAssignmentsKey(GROUP_ID), Assignments.toBytes(Set.of(forPeer(NODE_0_NAME)), assignmentsTimestamp));
+
+ waitForAcceptedLease();
+
+ metaStorageManager.remove(stablePartAssignmentsKey(GROUP_ID));
+
+ assertTrue(waitForCondition(() -> getAllLeasesFromMs().isEmpty(), 20_000));
+ }
+
+ @Test
+ public void testLeasesCleanupOfOneGroupFromMultiple() throws InterruptedException {
+ leaseGrantedMessageHandler = (n, lgm) -> completedFuture(createLeaseGrantedMessageResponse(true));
+
+ TablePartitionId groupId0 = new TablePartitionId(0, 0);
+ TablePartitionId groupId1 = new TablePartitionId(0, 1);
+
+ metaStorageManager.put(stablePartAssignmentsKey(groupId0), Assignments.toBytes(Set.of(forPeer(NODE_0_NAME)), assignmentsTimestamp));
+ metaStorageManager.put(stablePartAssignmentsKey(groupId1), Assignments.toBytes(Set.of(forPeer(NODE_1_NAME)), assignmentsTimestamp));
+
+ waitForAcceptedLease();
+ assertTrue(waitForCondition(() -> getAllLeasesFromMs().stream().allMatch(Lease::isAccepted), 3_000));
+
+ metaStorageManager.remove(stablePartAssignmentsKey(groupId0));
+
+ assertTrue(waitForCondition(() -> {
+ Collection<Lease> leases = getAllLeasesFromMs();
+ return leases.size() == 1 && leases.stream()
+ .allMatch(lease -> lease.replicationGroupId().equals(groupId1) && lease.isAccepted());
+ }, 20_000));
+ }
+
@Nullable
private Lease getLeaseFromMs() {
+ return getAllLeasesFromMs().stream().findFirst().orElse(null);
+ }
+
+ private Collection<Lease> getAllLeasesFromMs() {
CompletableFuture<Entry> f = metaStorageManager.get(PLACEMENTDRIVER_LEASES_KEY);
assertThat(f, willSucceedFast());
Entry e = f.join();
- if (e.empty() || e.tombstone()) {
- return null;
+ if (e.empty()) {
+ return emptyList();
}
LeaseBatch leases = LeaseBatch.fromBytes(ByteBuffer.wrap(e.value()).order(ByteOrder.LITTLE_ENDIAN));
- return leases.leases().stream().findFirst().orElse(null);
- }
-
- private void waitForLease() throws InterruptedException {
- assertTrue(waitForCondition(() -> {
- Lease lease = getLeaseFromMs();
-
- return lease != null;
- }, 10_000));
+ return leases.leases();
}
private void waitForAcceptedLease() throws InterruptedException {