You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@accumulo.apache.org by "kevinrr888 (via GitHub)" <gi...@apache.org> on 2024/05/03 20:19:51 UTC
[PR] Fate reservations moved out of memory [accumulo]
kevinrr888 opened a new pull request, #4524:
URL: https://github.com/apache/accumulo/pull/4524
closes #4131
Fate reservations were moved out of memory. This is a WIP PR* for one part that is needed for having Fate be distributed in the future.
- Reservations for MetaFateStore were moved out of memory into ZooKeeper
- Reservations for UserFateStore were moved out of memory into the Accumulo Fate table
- Is an additional column which just indicates whether the FateId of that row is reserved or not
- Added test MultipleStoresIT
- Tests the new functionality of how reservations are stored
- I also included a couple tests in MultipleStoresIT that are testing the reserve/unreserve functionality and that it still works as intended. This can be moved from this class if desired.
*There are still a few things I have marked as TODO that I would like input/suggestions on. These are marked in the code as "TODO 4131".
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1705575651
##########
test/src/main/java/org/apache/accumulo/test/fate/MultipleStoresIT.java:
##########
@@ -0,0 +1,553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.fate;
+
+import static org.apache.accumulo.test.fate.user.UserFateStoreIT.createFateTable;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.fate.Fate;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.FateStore;
+import org.apache.accumulo.core.fate.MetaFateStore;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore;
+import org.apache.accumulo.core.fate.Repo;
+import org.apache.accumulo.core.fate.user.UserFateStore;
+import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.test.util.Wait;
+import org.apache.accumulo.test.zookeeper.ZooKeeperTestingServer;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+// TODO 4131 could potentially have separate classes for testing MetaFateStore and UserFateStore
+// similar to how FateTestRunner is used, however that interface doesn't work as nicely here
+// since we are using multiple stores instead of just one. Can do something similar to
+// FateTestRunner here if desired
+public class MultipleStoresIT extends SharedMiniClusterBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MultipleStoresIT.class);
+ @TempDir
+ private static File tempDir;
+ private static ZooKeeperTestingServer szk = null;
+ private static ZooReaderWriter zk;
+ private static final String FATE_DIR = "/fate";
+ private ClientContext client;
+
+ @BeforeEach
+ public void beforeEachSetup() {
+ client = (ClientContext) Accumulo.newClient().from(getClientProps()).build();
+ }
+
+ @AfterEach
+ public void afterEachTeardown() {
+ client.close();
+ }
+
+ @BeforeAll
+ public static void beforeAllSetup() throws Exception {
+ SharedMiniClusterBase.startMiniCluster();
+ szk = new ZooKeeperTestingServer(tempDir);
+ zk = szk.getZooReaderWriter();
+ }
+
+ @AfterAll
+ public static void afterAllTeardown() throws Exception {
+ SharedMiniClusterBase.stopMiniCluster();
+ szk.close();
+ }
+
+ @Test
+ public void testReserveUnreserve() throws Exception {
+ testReserveUnreserve(FateInstanceType.META);
+ testReserveUnreserve(FateInstanceType.USER);
+ }
+
+ private void testReserveUnreserve(FateInstanceType storeType) throws Exception {
+ // reserving/unreserving a FateId should be reflected across instances of the stores
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final FateId fakeFateId = FateId.from(storeType, UUID.randomUUID());
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> activeReservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1, null);
+ store2 = new UserFateStore<>(client, tableName, lock2, null);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, lock1, null);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, lock2, null);
+ }
+
+ // Create the fate ids using store1
+ for (int i = 0; i < numFateIds; i++) {
+ assertTrue(allIds.add(store1.create()));
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Reserve half the fate ids using store1 and rest using store2, after reserving a fate id in
+ // one, should not be able to reserve the same in the other. Should also not matter that all the
+ // ids were created using store1
+ int count = 0;
+ for (FateId fateId : allIds) {
+ if (count % 2 == 0) {
+ reservations.add(store1.reserve(fateId));
+ assertTrue(store2.tryReserve(fateId).isEmpty());
+ } else {
+ reservations.add(store2.reserve(fateId));
+ assertTrue(store1.tryReserve(fateId).isEmpty());
+ }
+ count++;
+ }
+ // Try to reserve a non-existent fate id
+ assertTrue(store1.tryReserve(fakeFateId).isEmpty());
+ assertTrue(store2.tryReserve(fakeFateId).isEmpty());
+ // Both stores should return the same reserved transactions
+ activeReservations = store1.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+ activeReservations = store2.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+
+ // Test setting/getting the TStatus and unreserving the transactions
+ for (int i = 0; i < allIds.size(); i++) {
+ var reservation = reservations.get(i);
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, reservation.getStatus());
+ reservation.setStatus(ReadOnlyFateStore.TStatus.SUBMITTED);
+ assertEquals(ReadOnlyFateStore.TStatus.SUBMITTED, reservation.getStatus());
+ reservation.delete();
+ reservation.unreserve(Duration.ofMillis(0));
+ // Attempt to set a status on a tx that has been unreserved (should throw exception)
+ assertThrows(IllegalStateException.class,
+ () -> reservation.setStatus(ReadOnlyFateStore.TStatus.NEW));
+ }
+ assertTrue(store1.getActiveReservations().isEmpty());
+ assertTrue(store2.getActiveReservations().isEmpty());
+ }
+
+ @Test
+ public void testReserveNonExistentTxn() throws Exception {
+ testReserveNonExistentTxn(FateInstanceType.META);
+ testReserveNonExistentTxn(FateInstanceType.USER);
+ }
+
+ private void testReserveNonExistentTxn(FateInstanceType storeType) throws Exception {
+ // Tests that reserve() doesn't hang indefinitely and instead throws an error
+ // on reserve() a non-existent transaction.
+ final FateStore<SleepingTestEnv> store;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final String tableName = getUniqueNames(1)[0];
+ final FateId fakeFateId = FateId.from(storeType, UUID.randomUUID());
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock, null);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, lock, null);
+ }
+
+ assertThrows(IllegalStateException.class, () -> store.reserve(fakeFateId));
Review Comment:
Added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2271363148
I have compiled and wrote all the follow on issues for this to be merged. This covers all the issues I marked in this PR as `TODO 4131` and all the unresolved review comments.
_**Here is a synopsis of the follow on issues I will create:**_
* Refactor MultipleStoresIT to function more similarly to other Fate tests like FateIT
* Add FateKey toString() method
* Move MetaFateStore to org.apache.accumulo.core.fate.zookeeper
* Deprecate AbstractFateStore.createDummyLockID(): this includes creating a path in ZK for utilities to get a ZK lock, changing `admin fate fail` and `admin fate delete` commands to get a LockID at this path and no longer require the Manager to be down, and only use a LockID for a store if write operations are expected: the store should fail on write operations if writes are not expected.
* Replace AFS.verifyReserved with a condition on the RESERVATION_COLUMN to verify that it is reserved
* Make WorkFinder and the TransactionRunners critical to the Manager
* Refactor how the RESERVATION_COLUMN works for UserFateStore: create/delete column on reserve/unreserve
* Additional fate test case from https://github.com/apache/accumulo/pull/4524#discussion_r1702301996
_**Here are some questions I still have:**_
1) This was an existing TODO that you had marked in your "distributed FATE" WIP PR:
```
// TODO 4131
// TODO make the max time a function of the number of concurrent callers, as the number of
// concurrent callers increases then increase the max wait time
// TODO could support signaling within this instance for known events
// TODO made the maxWait low so this would be responsive... that may put a lot of load in
// the case there are lots of things waiting...
```
I made the max wait = curr num callers in seconds. Does this address the first TODO? For the other two TODOs, I was not sure what was to be done, so I left them. Can these be safely deleted without creating follow on issue(s)? If follow on issues should be made, perhaps it would be best for you to write these or explain it here because I am not sure what should be done.
2) Re one of your review comments:
"Not a change to make in this PR. Thinking if we push all of the Fate data into a single node it would be nice to use JSON. That would make the data stored in ZK more human readable and it would make it easier to serialize and de-serialize."
Could you elaborate on what you mean by "all of the Fate data into a single node". By "all" do you mean the Repos, TxInfo, Status, Reservation, and FateKey would be in one ZK node? Writing the issue, I want to be sure I'm understanding correctly.
3) Can https://github.com/apache/accumulo/pull/4524#discussion_r1663064467 be resolved? Or should an issue be made?
4) There was a TODO regarding whether the ZooUtil.LockID was the best type for the lock. Should I make a follow on issue regarding this or is ZooUtil.LockID okay?
5) Can this https://github.com/apache/accumulo/pull/4524#discussion_r1663064467 be resolved or should a follow on issue be made?
6) Can these be resolved: https://github.com/apache/accumulo/pull/4524#discussion_r1670934639, https://github.com/apache/accumulo/pull/4524#discussion_r1695958819
7) The DeadReservationCleaner runs every 30 seconds. Is this okay? Should this be longer? Shorter?
Once the above questions are answered, I can complete my full list of new follow-on issues. This will allow me to remove all the `TODO 4131` in this PR and all the comments in this PR can be resolved as all of these will be addressed or have a follow on issue created for them.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672888980
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
If the reservation parsing code is not really strict and the code writing reservations has a bug, then could end with the case were reservation is silently ignored because it did not have the correct format. Would be better to fail rather than ignore things that are not recognized. Some of the code seems to have the following pattern
```java
f(value look like <lock id>:<uuid) {
// process reserved
} else {
// assume not reserved, but could have been an incorrectly written reservation
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672826477
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -73,9 +75,12 @@ public class Fate<T> {
private final FateStore<T> store;
private final T environment;
private final ScheduledThreadPoolExecutor fatePoolWatcher;
- private final ExecutorService executor;
+ private final ExecutorService transactionExecutor;
+ private final ExecutorService deadResCleanerExecutor;
private static final EnumSet<TStatus> FINISHED_STATES = EnumSet.of(FAILED, SUCCESSFUL, UNKNOWN);
+ private static boolean userDeadReservationCleanerRunning = false;
+ private static boolean metaDeadReservationCleanerRunning = false;
Review Comment:
> Should these be Atomic? Or is it fine to keep them as-is?
Right now, there is only one Manager which runs, so these do not need to be Atomic, but the goal is to have multiple managers in the future, so not sure if this should be changed now or in the future.
> Can these static variables could be removed? Do these help testing in someway? In the manager we can assume that only a single instance of each type is created. If we wanted to guard against multiple instances per type, then a more general check could be done that is not limited to dead reservation cleaners.
The purpose of these was to avoid creating more than 1 DeadReservationCleaner running per store type. As I currently have this, the first Fate object to be created per store type will be running a DeadReservationCleaner (in the future, this probably should be changed as running on the first Fate object created may not be the best approach). In the Manager, only one Fate object will be created per store type, so this isn't a necessary check now, but isn't the plan in the future to have multiple Fate instance running per store type, so in that case we'd only ever want 1 DeadReservationCleaner running? I was premtively handling this case. What do you have in mind regarding a more general check? Or should I not be handling this case yet?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1693193627
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Yes, I was running my new `MultipleStoresIT.testDeadReservationsCleanup()` test, which would occasionally fail with my new changes. The test works as follows:
1) We have a fate object fate1 which is used to have 4 (the default MANAGER_FATE_THREADPOOL_SIZE) workers reserve 4 transactions with lock1
2) We then simulate that the imaginary manager dies and a new one starts, so lock1 is no longer held instead lock2 is now held
3) A new fate2 is created which spins up 4 more workers which should eventually reserve the 4 transactions with lock2 so long as the DeadReservationCleaner thread unreserves the transactions reserved with lock1
Took a while to track down, but found that when waiting for the transactions to be reserved with lock2, multiple threads would sometimes reserve the same transaction, so not all 4 transactions would be reserved causing the test to fail. Found that the issue was due to the change from storing an unreserved value in the RESERVATION column to just deleting/creating the column on unreserve/reserve.
I could not figure out why or how this could occur with these changes. I could potentially put a new commit up at some point with these changes to see if you could find anything wrong with how I am doing it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2098719562
Marking this as ready for review, since it may not be reviewed otherwise
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1702286986
##########
test/src/main/java/org/apache/accumulo/test/fate/MultipleStoresIT.java:
##########
@@ -0,0 +1,553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.fate;
+
+import static org.apache.accumulo.test.fate.user.UserFateStoreIT.createFateTable;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.fate.Fate;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.FateStore;
+import org.apache.accumulo.core.fate.MetaFateStore;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore;
+import org.apache.accumulo.core.fate.Repo;
+import org.apache.accumulo.core.fate.user.UserFateStore;
+import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.test.util.Wait;
+import org.apache.accumulo.test.zookeeper.ZooKeeperTestingServer;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+// TODO 4131 could potentially have separate classes for testing MetaFateStore and UserFateStore
+// similar to how FateTestRunner is used, however that interface doesn't work as nicely here
+// since we are using multiple stores instead of just one. Can do something similar to
+// FateTestRunner here if desired
+public class MultipleStoresIT extends SharedMiniClusterBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MultipleStoresIT.class);
+ @TempDir
+ private static File tempDir;
+ private static ZooKeeperTestingServer szk = null;
+ private static ZooReaderWriter zk;
+ private static final String FATE_DIR = "/fate";
+ private ClientContext client;
+
+ @BeforeEach
+ public void beforeEachSetup() {
+ client = (ClientContext) Accumulo.newClient().from(getClientProps()).build();
+ }
+
+ @AfterEach
+ public void afterEachTeardown() {
+ client.close();
+ }
+
+ @BeforeAll
+ public static void beforeAllSetup() throws Exception {
+ SharedMiniClusterBase.startMiniCluster();
+ szk = new ZooKeeperTestingServer(tempDir);
+ zk = szk.getZooReaderWriter();
+ }
+
+ @AfterAll
+ public static void afterAllTeardown() throws Exception {
+ SharedMiniClusterBase.stopMiniCluster();
+ szk.close();
+ }
+
+ @Test
+ public void testReserveUnreserve() throws Exception {
+ testReserveUnreserve(FateInstanceType.META);
+ testReserveUnreserve(FateInstanceType.USER);
+ }
+
+ private void testReserveUnreserve(FateInstanceType storeType) throws Exception {
+ // reserving/unreserving a FateId should be reflected across instances of the stores
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final FateId fakeFateId = FateId.from(storeType, UUID.randomUUID());
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> activeReservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1, null);
+ store2 = new UserFateStore<>(client, tableName, lock2, null);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, lock1, null);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, lock2, null);
+ }
+
+ // Create the fate ids using store1
+ for (int i = 0; i < numFateIds; i++) {
+ assertTrue(allIds.add(store1.create()));
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Reserve half the fate ids using store1 and rest using store2, after reserving a fate id in
+ // one, should not be able to reserve the same in the other. Should also not matter that all the
+ // ids were created using store1
+ int count = 0;
+ for (FateId fateId : allIds) {
+ if (count % 2 == 0) {
+ reservations.add(store1.reserve(fateId));
+ assertTrue(store2.tryReserve(fateId).isEmpty());
+ } else {
+ reservations.add(store2.reserve(fateId));
+ assertTrue(store1.tryReserve(fateId).isEmpty());
+ }
+ count++;
+ }
+ // Try to reserve a non-existent fate id
+ assertTrue(store1.tryReserve(fakeFateId).isEmpty());
+ assertTrue(store2.tryReserve(fakeFateId).isEmpty());
+ // Both stores should return the same reserved transactions
+ activeReservations = store1.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+ activeReservations = store2.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+
+ // Test setting/getting the TStatus and unreserving the transactions
+ for (int i = 0; i < allIds.size(); i++) {
+ var reservation = reservations.get(i);
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, reservation.getStatus());
+ reservation.setStatus(ReadOnlyFateStore.TStatus.SUBMITTED);
+ assertEquals(ReadOnlyFateStore.TStatus.SUBMITTED, reservation.getStatus());
+ reservation.delete();
+ reservation.unreserve(Duration.ofMillis(0));
+ // Attempt to set a status on a tx that has been unreserved (should throw exception)
+ assertThrows(IllegalStateException.class,
+ () -> reservation.setStatus(ReadOnlyFateStore.TStatus.NEW));
+ }
+ assertTrue(store1.getActiveReservations().isEmpty());
+ assertTrue(store2.getActiveReservations().isEmpty());
+ }
+
+ @Test
+ public void testReserveNonExistentTxn() throws Exception {
+ testReserveNonExistentTxn(FateInstanceType.META);
+ testReserveNonExistentTxn(FateInstanceType.USER);
+ }
+
+ private void testReserveNonExistentTxn(FateInstanceType storeType) throws Exception {
+ // Tests that reserve() doesn't hang indefinitely and instead throws an error
+ // on reserve() a non-existent transaction.
+ final FateStore<SleepingTestEnv> store;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final String tableName = getUniqueNames(1)[0];
+ final FateId fakeFateId = FateId.from(storeType, UUID.randomUUID());
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock, null);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, lock, null);
+ }
+
+ assertThrows(IllegalStateException.class, () -> store.reserve(fakeFateId));
Review Comment:
Could verify error messages contains the fate id.
```suggestion
var err = assertThrows(IllegalStateException.class, () -> store.reserve(fakeFateId));
assertTrue(err.getMessage().contains(fakeFateId.canonical()));
```
##########
test/src/main/java/org/apache/accumulo/test/fate/MultipleStoresIT.java:
##########
@@ -0,0 +1,553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.fate;
+
+import static org.apache.accumulo.test.fate.user.UserFateStoreIT.createFateTable;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.fate.Fate;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.FateStore;
+import org.apache.accumulo.core.fate.MetaFateStore;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore;
+import org.apache.accumulo.core.fate.Repo;
+import org.apache.accumulo.core.fate.user.UserFateStore;
+import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.test.util.Wait;
+import org.apache.accumulo.test.zookeeper.ZooKeeperTestingServer;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+// TODO 4131 could potentially have separate classes for testing MetaFateStore and UserFateStore
+// similar to how FateTestRunner is used, however that interface doesn't work as nicely here
+// since we are using multiple stores instead of just one. Can do something similar to
+// FateTestRunner here if desired
Review Comment:
This is interesting. Wonder if we could pass a factory to test like the following.
```
interface TestFateStoreFactory {
FateStore create(ZooUtil.LockID lockid);
}
```
Then test could use the factory to create as many stores as needed. We could run the test passing in an impl that creates MetaFateStore and another that creates UserFateStore.
##########
test/src/main/java/org/apache/accumulo/test/fate/MultipleStoresIT.java:
##########
@@ -0,0 +1,553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.fate;
+
+import static org.apache.accumulo.test.fate.user.UserFateStoreIT.createFateTable;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.fate.Fate;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.FateStore;
+import org.apache.accumulo.core.fate.MetaFateStore;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore;
+import org.apache.accumulo.core.fate.Repo;
+import org.apache.accumulo.core.fate.user.UserFateStore;
+import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.test.util.Wait;
+import org.apache.accumulo.test.zookeeper.ZooKeeperTestingServer;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+// TODO 4131 could potentially have separate classes for testing MetaFateStore and UserFateStore
+// similar to how FateTestRunner is used, however that interface doesn't work as nicely here
+// since we are using multiple stores instead of just one. Can do something similar to
+// FateTestRunner here if desired
+public class MultipleStoresIT extends SharedMiniClusterBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MultipleStoresIT.class);
+ @TempDir
+ private static File tempDir;
+ private static ZooKeeperTestingServer szk = null;
+ private static ZooReaderWriter zk;
+ private static final String FATE_DIR = "/fate";
+ private ClientContext client;
+
+ @BeforeEach
+ public void beforeEachSetup() {
+ client = (ClientContext) Accumulo.newClient().from(getClientProps()).build();
+ }
+
+ @AfterEach
+ public void afterEachTeardown() {
+ client.close();
+ }
+
+ @BeforeAll
+ public static void beforeAllSetup() throws Exception {
+ SharedMiniClusterBase.startMiniCluster();
+ szk = new ZooKeeperTestingServer(tempDir);
+ zk = szk.getZooReaderWriter();
+ }
+
+ @AfterAll
+ public static void afterAllTeardown() throws Exception {
+ SharedMiniClusterBase.stopMiniCluster();
+ szk.close();
+ }
+
+ @Test
+ public void testReserveUnreserve() throws Exception {
+ testReserveUnreserve(FateInstanceType.META);
+ testReserveUnreserve(FateInstanceType.USER);
+ }
+
+ private void testReserveUnreserve(FateInstanceType storeType) throws Exception {
+ // reserving/unreserving a FateId should be reflected across instances of the stores
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final FateId fakeFateId = FateId.from(storeType, UUID.randomUUID());
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> activeReservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1, null);
+ store2 = new UserFateStore<>(client, tableName, lock2, null);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, lock1, null);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, lock2, null);
+ }
+
+ // Create the fate ids using store1
+ for (int i = 0; i < numFateIds; i++) {
+ assertTrue(allIds.add(store1.create()));
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Reserve half the fate ids using store1 and rest using store2, after reserving a fate id in
+ // one, should not be able to reserve the same in the other. Should also not matter that all the
+ // ids were created using store1
+ int count = 0;
+ for (FateId fateId : allIds) {
+ if (count % 2 == 0) {
+ reservations.add(store1.reserve(fateId));
+ assertTrue(store2.tryReserve(fateId).isEmpty());
+ } else {
+ reservations.add(store2.reserve(fateId));
+ assertTrue(store1.tryReserve(fateId).isEmpty());
+ }
+ count++;
+ }
+ // Try to reserve a non-existent fate id
+ assertTrue(store1.tryReserve(fakeFateId).isEmpty());
+ assertTrue(store2.tryReserve(fakeFateId).isEmpty());
+ // Both stores should return the same reserved transactions
+ activeReservations = store1.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+ activeReservations = store2.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+
+ // Test setting/getting the TStatus and unreserving the transactions
+ for (int i = 0; i < allIds.size(); i++) {
+ var reservation = reservations.get(i);
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, reservation.getStatus());
+ reservation.setStatus(ReadOnlyFateStore.TStatus.SUBMITTED);
+ assertEquals(ReadOnlyFateStore.TStatus.SUBMITTED, reservation.getStatus());
+ reservation.delete();
+ reservation.unreserve(Duration.ofMillis(0));
+ // Attempt to set a status on a tx that has been unreserved (should throw exception)
+ assertThrows(IllegalStateException.class,
+ () -> reservation.setStatus(ReadOnlyFateStore.TStatus.NEW));
+ }
+ assertTrue(store1.getActiveReservations().isEmpty());
+ assertTrue(store2.getActiveReservations().isEmpty());
+ }
+
+ @Test
+ public void testReserveNonExistentTxn() throws Exception {
+ testReserveNonExistentTxn(FateInstanceType.META);
+ testReserveNonExistentTxn(FateInstanceType.USER);
+ }
+
+ private void testReserveNonExistentTxn(FateInstanceType storeType) throws Exception {
+ // Tests that reserve() doesn't hang indefinitely and instead throws an error
+ // on reserve() a non-existent transaction.
+ final FateStore<SleepingTestEnv> store;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final String tableName = getUniqueNames(1)[0];
+ final FateId fakeFateId = FateId.from(storeType, UUID.randomUUID());
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock, null);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, lock, null);
+ }
+
+ assertThrows(IllegalStateException.class, () -> store.reserve(fakeFateId));
+ }
+
+ @Test
+ public void testReserveReservedAndUnreserveUnreserved() throws Exception {
+ testReserveReservedAndUnreserveUnreserved(FateInstanceType.META);
+ testReserveReservedAndUnreserveUnreserved(FateInstanceType.USER);
+ }
+
+ private void testReserveReservedAndUnreserveUnreserved(FateInstanceType storeType)
+ throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final FateStore<SleepingTestEnv> store;
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock, null);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, lock, null);
+ }
+
+ // Create some FateIds and ensure that they can be reserved
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId = store.create();
+ assertTrue(allIds.add(fateId));
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservations.add(reservation.orElseThrow());
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Try to reserve again, should not reserve
+ for (FateId fateId : allIds) {
+ assertTrue(store.tryReserve(fateId).isEmpty());
+ }
+
+ // Unreserve all the FateIds
+ for (var reservation : reservations) {
+ reservation.unreserve(Duration.ofMillis(0));
+ }
+ // Try to unreserve again (should throw exception)
+ for (var reservation : reservations) {
+ assertThrows(IllegalStateException.class, () -> reservation.unreserve(Duration.ofMillis(0)));
+ }
+ }
+
+ @Test
+ public void testReserveAfterUnreserveAndReserveAfterDeleted() throws Exception {
+ testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType.META);
+ testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType.USER);
+ }
+
+ private void testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType storeType)
+ throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final FateStore<SleepingTestEnv> store;
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock, null);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, lock, null);
+ }
+
+ // Create some FateIds and ensure that they can be reserved
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId = store.create();
+ assertTrue(allIds.add(fateId));
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservations.add(reservation.orElseThrow());
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Unreserve all
+ for (var reservation : reservations) {
+ reservation.unreserve(Duration.ofMillis(0));
+ }
+
+ // Ensure they can be reserved again, and delete and unreserve this time
+ for (FateId fateId : allIds) {
+ // Verify that the tx status is still NEW after unreserving since it hasn't been deleted
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, store.read(fateId).getStatus());
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservation.orElseThrow().delete();
+ reservation.orElseThrow().unreserve(Duration.ofMillis(0));
+ }
+
+ for (FateId fateId : allIds) {
+ // Verify that the tx is now unknown since it has been deleted
+ assertEquals(ReadOnlyFateStore.TStatus.UNKNOWN, store.read(fateId).getStatus());
+ // Attempt to reserve a deleted txn, should throw an exception and not wait indefinitely
+ assertThrows(IllegalStateException.class, () -> store.reserve(fateId));
+ }
+ }
+
+ @Test
+ public void testMultipleFateInstances() throws Exception {
+ testMultipleFateInstances(FateInstanceType.META);
+ testMultipleFateInstances(FateInstanceType.USER);
+ }
+
+ private void testMultipleFateInstances(FateInstanceType storeType) throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final SleepingTestEnv testEnv1 = new SleepingTestEnv(50);
+ final SleepingTestEnv testEnv2 = new SleepingTestEnv(50);
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ final Set<ZooUtil.LockID> liveLocks = new HashSet<>();
+ final Predicate<ZooUtil.LockID> isLockHeld = liveLocks::contains;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1, isLockHeld);
+ store2 = new UserFateStore<>(client, tableName, lock2, isLockHeld);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, lock1, isLockHeld);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, lock2, isLockHeld);
+ }
+ liveLocks.add(lock1);
+ liveLocks.add(lock2);
+
+ Fate<SleepingTestEnv> fate1 =
+ new Fate<>(testEnv1, store1, true, Object::toString, DefaultConfiguration.getInstance());
+ Fate<SleepingTestEnv> fate2 =
+ new Fate<>(testEnv2, store2, false, Object::toString, DefaultConfiguration.getInstance());
+
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId;
+ // Start half the txns using fate1, and the other half using fate2
+ if (i % 2 == 0) {
+ fateId = fate1.startTransaction();
+ fate1.seedTransaction("op" + i, fateId, new SleepingTestRepo(), true, "test");
+ } else {
+ fateId = fate2.startTransaction();
+ fate2.seedTransaction("op" + i, fateId, new SleepingTestRepo(), true, "test");
+ }
+ allIds.add(fateId);
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Should be able to wait for completion on any fate instance
+ for (FateId fateId : allIds) {
+ fate2.waitForCompletion(fateId);
+ }
+ // Ensure that all txns have been executed and have only been executed once
+ assertTrue(Collections.disjoint(testEnv1.executedOps, testEnv2.executedOps));
+ assertEquals(allIds, Sets.union(testEnv1.executedOps, testEnv2.executedOps));
+
+ fate1.shutdown(1, TimeUnit.MINUTES);
+ fate2.shutdown(1, TimeUnit.MINUTES);
+ }
+
+ @Test
+ public void testDeadReservationsCleanup() throws Exception {
+ testDeadReservationsCleanup(FateInstanceType.META);
+ testDeadReservationsCleanup(FateInstanceType.USER);
+ }
+
+ private void testDeadReservationsCleanup(FateInstanceType storeType) throws Exception {
Review Comment:
This test looks good. Looking over this test realized we need another test in a follow on PR. Need a test that checks that when the reservation does not exists in the persisted store that every update fails. Test would look something like the following.
```
FateStore.FateTxStore<?> txStore = store1.reserve(fateId);
// TODO delete the reservation in persisted store
assertThrows(Exception.class, ()->txStore.push(repo));
assertThrows(Exception.class, ()->txStore.setStatus(ReadOnlyFateStore.TStatus.IN_PROGRESS));
assertThrows(Exception.class, ()->txStore.setTransactionInfo(t,v));
//etc test all methods that write
```
For now may only be able to make this work with the UserFateStore as the current zookeeper schema does not support the conditional checks of reservations on updates.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2263647852
@keith-turner - Addressed all review comments, this PR is ready for re-review.
See new commit bd1f17472c8d289878e19d7121f9fc9347e95596
I also fixed FateStoreIT to work with the new impl of createAndReserve(). A method was deleted in 6760035608387ae537af7050106c9645aac19aae that was no longer used which FateStoreIT relied on. Upon fixing this test, noticed that the original impl of createAndReserve expected an error to be thrown in a certain situation, but in the new implementation, an empty optional was returned instead. Fixed the new impl to match the previous impl for this case. This meant changing UFS.createAndReserve and MFS.createAndReserve.
Also verified existing Fate tests still pass: ZooMutatorIT,FateMutatorImplIT,UserFateStoreIT,FateInterleavingIT,FateIT,FateOpsCommandsIT,ComprehensiveFlakyFateIT,DeleteRowsFlakyFateIT,ManagerRepoIT,MultipleStoresIT,FateStoreIT
and verified sunny day tests still pass.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1597510917
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -150,6 +150,39 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // TODO 4131 should this throw an exception if the id doesn't exist (status = UNKNOWN)?
+ FateMutator.Status status = newMutator(fateId).putReservedTx(fateId).tryMutate();
Review Comment:
This code needs to be refactored to use a UUID similar to the zookeeper code. The UUID covers the problem of a write making it the server, but the client not knowing the write made it. Both zookeeper and accumulo can have this problem and both can have a similar solution. The following examples show how the UUID can help address this problem.
1. FATE1 exists and is not currently reserved.
2. Thread1 attempts reserve FATE1 by writing `isreserved` only if current value is `notreserved`.
3. Thread2 attempts reserve FATE1 by writing `isreserved` only if current value is `notreserved`.
4. One of the writes succeeds and the other fails. However before the tablet server can respond to Thread1 or Thread2, it dies. This causes both thread to receive an unknown status.
5. After getting the UNKNOWN status, Thread1 reads FATE1 and sees it marked reserved however it does not know if it reserved FATE1 or another thread.
6. After getting the UNKNOWN status, Thread2 reads FATE1 and sees it marked reserved however it does not know if it reserved FATE1 or another thread.
In the above example, FATE1 is marked reserved but no thread will be working on it. If instead of setting a value of `isreserved` we instead set a per thread UUID, then the above situation could look like the following.
1. FATE1 exists and is not currently reserved. Its reserved column has a value of empty string.
2. Thread1 wants to reserve FATE1 and generates UUID1 for this purpose
3. Thread2 wants to reserve FATE1 and generates UUID2 for this purpose
4. Thread1 attempts reserve FATE1 by writing UUID1 as a value for the reserved column only if current value is empty.
5. Thread2 attempts reserve FATE1 by writing UUID2 as a value for the reserved column only if current value is empty.
6. The write for Thread2 succeeds and the write for Thread1 fails because the col val is not empty. However before the tablet server can respond to Thread1 or Thread2, it dies. This causes both thread to receive an unknown status.
7. After getting the UNKNOWN status, Thread1 reads FATE1 reserved column and sees UUID2 so it knows it did not reserve it.
8. After getting the UNKNOWN status, Thread2 reads FATE1 reserved column and sees UUID2 so it knows it did reserve it.
In addition to the UUID mentioned above, the persisted reservation will need another field and that is a process lock id. The UUID mentioned above is scoped very narrowly, it scoped to an individual reservation attempt and its used to handle the case of server applying write and then dying. The process lock id would be used to detect reservation that are held by a process that is no longer alive. The following is an example of how all of this works together.
1. A manager process starts in Accumulo and acquires a lock in ZK, let calls it PL1
2. The manager process reserves FATE1 setting the reservation column to UUID1:PL1. UUID1 is scoped to this reservation thread/code block as mentioned above.
3. After successfully reserving (could have seen unknown status and check for UUID1) the manager process dies
4. A new manager process is started and acquires a lock in ZK, let calls it PL2
5. The new manager process starts FATE, which starts a periodic task to look for reservations held by dead process. The periodic task sees FATE1 is reserved by PL1 which no longer has lock in ZK and is therefore presumed dead.
6. The periodic task issues a conditional mutation (or conditional update for ZK) to delete the reserved value for FATE1 only if the currently value is UUID1:PL1
Taking all of this together, may want to do the following.
1. Create a new type that represent the reservation value, maybe like
```java
class FateReseveration {
UUID reservationId;
ZooUtil.LockID processID; // TODO not sure if this is the best type for this
}
```
2. Use the above type internally in the User and Meta Fatestores for their reservations
3. Add methods to FateStore like `Stream<FateReservation> getActiveReservations()` and `void deleteReservation(FateReservation)`
4. Add a periodic task to Fate that calls the two methods above and looks active reservations held by dead processes and then deletes them.
5. When creating a new FateStore instance, pass in the lock of the process where it is running. For now this would mean when the manager creates a new FateStore instance it will pass in its lock. This will be used to set the processID on reservations.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672908658
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Ah I see. Yeah I don't think I ever check if something is reserved like that and instead use a conditional mutation in FateMutatorImpl to reserve/unreserve. Should be pretty straightforward to change to create/delete the column on reserve/unreserve
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2203359587
@keith-turner Thanks for the review. I addressed the suggested changes. This PR is ready for re-review
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1674694427
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Yeah I think that would work and sounds like a good idea. It would clean up/get rid of some code. Don't think converting the FateReservation to/from a string would be needed anymore.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2145316505
I have made the following changes in https://github.com/apache/accumulo/pull/4524/commits/e6671fbb903da53014bd64e53ad125351add6a04:
- Combined the LockID and reservation attempt UUID into one new object: FateReservation
- Now cover edge case with tryReserve() where a write may make it to the server, but the server dies before a response is received
- New FATE Thread: DeadReservationCleaner which deletes reservations held by Managers that have since died
- Created new classes: ColumnValueMappingIterator and ReservationMappingIterator. ColumnValueMappingIterator abstracts out the common functionality of ReservationMappingIterator and StatusMappingIterator. ReservationMappingIterator is an iterator used for determining if the reservation column for a FateId has a FateReservation set or not
- Expanded/improved/simplified MultipleStoresIT tests
There are also some comments/questions labeled "TODO 4131" that I would like input on.
I also have to resolve the merge conflicts which I will do now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1693193627
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Yes, I was running my new `MultipleStoresIT.testDeadReservationsCleanup()` test, which would occasionally fail with my new changes. The test works as follows:
1) We have a fate object fate1 which is used to have 4 (the default MANAGER_FATE_THREADPOOL_SIZE) workers reserve 4 transactions with lock1
2) We then simulate that the imaginary manager dies and a new one starts, so lock1 is no longer held instead lock2 is now held
3) A new fate2 is created which spins up 4 more workers which should eventually reserve the 4 transactions so long as the DeadReservationCleaner thread unreserves the transactions reserved with lock1
Took a while to track down, but found that when waiting for the transactions to be reserved with lock2, multiple threads would sometimes reserve the same transaction, so not all 4 transactions would be reserved causing the test to fail. Found that the issue was due to the change from storing an unreserved value in the RESERVATION column to just deleting/creating the column on unreserve/reserve.
I could not figure out why or how this could occur with these changes. I could potentially put a new commit up at some point with these changes to see if you could find anything wrong with how I am doing it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672826477
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -73,9 +75,12 @@ public class Fate<T> {
private final FateStore<T> store;
private final T environment;
private final ScheduledThreadPoolExecutor fatePoolWatcher;
- private final ExecutorService executor;
+ private final ExecutorService transactionExecutor;
+ private final ExecutorService deadResCleanerExecutor;
private static final EnumSet<TStatus> FINISHED_STATES = EnumSet.of(FAILED, SUCCESSFUL, UNKNOWN);
+ private static boolean userDeadReservationCleanerRunning = false;
+ private static boolean metaDeadReservationCleanerRunning = false;
Review Comment:
> Should these be Atomic? Or is it fine to keep them as-is?
Right now, there is only one Manager which runs, so these do not need to be Atomic, but the goal is to have multiple managers in the future, so not sure if this should be changed now or in the future.
Edit for further clarification: Only one thread will be accessing these at a time (the Manager). The only places these are accessed or used are in the constructor for Fate, and in Fate.shutdown(), neither of which have multiple threads executing. This will change when/if more than 1 Manager is running in the future.
> Can these static variables could be removed? Do these help testing in someway? In the manager we can assume that only a single instance of each type is created. If we wanted to guard against multiple instances per type, then a more general check could be done that is not limited to dead reservation cleaners.
The purpose of these was to avoid creating more than 1 DeadReservationCleaner running per store type. As I currently have this, the first Fate object to be created per store type will be running a DeadReservationCleaner (in the future, this probably should be changed as running on the first Fate object created may not be the best approach). In the Manager, only one Fate object will be created per store type, so this isn't a necessary check now, but isn't the plan in the future to have multiple Fate instance running per store type, so in that case we'd only ever want 1 DeadReservationCleaner running? Also, in my tests I create more than 1 Fate per store type, so without a check I'd be running more than 1 here which I don't really want. What do you have in mind regarding a more general check?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1706121918
##########
core/src/main/java/org/apache/accumulo/core/fate/FateStore.java:
##########
@@ -107,11 +118,155 @@ interface FateTxStore<T> extends ReadOnlyFateTxStore<T> {
void unreserve(Duration deferTime);
}
+ /**
+ * The value stored to indicate a FATE transaction ID ({@link FateId}) has been reserved
+ */
+ class FateReservation {
+
+ // The LockID (provided by the Manager running the FATE which uses this store) which is used for
+ // identifying dead Managers, so their reservations can be deleted and picked up again since
+ // they can no longer be worked on.
+ private final ZooUtil.LockID lockID; // TODO 4131 not sure if this is the best type for this
+ // The UUID generated on a reservation attempt (tryReserve()) used to uniquely identify that
+ // attempt. This is useful for the edge case where the reservation is sent to the server
+ // (Tablet Server for UserFateStore and the ZooKeeper Server for MetaFateStore), but the server
+ // dies before the store receives the response. It allows us to determine if the reservation
+ // was successful and was written by this reservation attempt (could have been successfully
+ // reserved by another attempt or not reserved at all, in which case, we wouldn't want to
+ // expose a FateTxStore).
+ private final UUID reservationUUID;
+ private final byte[] serialized;
+ private static final Pattern UUID_PATTERN =
+ Pattern.compile("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$");
+ private static final Pattern LOCKID_PATTERN = Pattern.compile("^.+/.+\\$[0-9a-fA-F]+$");
+
+ private FateReservation(ZooUtil.LockID lockID, UUID reservationUUID) {
+ this.lockID = Objects.requireNonNull(lockID);
+ this.reservationUUID = Objects.requireNonNull(reservationUUID);
+ this.serialized = serialize(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(ZooUtil.LockID lockID, UUID reservationUUID) {
+ return new FateReservation(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(byte[] serialized) {
+ try (DataInputBuffer buffer = new DataInputBuffer()) {
+ buffer.reset(serialized, serialized.length);
+ ZooUtil.LockID lockID = new ZooUtil.LockID("", buffer.readUTF());
+ UUID reservationUUID = UUID.fromString(buffer.readUTF());
+ return new FateReservation(lockID, reservationUUID);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static FateReservation from(String fateReservationStr) {
+ if (isFateReservation(fateReservationStr)) {
+ String[] fields = fateReservationStr.split(":");
+ ZooUtil.LockID lockId = new ZooUtil.LockID("", fields[0]);
+ UUID reservationUUID = UUID.fromString(fields[1]);
+ return new FateReservation(lockId, reservationUUID);
+ } else {
+ throw new IllegalArgumentException(
+ "Tried to create a FateReservation from an invalid string: " + fateReservationStr);
+ }
+ }
+
+ /**
+ *
+ * @param fateReservationStr the string from a call to {@link FateReservation#toString()}
+ * @return true if the string represents a valid FateReservation object, false otherwise
+ */
+ public static boolean isFateReservation(String fateReservationStr) {
+ if (fateReservationStr != null) {
+ String[] fields = fateReservationStr.split(":");
+ if (fields.length == 2) {
+ return LOCKID_PATTERN.matcher(fields[0]).matches()
+ && UUID_PATTERN.matcher(fields[1]).matches();
+ }
+ }
+ return false;
+ }
+
+ public ZooUtil.LockID getLockID() {
+ return lockID;
+ }
+
+ public UUID getReservationUUID() {
+ return reservationUUID;
+ }
+
+ public byte[] getSerialized() {
+ return serialized;
+ }
+
+ private static byte[] serialize(ZooUtil.LockID lockID, UUID reservationUUID) {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos)) {
+ dos.writeUTF(lockID.serialize("/"));
+ dos.writeUTF(reservationUUID.toString());
+ dos.close();
Review Comment:
@DomGarguilo if you think it would be best to call flush everywhere can you please create and issue and resolve this comment? There are so many comments on this issue, we are trying to get them all resolved, if no issue is needed can just resolve it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1707504129
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Yes, resolved. This conversation will be linked in the new issue to address this change as context.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700583872
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -104,19 +109,108 @@ public FateId create() {
}
@Override
- protected void create(FateId fateId, FateKey key) {
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
- NodeExistsPolicy.FAIL);
- } catch (KeeperException | InterruptedException e) {
+ byte[] nodeVal = zk.mutateOrCreate(getTXPath(fateId),
+ new NodeValue(TStatus.NEW, reservation, fateKey).serialize(), currSerNodeVal -> {
+ // We are only returning a non-null value for the following cases:
+ // 1) The existing NodeValue for fateId is exactly the same as the value set for the
+ // node if it doesn't yet exist:
+ // TStatus = TStatus.NEW, FateReservation = reservation, FateKey = fateKey
+ // This might occur if there was a ZK server fault and the same write is running a 2nd
+ // time
+ // 2) The existing NodeValue for fateId has:
+ // TStatus = TStatus.NEW, no FateReservation present, FateKey = fateKey
+ // The fateId is NEW/unseeded and not reserved, so we can allow it to be reserved
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ if (currNodeVal.status == TStatus.NEW && currNodeVal.isReservedBy(reservation)) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ return currSerNodeVal;
+ } else if (currNodeVal.status == TStatus.NEW && !currNodeVal.isReserved()) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ // NEW/unseeded transaction and not reserved, so we can allow it to be reserved
+ return new NodeValue(TStatus.NEW, reservation, fateKey).serialize();
+ } else {
+ log.trace(
+ "fate id {} tstatus {} fate key {} is reserved {} is either currently reserved "
+ + "or has already been seeded with work (non-NEW status), or both",
+ fateId, currNodeVal.status, currNodeVal.fateKey.orElse(null),
+ currNodeVal.isReserved());
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (nodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ // The uuid handles the case where there was a ZK server fault and the write for this thread
+ // went through but that was not acknowledged, and we are reading our own write for 2nd
+ // time.
+ if (!currNodeVal.isReserved() || currNodeVal.isReservedBy(reservation)) {
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ // Add the FateReservation to the node to reserve
+ return new NodeValue(currNodeVal.status, reservation, currFateKey).serialize();
+ } else {
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (newSerNodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
throw new IllegalStateException(e);
}
}
@Override
- protected Pair<TStatus,Optional<FateKey>> getStatusAndKey(FateId fateId) {
- final NodeValue node = getNode(fateId);
- return new Pair<>(node.status, node.fateKey);
+ public void deleteDeadReservations() {
+ for (Map.Entry<FateId,FateReservation> entry : getActiveReservations().entrySet()) {
+ FateId fateId = entry.getKey();
+ FateReservation reservation = entry.getValue();
+ if (isLockHeld.test(reservation.getLockID())) {
+ continue;
+ }
+ try {
+ zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ // Make sure the current node is still reserved and reserved with the expected reservation
+ // and it is dead
+ if (currNodeVal.isReservedBy(reservation)
+ && !isLockHeld.test(currNodeVal.reservation.orElseThrow().getLockID())) {
+ // Delete the reservation
+ log.trace("Deleted the dead reservation {} for fate id {}", reservation, fateId);
+ return new NodeValue(currNodeVal.status, null, currNodeVal.fateKey.orElse(null))
+ .serialize();
+ } else {
+ // No change
+ return null;
+ }
+ });
+ } catch (KeeperException | InterruptedException | AcceptableThriftTableOperationException e) {
Review Comment:
Good catch. Fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1702292110
##########
server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/meta/MetaFateMetrics.java:
##########
@@ -59,7 +60,8 @@ public void registerMetrics(MeterRegistry registry) {
@Override
protected ReadOnlyFateStore<FateMetrics<MetaFateMetricValues>> buildStore(ServerContext context) {
try {
- return new MetaFateStore<>(getFateRootPath(context), context.getZooReaderWriter());
+ return new MetaFateStore<>(getFateRootPath(context), context.getZooReaderWriter(),
+ AbstractFateStore.createDummyLockID(), null);
Review Comment:
Yeah, can leave this unresolved for making a follow on issue. I put the dummy lock in many places where a store was created, without regard to whether reservations would actually occur. A follow on issue can address this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1702220425
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -104,19 +109,108 @@ public FateId create() {
}
@Override
- protected void create(FateId fateId, FateKey key) {
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
- NodeExistsPolicy.FAIL);
- } catch (KeeperException | InterruptedException e) {
+ byte[] nodeVal = zk.mutateOrCreate(getTXPath(fateId),
+ new NodeValue(TStatus.NEW, reservation, fateKey).serialize(), currSerNodeVal -> {
+ // We are only returning a non-null value for the following cases:
+ // 1) The existing NodeValue for fateId is exactly the same as the value set for the
+ // node if it doesn't yet exist:
+ // TStatus = TStatus.NEW, FateReservation = reservation, FateKey = fateKey
+ // This might occur if there was a ZK server fault and the same write is running a 2nd
+ // time
+ // 2) The existing NodeValue for fateId has:
+ // TStatus = TStatus.NEW, no FateReservation present, FateKey = fateKey
+ // The fateId is NEW/unseeded and not reserved, so we can allow it to be reserved
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ if (currNodeVal.status == TStatus.NEW && currNodeVal.isReservedBy(reservation)) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ return currSerNodeVal;
+ } else if (currNodeVal.status == TStatus.NEW && !currNodeVal.isReserved()) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ // NEW/unseeded transaction and not reserved, so we can allow it to be reserved
+ return new NodeValue(TStatus.NEW, reservation, fateKey).serialize();
+ } else {
+ log.trace(
+ "fate id {} tstatus {} fate key {} is reserved {} is either currently reserved "
+ + "or has already been seeded with work (non-NEW status), or both",
+ fateId, currNodeVal.status, currNodeVal.fateKey.orElse(null),
+ currNodeVal.isReserved());
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (nodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
Review Comment:
Yeah, I considered adding the check to that test, but chose to add it to MultipleStoresIT.testReserveUnreserve() since there it also makes sure it doesn't affect the current active reservations at all.
Are you suggesting I also have the check in testAbsent()? Or that I move the check?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672827461
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Yes, the reservation column always exists and the filter is needed for the case where it is not reserved. The value stored if the fate id is not reserved is FateMutatorImpl.NOT_RESERVED = "", and this is the value set when a new fate id is created and when a reserved fate id is unreserved. Thinking about this again, it would probably be better to just delete/create the column as needed instead of storing a not reserved value. What do you think?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672908658
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Ah I see. Yeah I don't think I ever check if something is reserved like that and instead use a conditional mutation in FateMutatorImpl to reserve/unreserve. Should be pretty straightforward to change to create/delete the column on reserve/unreserve
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2361698043
@kevinrr888 -
> This was an existing TODO that you had marked in your "distributed FATE" WIP PR:
> // TODO 4131
> // TODO make the max time a function of the number of concurrent callers, as the number of
> // concurrent callers increases then increase the max wait time
> // TODO could support signaling within this instance for known events
> // TODO made the maxWait low so this would be responsive... that may put a lot of load in
> // the case there are lots of things waiting...
> I made the max wait = curr num callers in seconds. Does this address the first TODO? For the other two TODOs, I was not sure what was to be done, so I left them. Can these be safely deleted without creating follow on issue(s)? If follow on issues should be made, perhaps it would be best for you to write these or explain it here because I am not sure what should be done.
@keith-turner -
> Yeah you can delete those. I will look into it and open an issue if needed.
This may still need to be looked into just posting this as a reminder just in case
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1597510917
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -150,6 +150,39 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // TODO 4131 should this throw an exception if the id doesn't exist (status = UNKNOWN)?
+ FateMutator.Status status = newMutator(fateId).putReservedTx(fateId).tryMutate();
Review Comment:
This code needs to be refactored to use a UUID similar to the zookeeper code. The UUID covers the problem of a write making it the server, but the client not knowing the write made it. Both zookeeper and accumulo can have this problem and both can have a similar solution. The following examples show how the UUID can help address this problem.
1. FATE1 exists and is not currently reserved.
2. Thread1 attempts reserve FATE1 by writing `isreserved` only if current value is `notreserved`.
3. Thread2 attempts reserve FATE1 by writing `isreserved` only if current value is `notreserved`.
4. One of the writes succeeds and the other fails. However before the tablet server can respond to Thread1 or Thread2, it dies. This causes both thread to receive an unknown status.
5. After getting the UNKNOWN status, Thread1 reads FATE1 and sees it marked reserved however it does not know if it reserved FATE1 or another thread.
6. After getting the UNKNOWN status, Thread2 reads FATE1 and sees it marked reserved however it does not know if it reserved FATE1 or another thread.
In the above example, FATE1 is marked reserved but no thread will be working on it. If instead of setting a value of `isreserved` we instead set a per thread UUID, then the above situation could look like the following.
1. FATE1 exists and is not currently reserved. Its reserved column has a value of empty string.
2. Thread1 wants to reserve FATE1 and generates UUID1 for this purpose
3. Thread2 wants to reserve FATE1 and generates UUID2 for this purpose
4. Thread1 attempts reserve FATE1 by writing UUID1 as a value for the reserved column only if current value is empty.
5. Thread2 attempts reserve FATE1 by writing UUID2 as a value for the reserved column only if current value is empty.
6. The write for Thread2 succeeds and the write for Thread1 fails because the col val is not empty. However before the tablet server can respond to Thread1 or Thread2, it dies. This causes both thread to receive an unknown status.
7. After getting the UNKNOWN status, Thread1 reads FATE1 reserved column and sees UUID2 so it knows it did not reserve it.
8. After getting the UNKNOWN status, Thread2 reads FATE1 reserved column and sees UUID2 so it knows it did reserve it.
In addition to the UUID mentioned above, the persisted reservation will need another field and that is a process lock id. The UUID mentioned above is scoped very narrowly, it scoped to an individual reservation attempt and its used to handle the case of server applying write and then dying. The process lock id would be used to detect reservation that are held by a process that is no longer alive. The following is an example of how all of this works together.
1. A manager process starts in Accumulo and acquires a lock in ZK, let calls it PL1
2. The manager process reserves FATE1 setting the reservation column to UUID1:PL1. UUID1 is scoped to this reservation thread/code block as mentioned above.
3. After successfully reserving (could have seen unknown status and check for UUID1) the manager process dies
4. A new manager process is started and acquires a lock in ZK, let calls it PL2
5. The new manager process starts FATE, which starts a periodic task to look for reservations held by dead process. The periodic task sees FATE1 is reserved by PL1 which no longer has lock in ZK and is therefore presumed dead.
6. The periodic task issues a condtional mutation (or conditional update for ZK) to delete the reserved value for FATE1 only if the currently value is UUID1:PL1
Taking all of this together, may want to do the following.
1. Create a new type that represent the reservation value, maybe like
```java
class FateReseveration {
UUID reservationId;
ZooUtil.LockID processID; // TODO not sure if this is the best type for this
}
```
2. Use the above type internall in the User and Meta Fatestores for their reservations
3. Add methods to FateStore like `Stream<FateReservation> getActiveReservations()` and `void deleteReservation(FateReservation)`
4. Add a periodic task to Fate that calls the two methods above and looks active reservations held by dead processes and then deletes them.
5. When creating a new FateStore instance, pass in the lock of the process where it is running. For now this would mean when the manager creates a new FateStore instance it will pass in its lock. This will be used to set the processID on reservations.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1597510917
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -150,6 +150,39 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // TODO 4131 should this throw an exception if the id doesn't exist (status = UNKNOWN)?
+ FateMutator.Status status = newMutator(fateId).putReservedTx(fateId).tryMutate();
Review Comment:
This code needs to be refactored to use a UUID similar to the zookeeper code. The UUID covers the problem of a write making it the server, but the client not knowing the write made it. Both zookeeper and accumulo can have this problem and both can have a similar solution. The following examples show how the UUID can help address this problem.
1. FATE1 exists and is not currently reserved.
2. Thread1 attempts reserve FATE1 by writing `isreserved` only if current value is `notreserved`.
3. Thread2 attempts reserve FATE1 by writing `isreserved` only if current value is `notreserved`.
4. One of the writes succeeds and the other fails. However before the tablet server can respond to Thread1 or Thread2, it dies. This causes both thread to receive an unknown status.
5. After getting the UNKNOWN status, Thread1 reads FATE1 and sees it marked reserved however it does not know if it reserved FATE1 or another thread.
6. After getting the UNKNOWN status, Thread2 reads FATE1 and sees it marked reserved however it does not know if it reserved FATE1 or another thread.
In the above example, FATE1 is marked reserved but no thread will be working on it. If instead of setting a value of `isreserved` we instead set a per thread UUID, then the above situation could look like the following.
1. FATE1 exists and is not currently reserved. Its reserved column has a value of empty string.
2. Thread1 wants to reserve FATE1 and generates UUID1 for this purpose
3. Thread2 wants to reserve FATE1 and generates UUID2 for this purpose
4. Thread1 attempts reserve FATE1 by writing UUID1 as a value for the reserved column only if current value is empty.
5. Thread2 attempts reserve FATE1 by writing UUID2 as a value for the reserved column only if current value is empty.
6. The write for Thread2 succeeds and the write for Thread1 fails because the col val is not empty. However before the tablet server can respond to Thread1 or Thread2, it dies. This causes both thread to receive an unknown status.
7. After getting the UNKNOWN status, Thread1 reads FATE1 reserved column and sees UUID2 so it knows it did not reserve it.
8. After getting the UNKNOWN status, Thread2 reads FATE1 reserved column and sees UUID2 so it knows it did reserve it.
In addition to the UUID mentioned above, the persisted reservation will need another field and that is a process lock id. The UUID mentioned above is scoped very narrowly, it scoped to an individual reservation attempt and its used to handle the case of server applying write and then dying. The process lock id would be used to detect reservation that are held by a process that is no longer alive. The following is an example of how all of this works together.
1. A manager process starts in Accumulo and acquires a lock in ZK, let calls it PL1
2. The manager process reserves FATE1 setting the reservation column to UUID1:PL1. UUID1 is scoped to this reservation thread/code block as mentioned above.
3. After successfully reserving (could have seen unknown status and check for UUID1) the manager process dies
4. A new manager process is started and acquires a lock in ZK, let calls it PL2
5. The new manager process starts FATE, which starts a periodic task to look for reservations held by dead process. The periodic task sees FATE1 is reserved by PL1 which no longer has lock in ZK and is there presumed dead.
6. The periodic task issues a condtional mutation (or conditional update for ZK) to delete the reserved value for FATE1 only if the currently value is UUID1:PL1
Taking all of this together, may want to do the following.
1. Create a new type that represent the reservation value, maybe like
```java
class FateReseveration {
UUID reservationId;
ZooUtil.LockID processID; // TODO not sure if this is the best type for this
}
```
2. Use the above type internall in the User and Meta Fatestores for their reservations
3. Add methods to FateStore like `Stream<FateReservation> getActiveReservations()` and `void deleteReservation(FateReservation)`
4. Add a periodic task to Fate that calls the two methods above and looks active reservations held by dead processes and then deletes them.
5. When creating a new FateStore instance, pass in the lock of the process where it is running. For now this would mean when the manager creates a new FateStore instance it will pass in its lock. This will be used to set the processID on reservations.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688552235
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -106,13 +112,95 @@ public FateId create() {
@Override
protected void create(FateId fateId, FateKey key) {
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
+ zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, null, key).serialize(),
NodeExistsPolicy.FAIL);
} catch (KeeperException | InterruptedException e) {
throw new IllegalStateException(e);
}
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ // The uuid handles the case where there was a ZK server fault and the write for this thread
+ // went through but that was not acknowledged, and we are reading our own write for 2nd
+ // time.
+ if (!currNodeVal.isReserved() || (currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(reservation))) {
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ // Add the FateReservation to the node to reserve
+ return new NodeValue(currNodeVal.status, reservation, currFateKey).serialize();
+ } else {
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (newSerNodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return getNode(fateId).isReserved();
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ Map<FateId,FateReservation> activeReservations = new HashMap<>();
+
+ try {
+ for (String strTxId : zk.getChildren(path)) {
+ String txUUIDStr = strTxId.split("_")[1];
+ FateId fateId = FateId.from(fateInstanceType, txUUIDStr);
+ if (isReserved(fateId)) {
+ FateReservation reservation = getNode(fateId).reservation.orElseThrow();
+ activeReservations.put(fateId, reservation);
+ }
+ }
+ } catch (KeeperException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ return activeReservations;
+ }
+
+ @Override
+ public void deleteDeadReservations() {
+ for (Map.Entry<FateId,FateReservation> entry : getActiveReservations().entrySet()) {
+ FateId fateId = entry.getKey();
+ FateReservation reservation = entry.getValue();
Review Comment:
Nice improvement. Added
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -224,9 +316,21 @@ public void setStatus(TStatus status) {
verifyReserved(true);
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(status).serialize(),
- NodeExistsPolicy.OVERWRITE);
- } catch (KeeperException | InterruptedException e) {
+ zk.mutateExisting(getTXPath(fateId), currSerializedData -> {
+ NodeValue currNodeVal = new NodeValue(currSerializedData);
+ // Ensure the FateId is reserved in ZK, and it is reserved with the expected reservation
+ if (currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(this.reservation)) {
Review Comment:
Yeah I do that check quite a bit... I think it's a good change and is more clear. Added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688549092
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -168,18 +173,16 @@ public void runnable(AtomicBoolean keepWaiting, Consumer<FateId> idConsumer) {
var transactions = Stream.concat(inProgress, other);
transactions.filter(fateIdStatus -> isRunnable(fateIdStatus.getStatus()))
.map(FateIdStatus::getFateId).filter(fateId -> {
- synchronized (AbstractFateStore.this) {
- var deferredTime = deferred.get(fateId);
- if (deferredTime != null) {
- if (deferredTime.elapsed().isNegative()) {
- // negative elapsed time indicates the deferral time is in the future
- return false;
- } else {
- deferred.remove(fateId);
- }
+ var deferredTime = deferred.get(fateId);
+ if (deferredTime != null) {
+ if (deferredTime.elapsed().isNegative()) {
+ // negative elapsed time indicates the deferral time is in the future
+ return false;
+ } else {
+ deferred.remove(fateId);
}
- return !reserved.contains(fateId);
}
+ return !isReserved(fateId);
Review Comment:
Addressed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1670964150
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Looking into this a bit more, I think the reservation column may always exist and have a not reserved value. So is the filter needed for that case?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1692199252
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
> Multiple threads would occassionally reserve the same fate id.
Not sure what the problem was, but its kinda neat you detected that bug. Where you running an IT when you saw multiple threads reserve the same fate id?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1702301680
##########
core/src/main/java/org/apache/accumulo/core/fate/user/RowFateStatusFilter.java:
##########
@@ -53,19 +56,28 @@ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> op
}
@Override
- public boolean accept(Key k, Value v) {
- var tstatus = ReadOnlyFateStore.TStatus.valueOf(v.toString());
- return valuesToAccept.contains(tstatus);
+ protected boolean filter(Text currentRow, List<Key> keys, List<Value> values) {
+ for (int i = 0; i < keys.size(); i++) {
+ Key key = keys.get(i);
+ if (key.getColumnQualifier()
+ .equals(FateSchema.TxColumnFamily.STATUS_COLUMN.getColumnQualifier())
Review Comment:
Changed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1706151889
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
This conversion is really long, but it seems like everything in it is resolved. @kevinrr888 can you resolve this if you also don't see anything outstanding in the conversations?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2272215995
@kevinrr888 it seems like the only unresolved conversations are covered by your list of issues to open. I just left those open for now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672878600
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
>If going with the NOT_RESERVED value approach it would be good to make the validation of the column value more strict, will make a comment about that elsewhere.
Nevermind the code does not work exactly the way I thought it did. I was thinking when reading reservations could do something like the following
```java
if(value look like <lock id>:<uuid) {
// process reserved
} else if (value is NOT_RESERVED) {
//process not reserved
} else {
throw new IllegalArgumentException("invalid reseved column value");
}
```
However there is not a place in the code that reads reservations in this way. Was thinking the code should fail if the reserved column value was something like '`1234` because it does not look like a reservation and it also does not look like NOT_RESERVED
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688550936
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -298,7 +299,13 @@ private Optional<FateId> create(FateKey fateKey) {
@Override
public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
- FateId fateId = fateIdGenerator.fromTypeAndKey(getInstanceType(), fateKey);
+ // TODO 4131 not confident about this new implementation of createAndReserve.
Review Comment:
Implemented in User/MetaFateStore.createAndReserve() (not currently fully working - see comment https://github.com/apache/accumulo/pull/4524#issuecomment-2246036407). Thank you for this detailed description.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688548666
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
+ .orElse(null);
+ if (persistedRes != null && persistedRes.equals(reservation)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ }
Review Comment:
I wrote what would be the equivalent stream needed, and it was pretty complicated. I personally think the way it is now is a bit more readable/easy to follow. Since it would be the same, I'm gonna leave it as is.
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -106,13 +112,95 @@ public FateId create() {
@Override
protected void create(FateId fateId, FateKey key) {
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
+ zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, null, key).serialize(),
NodeExistsPolicy.FAIL);
} catch (KeeperException | InterruptedException e) {
throw new IllegalStateException(e);
}
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ // The uuid handles the case where there was a ZK server fault and the write for this thread
+ // went through but that was not acknowledged, and we are reading our own write for 2nd
+ // time.
+ if (!currNodeVal.isReserved() || (currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(reservation))) {
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ // Add the FateReservation to the node to reserve
+ return new NodeValue(currNodeVal.status, reservation, currFateKey).serialize();
+ } else {
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (newSerNodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return getNode(fateId).isReserved();
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ Map<FateId,FateReservation> activeReservations = new HashMap<>();
+
+ try {
+ for (String strTxId : zk.getChildren(path)) {
+ String txUUIDStr = strTxId.split("_")[1];
+ FateId fateId = FateId.from(fateInstanceType, txUUIDStr);
+ if (isReserved(fateId)) {
+ FateReservation reservation = getNode(fateId).reservation.orElseThrow();
Review Comment:
Addressed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700586434
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -421,14 +569,30 @@ private Optional<FateKey> deserializeFateKey(DataInputBuffer buffer) throws IOEx
return Optional.empty();
}
+ private Optional<FateReservation> deserializeFateReservation(DataInputBuffer buffer)
+ throws IOException {
+ int length = buffer.readInt();
+ if (length > 0) {
Review Comment:
Added check to both deserializeFateKey() and deserializeFateReservation()
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700585605
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -363,13 +498,23 @@ protected Stream<FateIdStatus> getTransactions(Set<TStatus> statuses) {
Stream<FateIdStatus> stream = zk.getChildren(path).stream().map(strTxid -> {
String txUUIDStr = strTxid.split("_")[1];
FateId fateId = FateId.from(fateInstanceType, txUUIDStr);
- // Memoizing for two reasons. First the status may never be requested, so in that case avoid
- // the lookup. Second, if its requested multiple times the result will always be consistent.
- Supplier<TStatus> statusSupplier = Suppliers.memoize(() -> _getStatus(fateId));
+ // Memoizing for two reasons. First the status or reservation may never be requested, so
+ // in that case avoid the lookup. Second, if it's requested multiple times the result will
+ // always be consistent.
+ Supplier<Pair<TStatus,Optional<FateReservation>>> statusAndResSupplier =
+ Suppliers.memoize(() -> {
+ NodeValue zkNode = getNode(fateId);
+ return new Pair<>(zkNode.status, zkNode.reservation);
+ });
Review Comment:
Simplified
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1702229039
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -104,19 +109,108 @@ public FateId create() {
}
@Override
- protected void create(FateId fateId, FateKey key) {
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
- NodeExistsPolicy.FAIL);
- } catch (KeeperException | InterruptedException e) {
+ byte[] nodeVal = zk.mutateOrCreate(getTXPath(fateId),
+ new NodeValue(TStatus.NEW, reservation, fateKey).serialize(), currSerNodeVal -> {
+ // We are only returning a non-null value for the following cases:
+ // 1) The existing NodeValue for fateId is exactly the same as the value set for the
+ // node if it doesn't yet exist:
+ // TStatus = TStatus.NEW, FateReservation = reservation, FateKey = fateKey
+ // This might occur if there was a ZK server fault and the same write is running a 2nd
+ // time
+ // 2) The existing NodeValue for fateId has:
+ // TStatus = TStatus.NEW, no FateReservation present, FateKey = fateKey
+ // The fateId is NEW/unseeded and not reserved, so we can allow it to be reserved
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ if (currNodeVal.status == TStatus.NEW && currNodeVal.isReservedBy(reservation)) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ return currSerNodeVal;
+ } else if (currNodeVal.status == TStatus.NEW && !currNodeVal.isReserved()) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ // NEW/unseeded transaction and not reserved, so we can allow it to be reserved
+ return new NodeValue(TStatus.NEW, reservation, fateKey).serialize();
+ } else {
+ log.trace(
+ "fate id {} tstatus {} fate key {} is reserved {} is either currently reserved "
+ + "or has already been seeded with work (non-NEW status), or both",
+ fateId, currNodeVal.status, currNodeVal.fateKey.orElse(null),
+ currNodeVal.isReserved());
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (nodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
Review Comment:
The test in MultipleStoresIT is fine, I had not looked at that yet. I was looking in the elasticity branch to see if this case was tested there. As long tryReserve for non existent is tested somewhere its good.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1670934639
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -298,7 +299,13 @@ private Optional<FateId> create(FateKey fateKey) {
@Override
public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
- FateId fateId = fateIdGenerator.fromTypeAndKey(getInstanceType(), fateKey);
+ // TODO 4131 not confident about this new implementation of createAndReserve.
Review Comment:
The way this method previously worked and the way the code was structured it was driven by the lock object being in AbstractFateStore. Now that there is no longer a lock we want to implement the following algorithm for both stores using conditional updates (@cshannon you added the fatekey feature, does the following algorithm seem ok to you?).
1. var fateId = FateId fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
2. in the store conditionally create fateId setting the status, reservation and fateKey atomically at creation time. The condition should check that nothing currently exist for the fateId, its expected to be absent.
3. If the conditional update fails for some reason, read the fateId and see if it already has the expected status, fateKey and reservation. If it does, then probably running for a second time and already reserved. If it does not then need to take the following actions.
1. If the fateId exists and has the same fateKey, but different status (should have NEW status) or reservation than expected then return Optional.empty(). This is the case where another thread has already created and reserved using the fateKey so there is nothing to do.
2. If the fateId exists and has different fateKey or no fateKey then this represents an unexpected collision ( this unexpected with the 128 bit keys) so throw an exception in this case.
5. If the conditional update succeed , then we can return a non empty Optional w/ a FateTxStore obj.
Now that the lock object is no longer needed in AbstractFateStore it may make sense to completely push the full implementation of the `createAndReserve` method into MetaFateStore and UserFateStore and have nothing in AbstractFateStore for the method. The reason I am thinking this may be better is because the algorithm above is based on conditional updates now and how ZK and accumulo do conditional updates is very different. However there may still be opportunities to share common code, so not sure of the best way to structure the code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1671222699
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -114,38 +131,26 @@ public static Object deserialize(byte[] ser) {
}
}
- /**
- * Attempt to reserve the fate transaction.
- *
- * @param fateId The FateId
- * @return An Optional containing the FateTxStore if the transaction was successfully reserved, or
- * an empty Optional if the transaction was already reserved.
- */
- @Override
- public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
- synchronized (this) {
- if (!reserved.contains(fateId)) {
- return Optional.of(reserve(fateId));
- }
- return Optional.empty();
- }
- }
-
@Override
public FateTxStore<T> reserve(FateId fateId) {
- synchronized (AbstractFateStore.this) {
- while (reserved.contains(fateId)) {
- try {
- AbstractFateStore.this.wait(100);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new IllegalStateException(e);
- }
+ Preconditions.checkState(!_getStatus(fateId).equals(TStatus.UNKNOWN),
+ "Attempted to reserve a tx that does not exist: " + fateId);
Review Comment:
Something could exist and be reserved when we enter the method and be deleted while we are waiting to reserve it. This will do an RPC to read data and then tryRerserve will do another RPC. For efficiency and correctness would be better to only do this check in the case when tryReserve returns empty. If the check is done in the loop it will handle the case of something being deleted while waiting. Also if the check is done in the loop, then when tryReserve is successful then the 2nd RPC will never be made.
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -361,90 +356,108 @@ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
protected abstract Optional<FateKey> getKey(FateId fateId);
- protected abstract FateTxStore<T> newFateTxStore(FateId fateId, boolean isReserved);
-
- protected abstract FateInstanceType getInstanceType();
+ protected abstract FateTxStore<T> newUnreservedFateTxStore(FateId fateId);
protected abstract class AbstractFateTxStoreImpl<T> implements FateTxStore<T> {
protected final FateId fateId;
- protected final boolean isReserved;
+ protected boolean deleted;
+ protected FateReservation reservation;
protected TStatus observedStatus = null;
- protected AbstractFateTxStoreImpl(FateId fateId, boolean isReserved) {
+ protected AbstractFateTxStoreImpl(FateId fateId) {
+ this.fateId = fateId;
+ this.deleted = false;
+ this.reservation = null;
+ }
+
+ protected AbstractFateTxStoreImpl(FateId fateId, FateReservation reservation) {
this.fateId = fateId;
- this.isReserved = isReserved;
+ this.deleted = false;
+ this.reservation = Objects.requireNonNull(reservation);
+ }
+
+ protected boolean isReserved() {
+ return this.reservation != null;
}
@Override
public TStatus waitForStatusChange(EnumSet<TStatus> expected) {
- Preconditions.checkState(!isReserved,
- "Attempted to wait for status change while reserved " + fateId);
- while (true) {
+ Preconditions.checkState(!isReserved(),
+ "Attempted to wait for status change while reserved: " + fateId);
+ verifyReserved(false);
- long countBefore = unreservedNonNewCount.getCount();
+ int currNumCallers = concurrentStatusChangeCallers.incrementAndGet();
Review Comment:
This change makes the polling more efficient in the case of many things waiting for status change. All the code following this should be placed in a try catch, something like the following
```java
try {
// polling loop
} finally {
concurrentStatusChangeCallers.decrementAndGet();
}
```
this ensures that its decremented even when an exception happens.
One thing that is being lost in this change is the in memory signalling. Not something for this PR, but would be good to have something that does some in memory signalling and is efficient w/ the polling as the number of pollers increases. The less latency there is in the impl of the method the faster things like create table, bulk import, etc will be to complete from the user API perspective. The in memory signalling can help reduce latency.
##########
core/src/main/java/org/apache/accumulo/core/fate/FateStore.java:
##########
@@ -107,11 +118,155 @@ interface FateTxStore<T> extends ReadOnlyFateTxStore<T> {
void unreserve(Duration deferTime);
}
+ /**
+ * The value stored to indicate a FATE transaction ID ({@link FateId}) has been reserved
+ */
+ class FateReservation {
+
+ // The LockID (provided by the Manager running the FATE which uses this store) which is used for
+ // identifying dead Managers, so their reservations can be deleted and picked up again since
+ // they can no longer be worked on.
+ private final ZooUtil.LockID lockID; // TODO 4131 not sure if this is the best type for this
+ // The UUID generated on a reservation attempt (tryReserve()) used to uniquely identify that
+ // attempt. This is useful for the edge case where the reservation is sent to the server
+ // (Tablet Server for UserFateStore and the ZooKeeper Server for MetaFateStore), but the server
+ // dies before the store receives the response. It allows us to determine if the reservation
+ // was successful and was written by this reservation attempt (could have been successfully
+ // reserved by another attempt or not reserved at all, in which case, we wouldn't want to
+ // expose a FateTxStore).
+ private final UUID reservationUUID;
+ private final byte[] serialized;
+ private static final Pattern UUID_PATTERN =
+ Pattern.compile("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$");
+ private static final Pattern LOCKID_PATTERN = Pattern.compile("^.+/.+\\$[0-9a-fA-F]+$");
+
+ private FateReservation(ZooUtil.LockID lockID, UUID reservationUUID) {
+ this.lockID = Objects.requireNonNull(lockID);
+ this.reservationUUID = Objects.requireNonNull(reservationUUID);
+ this.serialized = serialize(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(ZooUtil.LockID lockID, UUID reservationUUID) {
+ return new FateReservation(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(byte[] serialized) {
+ try (DataInputBuffer buffer = new DataInputBuffer()) {
+ buffer.reset(serialized, serialized.length);
+ ZooUtil.LockID lockID = new ZooUtil.LockID("", buffer.readUTF());
+ UUID reservationUUID = UUID.fromString(buffer.readUTF());
+ return new FateReservation(lockID, reservationUUID);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static FateReservation from(String fateReservationStr) {
+ if (isFateReservation(fateReservationStr)) {
+ String[] fields = fateReservationStr.split(":");
+ ZooUtil.LockID lockId = new ZooUtil.LockID("", fields[0]);
+ UUID reservationUUID = UUID.fromString(fields[1]);
+ return new FateReservation(lockId, reservationUUID);
+ } else {
+ throw new IllegalArgumentException(
+ "Tried to create a FateReservation from an invalid string: " + fateReservationStr);
+ }
+ }
+
+ /**
+ *
+ * @param fateReservationStr the string from a call to {@link FateReservation#toString()}
+ * @return true if the string represents a valid FateReservation object, false otherwise
+ */
+ public static boolean isFateReservation(String fateReservationStr) {
+ if (fateReservationStr != null) {
+ String[] fields = fateReservationStr.split(":");
+ if (fields.length == 2) {
+ return LOCKID_PATTERN.matcher(fields[0]).matches()
+ && UUID_PATTERN.matcher(fields[1]).matches();
Review Comment:
Could use UuidUtil.isUUID here
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -106,13 +112,95 @@ public FateId create() {
@Override
protected void create(FateId fateId, FateKey key) {
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
+ zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, null, key).serialize(),
NodeExistsPolicy.FAIL);
} catch (KeeperException | InterruptedException e) {
throw new IllegalStateException(e);
}
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ // The uuid handles the case where there was a ZK server fault and the write for this thread
+ // went through but that was not acknowledged, and we are reading our own write for 2nd
+ // time.
+ if (!currNodeVal.isReserved() || (currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(reservation))) {
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ // Add the FateReservation to the node to reserve
+ return new NodeValue(currNodeVal.status, reservation, currFateKey).serialize();
+ } else {
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (newSerNodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return getNode(fateId).isReserved();
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ Map<FateId,FateReservation> activeReservations = new HashMap<>();
+
+ try {
+ for (String strTxId : zk.getChildren(path)) {
+ String txUUIDStr = strTxId.split("_")[1];
+ FateId fateId = FateId.from(fateInstanceType, txUUIDStr);
+ if (isReserved(fateId)) {
+ FateReservation reservation = getNode(fateId).reservation.orElseThrow();
+ activeReservations.put(fateId, reservation);
+ }
+ }
+ } catch (KeeperException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ return activeReservations;
+ }
+
+ @Override
+ public void deleteDeadReservations() {
+ for (Map.Entry<FateId,FateReservation> entry : getActiveReservations().entrySet()) {
+ FateId fateId = entry.getKey();
+ FateReservation reservation = entry.getValue();
Review Comment:
Can avoid calling `zk.mutateExisting` and reading from zookeeper in the case where the lock is held with a check like the following.
```suggestion
FateReservation reservation = entry.getValue();
if(isLockHeld(reservation.getLockID())) {
continue;
}
```
##########
core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java:
##########
@@ -175,6 +176,21 @@ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
}
return txStore;
}
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return store.isReserved(fateId);
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ return store.getActiveReservations();
+ }
+
+ @Override
+ public void deleteDeadReservations() {
+ store.deleteDeadReservations();
Review Comment:
Probably would not go here, but it would be nice to have consistent logging somehow for each dead reservation that is detected and deleted.
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -55,7 +60,7 @@
//TODO use zoocache? - ACCUMULO-1297
//TODO handle zookeeper being down gracefully - ACCUMULO-1297
-
+// TODO 4131 noticed this class is not in the fate.zookeeper package. Should it be?
Review Comment:
Yeah it probably should be. That would be consistent with UserFateStore, which is in a sub package.
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -224,9 +316,21 @@ public void setStatus(TStatus status) {
verifyReserved(true);
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(status).serialize(),
- NodeExistsPolicy.OVERWRITE);
- } catch (KeeperException | InterruptedException e) {
+ zk.mutateExisting(getTXPath(fateId), currSerializedData -> {
+ NodeValue currNodeVal = new NodeValue(currSerializedData);
+ // Ensure the FateId is reserved in ZK, and it is reserved with the expected reservation
+ if (currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(this.reservation)) {
Review Comment:
Not sure if this change is worthwhile. Saw this pattern a few times, could create a method in NodeValue and push the check there to shorten code elsewhere.
```suggestion
if (currNodeVal.isReservedBy(this.reservation)) {
```
##########
core/src/main/java/org/apache/accumulo/core/fate/user/FateMutatorImpl.java:
##########
@@ -79,6 +82,33 @@ public FateMutator<T> putCreateTime(long ctime) {
return this;
}
+ @Override
+ public FateMutator<T> putReservedTx(FateStore.FateReservation reservation) {
+ Condition condition = new Condition(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier()).setValue(NOT_RESERVED);
+ mutation.addCondition(condition);
+ TxColumnFamily.RESERVATION_COLUMN.put(mutation, new Value(reservation.toString()));
Review Comment:
Its nice to avoid calling toString() when its return value is used with serialized data as the expectations around toString() are not well defined. For example someone could change the toString impl to improve a log message or something like that and not realize the implications for serialized data.
```suggestion
TxColumnFamily.RESERVATION_COLUMN.put(mutation, new Value(reservation.getSerialized()));
```
##########
core/src/main/java/org/apache/accumulo/core/fate/user/FateMutatorImpl.java:
##########
@@ -79,6 +82,33 @@ public FateMutator<T> putCreateTime(long ctime) {
return this;
}
+ @Override
+ public FateMutator<T> putReservedTx(FateStore.FateReservation reservation) {
+ Condition condition = new Condition(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier()).setValue(NOT_RESERVED);
+ mutation.addCondition(condition);
+ TxColumnFamily.RESERVATION_COLUMN.put(mutation, new Value(reservation.toString()));
+ return this;
+ }
+
+ @Override
+ public FateMutator<T> putUnreserveTx(FateStore.FateReservation reservation) {
+ Condition condition = new Condition(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier()).setValue(reservation.toString());
Review Comment:
```suggestion
TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier()).setValue(reservation.getSerialized());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1671308191
##########
core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java:
##########
@@ -175,6 +176,21 @@ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
}
return txStore;
}
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return store.isReserved(fateId);
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ return store.getActiveReservations();
+ }
+
+ @Override
+ public void deleteDeadReservations() {
+ store.deleteDeadReservations();
Review Comment:
Not sure if this is workable, maybe the method could take a consumer that is passes found dead reservations to like : `deleteDeadReservations(Consumer<FateReservation> foundDead)` then this could pass in a consumer that does logging. Alternatively could just put the same log stmts in each impl of the store.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672979188
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Made this comment as an example
https://github.com/apache/accumulo/pull/4524/files?diff=unified&w=1#r1672972761
I am not sure if MetaFateStore has any problems, was only thinking through the UserFateStore code. Ideally both should be very strict w/ reservation parsing.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2273932737
The `TODO 4131` comments have now been removed and I believe this PR is ready to be merged :tada:
I was planning on creating the follow-on issues after this is merged in as a batch of new issues.
As suggested by @ctubbsii in our meeting today, I will post a list of all the outstanding changes to be made below. The trivial ones, I will just make a PR for without creating an issue, the more involved follow-ons will be tracked in an issue.
**_Here is the list of changes to be made:_**
_To be tracked by issue:_
- [ ] Refactor MultipleStoresIT to function more similarly to other Fate tests like FateIT
- [ ] Deprecate AbstractFateStore.createDummyLockID(): this includes creating a path in ZK for utilities to get a ZK lock, changing `admin fate fail` and `admin fate delete` commands to get a LockID at this path and no longer require the Manager to be down, and only use a LockID for a store if write operations are expected: the store should fail on write operations if writes are not expected.
- [ ] A single ZK node for all fate data for each fate id
- [ ] Replace AFS.verifyReserved with a condition on the RESERVATION_COLUMN to verify that it is reserved
- [ ] Make WorkFinder and the TransactionRunners critical to the Manager
- [ ] Refactor how the RESERVATION_COLUMN works for UserFateStore: create/delete column on reserve/unreserve
_Will do in one or a few small PRs:_
- [ ] Add a toString() to FateKey
- [ ] Move MetaFateStore to org.apache.accumulo.core.fate.zookeeper
- [ ] Periodic clean up of dead reservations should be increased from every 30 seconds to every few minutes
- [ ] Add additional fate test case suggested in https://github.com/apache/accumulo/pull/4524#discussion_r1702301996
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688547821
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -360,7 +380,27 @@ public Fate(T environment, FateStore<T> store, Function<Repo<T>,String> toLogStr
}
}
}, 3, SECONDS));
- this.executor = pool;
+ this.transactionExecutor = pool;
+
+ // Create a dead reservation cleaner for this store that will periodically (every 30 seconds)
+ // clean up reservations held by dead processes, if they exist. Only created if a dead
+ // reservation cleaner is not already running for the given store type.
+ // TODO 4131 periodic cleanup runs every 30 seconds
+ // Should this be longer? Shorter? A configurable Property? A function of something?
+ ScheduledExecutorService deadResCleanerExecutor = ThreadPools.getServerThreadPools()
+ .createScheduledExecutorService(1, store.type() + "-dead-reservation-cleaner-pool");
+ if ((store.type() == FateInstanceType.USER && !userDeadReservationCleanerRunning)
+ || (store.type() == FateInstanceType.META && !metaDeadReservationCleanerRunning)) {
+ ScheduledFuture<?> deadReservationCleaner = deadResCleanerExecutor
+ .scheduleWithFixedDelay(new DeadReservationCleaner(), 3, 30, SECONDS);
+ ThreadPools.watchCriticalScheduledTask(deadReservationCleaner);
+ if (store.type() == FateInstanceType.USER) {
+ userDeadReservationCleanerRunning = true;
+ } else if (store.type() == FateInstanceType.META) {
+ metaDeadReservationCleanerRunning = true;
+ }
+ }
+ this.deadResCleanerExecutor = deadResCleanerExecutor;
Review Comment:
yes, it won't ever be used. I did it this way for a simpler shutdown method (no need to check for null), but you are right it would be better if this wasn't created in the first place. Changed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 merged PR #4524:
URL: https://github.com/apache/accumulo/pull/4524
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1692192823
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -73,9 +75,12 @@ public class Fate<T> {
private final FateStore<T> store;
private final T environment;
private final ScheduledThreadPoolExecutor fatePoolWatcher;
- private final ExecutorService executor;
+ private final ExecutorService transactionExecutor;
+ private final ExecutorService deadResCleanerExecutor;
private static final EnumSet<TStatus> FINISHED_STATES = EnumSet.of(FAILED, SUCCESSFUL, UNKNOWN);
+ private static boolean userDeadReservationCleanerRunning = false;
+ private static boolean metaDeadReservationCleanerRunning = false;
Review Comment:
In #3964 I was thinking of having this [PartitionData](https://github.com/apache/accumulo/blob/208e39927310f8ff7dd5978f2794e26bee6be1b4/core/src/main/java/org/apache/accumulo/core/manager/PartitionData.java) object that would be passed to Fate from the manager. Fate could use that to partition fate operations to run and to determine if it should run a singleton service using the `PartitionData.shouldRun` method. Was thinking that PatitionData could be passed to all the managers high level services (like Fate, CompactionCoordinator, TabletGroupWatcher) and that all could use it to partition their data in their own way.
With something like that, in test if you create 3 fate instances then could pass them `ParitionData(0,3)`, `ParitionData(1,3)`, and `ParitionData(2,3)`. It probably does not makes sense to add that in this PR as it will not partition the fate ids at this point.
For this PR we could simplify things and have a boolean argument on the Fate constructor that indicates if the dead reservation detection should be started and not have any static variable that attempt to enforce one per process.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1692342340
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -67,27 +76,36 @@ public FateId fromTypeAndKey(FateInstanceType instanceType, FateKey fateKey) {
}
};
- protected final Set<FateId> reserved;
+ // The ZooKeeper lock for the process that's running this store instance
+ protected final ZooUtil.LockID lockID;
+ protected final Predicate<ZooUtil.LockID> isLockHeld;
protected final Map<FateId,NanoTime> deferred;
+ protected final FateIdGenerator fateIdGenerator;
private final int maxDeferred;
private final AtomicBoolean deferredOverflow = new AtomicBoolean();
- private final FateIdGenerator fateIdGenerator;
-
- // This is incremented each time a transaction was unreserved that was non new
- protected final SignalCount unreservedNonNewCount = new SignalCount();
// This is incremented each time a transaction is unreserved that was runnable
- protected final SignalCount unreservedRunnableCount = new SignalCount();
+ private final SignalCount unreservedRunnableCount = new SignalCount();
+
+ // Keeps track of the number of concurrent callers to waitForStatusChange()
+ private final AtomicInteger concurrentStatusChangeCallers = new AtomicInteger(0);
public AbstractFateStore() {
- this(DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ this(null, null, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
}
- public AbstractFateStore(int maxDeferred, FateIdGenerator fateIdGenerator) {
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld) {
+ this(lockID, isLockHeld, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ }
+
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld,
+ int maxDeferred, FateIdGenerator fateIdGenerator) {
this.maxDeferred = maxDeferred;
this.fateIdGenerator = Objects.requireNonNull(fateIdGenerator);
- this.reserved = new HashSet<>();
- this.deferred = new HashMap<>();
+ this.deferred = Collections.synchronizedMap(new HashMap<>());
+ this.lockID = Objects.requireNonNullElseGet(lockID, AbstractFateStore::createDummyLockID);
Review Comment:
I did not know this function existed. Went looked at what new methods exists in Objects and found checkFromIndexSize which could be useful in other places in the code base for efficiently checking bounds. Opened https://github.com/apache/accumulo-access/pull/80 as a result of finding these new methods. I think we can use these index bounds checking methods in other places.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700582782
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -104,19 +109,108 @@ public FateId create() {
}
@Override
- protected void create(FateId fateId, FateKey key) {
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
- NodeExistsPolicy.FAIL);
- } catch (KeeperException | InterruptedException e) {
+ byte[] nodeVal = zk.mutateOrCreate(getTXPath(fateId),
+ new NodeValue(TStatus.NEW, reservation, fateKey).serialize(), currSerNodeVal -> {
+ // We are only returning a non-null value for the following cases:
+ // 1) The existing NodeValue for fateId is exactly the same as the value set for the
+ // node if it doesn't yet exist:
+ // TStatus = TStatus.NEW, FateReservation = reservation, FateKey = fateKey
+ // This might occur if there was a ZK server fault and the same write is running a 2nd
+ // time
+ // 2) The existing NodeValue for fateId has:
+ // TStatus = TStatus.NEW, no FateReservation present, FateKey = fateKey
+ // The fateId is NEW/unseeded and not reserved, so we can allow it to be reserved
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ if (currNodeVal.status == TStatus.NEW && currNodeVal.isReservedBy(reservation)) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ return currSerNodeVal;
+ } else if (currNodeVal.status == TStatus.NEW && !currNodeVal.isReserved()) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ // NEW/unseeded transaction and not reserved, so we can allow it to be reserved
+ return new NodeValue(TStatus.NEW, reservation, fateKey).serialize();
+ } else {
+ log.trace(
+ "fate id {} tstatus {} fate key {} is reserved {} is either currently reserved "
+ + "or has already been seeded with work (non-NEW status), or both",
+ fateId, currNodeVal.status, currNodeVal.fateKey.orElse(null),
+ currNodeVal.isReserved());
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (nodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
Review Comment:
Ah okay, I thought we would want to throw the exception in the case the node doesn't exist. Changed and added check to MultipleStoresIT.testReserveUnreserve(). I see now that the tryReserve() for UFS and MFS were different for this case: UFS would return empty optional, MFS would throw an error, so this change is good and keeps them consistent. I remember noticing this difference a while ago when I was writing this, but must have forgot to write down/ask which functionality is expected. Glad this was spotted again.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700580931
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -67,27 +76,36 @@ public FateId fromTypeAndKey(FateInstanceType instanceType, FateKey fateKey) {
}
};
- protected final Set<FateId> reserved;
+ // The ZooKeeper lock for the process that's running this store instance
+ protected final ZooUtil.LockID lockID;
+ protected final Predicate<ZooUtil.LockID> isLockHeld;
protected final Map<FateId,NanoTime> deferred;
+ protected final FateIdGenerator fateIdGenerator;
private final int maxDeferred;
private final AtomicBoolean deferredOverflow = new AtomicBoolean();
- private final FateIdGenerator fateIdGenerator;
-
- // This is incremented each time a transaction was unreserved that was non new
- protected final SignalCount unreservedNonNewCount = new SignalCount();
// This is incremented each time a transaction is unreserved that was runnable
- protected final SignalCount unreservedRunnableCount = new SignalCount();
+ private final SignalCount unreservedRunnableCount = new SignalCount();
+
+ // Keeps track of the number of concurrent callers to waitForStatusChange()
+ private final AtomicInteger concurrentStatusChangeCallers = new AtomicInteger(0);
public AbstractFateStore() {
- this(DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ this(null, null, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
}
- public AbstractFateStore(int maxDeferred, FateIdGenerator fateIdGenerator) {
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld) {
+ this(lockID, isLockHeld, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ }
+
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld,
+ int maxDeferred, FateIdGenerator fateIdGenerator) {
this.maxDeferred = maxDeferred;
this.fateIdGenerator = Objects.requireNonNull(fateIdGenerator);
- this.reserved = new HashSet<>();
- this.deferred = new HashMap<>();
+ this.deferred = Collections.synchronizedMap(new HashMap<>());
+ this.lockID = Objects.requireNonNullElseGet(lockID, AbstractFateStore::createDummyLockID);
Review Comment:
This is simple to address in this PR; changed to only accept non-null and changed to caller to create the dummy lock instead
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700579471
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -73,9 +75,12 @@ public class Fate<T> {
private final FateStore<T> store;
private final T environment;
private final ScheduledThreadPoolExecutor fatePoolWatcher;
- private final ExecutorService executor;
+ private final ExecutorService transactionExecutor;
+ private final ExecutorService deadResCleanerExecutor;
private static final EnumSet<TStatus> FINISHED_STATES = EnumSet.of(FAILED, SUCCESSFUL, UNKNOWN);
+ private static boolean userDeadReservationCleanerRunning = false;
+ private static boolean metaDeadReservationCleanerRunning = false;
Review Comment:
:+1: re the PartitionData object
Got it, I've changed it to take a boolean argument for whether the cleaner should run.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700588116
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -120,35 +127,161 @@ public FateId getFateId() {
}
@Override
- protected void create(FateId fateId, FateKey fateKey) {
- final int maxAttempts = 5;
-
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+ Optional<FateTxStore<T>> txStore = Optional.empty();
+ int maxAttempts = 5;
+ FateMutator.Status status = null;
+
+ // We first need to write the initial/unreserved value for the reservation column
+ // Only need to retry if it is UNKNOWN
for (int attempt = 0; attempt < maxAttempts; attempt++) {
-
- if (attempt >= 1) {
- log.debug("Failed to create transaction with fateId {} and fateKey {}, trying again",
- fateId, fateKey);
- UtilWaitThread.sleep(100);
+ status = newMutator(fateId).putInitReservationVal().tryMutate();
+ if (status != FateMutator.Status.UNKNOWN) {
+ break;
}
+ UtilWaitThread.sleep(100);
+ }
+ for (int attempt = 0; attempt < maxAttempts; attempt++) {
Review Comment:
I agree it shouldn't be doing two mutations. With the exisiting FateMutator methods, this was the only way for this to work (since putInitReservationVal() requires the column to be empty and putReservedTx() requires the column to be NOT_RESERVED so always fails when both conditions are used together), and I figured this will be changed anyways once reserving/unreserving is replaced with creating/deleting the column. Added new FateMutator method for this use case here to avoid two mutations.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1702174500
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -104,19 +109,108 @@ public FateId create() {
}
@Override
- protected void create(FateId fateId, FateKey key) {
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
- NodeExistsPolicy.FAIL);
- } catch (KeeperException | InterruptedException e) {
+ byte[] nodeVal = zk.mutateOrCreate(getTXPath(fateId),
+ new NodeValue(TStatus.NEW, reservation, fateKey).serialize(), currSerNodeVal -> {
+ // We are only returning a non-null value for the following cases:
+ // 1) The existing NodeValue for fateId is exactly the same as the value set for the
+ // node if it doesn't yet exist:
+ // TStatus = TStatus.NEW, FateReservation = reservation, FateKey = fateKey
+ // This might occur if there was a ZK server fault and the same write is running a 2nd
+ // time
+ // 2) The existing NodeValue for fateId has:
+ // TStatus = TStatus.NEW, no FateReservation present, FateKey = fateKey
+ // The fateId is NEW/unseeded and not reserved, so we can allow it to be reserved
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ if (currNodeVal.status == TStatus.NEW && currNodeVal.isReservedBy(reservation)) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ return currSerNodeVal;
+ } else if (currNodeVal.status == TStatus.NEW && !currNodeVal.isReserved()) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ // NEW/unseeded transaction and not reserved, so we can allow it to be reserved
+ return new NodeValue(TStatus.NEW, reservation, fateKey).serialize();
+ } else {
+ log.trace(
+ "fate id {} tstatus {} fate key {} is reserved {} is either currently reserved "
+ + "or has already been seeded with work (non-NEW status), or both",
+ fateId, currNodeVal.status, currNodeVal.fateKey.orElse(null),
+ currNodeVal.isReserved());
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (nodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
Review Comment:
I looked into adding a test for this in elasticity branch, but found it was not worthwhile if the behavior will change. Could add at test to `FateStoreIT.testAbsent()` in this PR that does the following.
```java
var optTxStore = store.tryReserve(fateId);
assertTrue(optTxStore.isEmpty());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1702242399
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -104,19 +109,108 @@ public FateId create() {
}
@Override
- protected void create(FateId fateId, FateKey key) {
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
- NodeExistsPolicy.FAIL);
- } catch (KeeperException | InterruptedException e) {
+ byte[] nodeVal = zk.mutateOrCreate(getTXPath(fateId),
+ new NodeValue(TStatus.NEW, reservation, fateKey).serialize(), currSerNodeVal -> {
+ // We are only returning a non-null value for the following cases:
+ // 1) The existing NodeValue for fateId is exactly the same as the value set for the
+ // node if it doesn't yet exist:
+ // TStatus = TStatus.NEW, FateReservation = reservation, FateKey = fateKey
+ // This might occur if there was a ZK server fault and the same write is running a 2nd
+ // time
+ // 2) The existing NodeValue for fateId has:
+ // TStatus = TStatus.NEW, no FateReservation present, FateKey = fateKey
+ // The fateId is NEW/unseeded and not reserved, so we can allow it to be reserved
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ if (currNodeVal.status == TStatus.NEW && currNodeVal.isReservedBy(reservation)) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ return currSerNodeVal;
+ } else if (currNodeVal.status == TStatus.NEW && !currNodeVal.isReserved()) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ // NEW/unseeded transaction and not reserved, so we can allow it to be reserved
+ return new NodeValue(TStatus.NEW, reservation, fateKey).serialize();
+ } else {
+ log.trace(
+ "fate id {} tstatus {} fate key {} is reserved {} is either currently reserved "
+ + "or has already been seeded with work (non-NEW status), or both",
+ fateId, currNodeVal.status, currNodeVal.fateKey.orElse(null),
+ currNodeVal.isReserved());
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (nodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
Review Comment:
Gotcha. I added that check to testAbsent() in this PR. Doesn't hurt to have it tested multiple places and makes to have in testAbsent()
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672859524
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Yeah, I also think deleting would be better. I will make this change.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1670838100
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -106,13 +112,95 @@ public FateId create() {
@Override
protected void create(FateId fateId, FateKey key) {
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
+ zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, null, key).serialize(),
NodeExistsPolicy.FAIL);
} catch (KeeperException | InterruptedException e) {
throw new IllegalStateException(e);
}
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ // The uuid handles the case where there was a ZK server fault and the write for this thread
+ // went through but that was not acknowledged, and we are reading our own write for 2nd
+ // time.
+ if (!currNodeVal.isReserved() || (currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(reservation))) {
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ // Add the FateReservation to the node to reserve
+ return new NodeValue(currNodeVal.status, reservation, currFateKey).serialize();
+ } else {
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (newSerNodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return getNode(fateId).isReserved();
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ Map<FateId,FateReservation> activeReservations = new HashMap<>();
+
+ try {
+ for (String strTxId : zk.getChildren(path)) {
+ String txUUIDStr = strTxId.split("_")[1];
+ FateId fateId = FateId.from(fateInstanceType, txUUIDStr);
+ if (isReserved(fateId)) {
+ FateReservation reservation = getNode(fateId).reservation.orElseThrow();
Review Comment:
This is sometime calling getNode twice, making two RPCs to zookeeper because `isReserved(fateId)` calls `getNode()`. Could refactor this to only read from zookeeper once.
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -168,18 +173,16 @@ public void runnable(AtomicBoolean keepWaiting, Consumer<FateId> idConsumer) {
var transactions = Stream.concat(inProgress, other);
transactions.filter(fateIdStatus -> isRunnable(fateIdStatus.getStatus()))
.map(FateIdStatus::getFateId).filter(fateId -> {
- synchronized (AbstractFateStore.this) {
- var deferredTime = deferred.get(fateId);
- if (deferredTime != null) {
- if (deferredTime.elapsed().isNegative()) {
- // negative elapsed time indicates the deferral time is in the future
- return false;
- } else {
- deferred.remove(fateId);
- }
+ var deferredTime = deferred.get(fateId);
+ if (deferredTime != null) {
+ if (deferredTime.elapsed().isNegative()) {
+ // negative elapsed time indicates the deferral time is in the future
+ return false;
+ } else {
+ deferred.remove(fateId);
}
- return !reserved.contains(fateId);
}
+ return !isReserved(fateId);
Review Comment:
This will do a scan or zookeeper read of data inside a loop that is processing scanned data. So overall this will look like the following conceptually.
```java
for(scan1) {
// run scan 2 that will run a RPC to a remote system
}
```
In both impls we could avoid this scan in the inner loop by obtaining the information in the outer scan. This wold require refactoring `FateIdStatus` to include reservation information. For UserFateStore then the original scan could easily obtain the information. For MetaFateStore it could obtain the status and reservation in the same ZK read as these are both stored in
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -298,7 +299,13 @@ private Optional<FateId> create(FateKey fateKey) {
@Override
public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
- FateId fateId = fateIdGenerator.fromTypeAndKey(getInstanceType(), fateKey);
+ // TODO 4131 not confident about this new implementation of createAndReserve.
Review Comment:
The way this method previously worked and the way the code was structured it was driven by the lock object being in AbstractFateStore. Now that there is no longer a lock we want to implement the following algorithm for both stores using conditional updates (@cshannon you added the fatekey feature, does the following algorithm seem ok to you?).
1. var fateId = FateId fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
2. in the store conditionally create fateId setting the status, reservation and fateKey atomically at creation time. The condition should check that nothing currently exist for the fateId, its expected to be absent.
3. If the conditional update fails for some reason, read the fateId and see if it already has the expected status, fateKey and reservation. If it does, then probably running for a second time and already reserved. If it does not then need to take the following actions.
1. If the fateId exists and has the same fateKey, but different status or reservation than expected then return Optional.empty(). This is the case where another thread has already created and reserved using the fateKey so there is nothing to do.
2. If the fateId exists and has different fateKey or no fateKey then this represents an unexpected collision (and this unexpected with the 128 bit keys) so throw an exception in this case.
5. If the conditional update succeed , then we can return a non empty Optional w/ a FateTxStore obj.
Now that the lock object is no longer needed in AbstractFateStore it may make sense to completely push the full implementation of the `createAndReserve` method into MetaFateStore and UserFateStore and have nothing in AbstractFateStore for the method. The reason I am thinking this may be better is because the algorithm above is based on conditional updates now and how ZK and accumulo do conditional updates is very different. However there may still be opportunities to share common code, so not sure of the best way to structure the code.
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -73,9 +75,12 @@ public class Fate<T> {
private final FateStore<T> store;
private final T environment;
private final ScheduledThreadPoolExecutor fatePoolWatcher;
- private final ExecutorService executor;
+ private final ExecutorService transactionExecutor;
+ private final ExecutorService deadResCleanerExecutor;
private static final EnumSet<TStatus> FINISHED_STATES = EnumSet.of(FAILED, SUCCESSFUL, UNKNOWN);
+ private static boolean userDeadReservationCleanerRunning = false;
+ private static boolean metaDeadReservationCleanerRunning = false;
Review Comment:
Can these static variables could be removed? Do these help testing in someway? In the manager we can assume that only a single instance of each type is created. If we wanted to guard against multiple instances per type, then a more general check could be done that is not limited to dead reservation cleaners.
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Given the way the scanner was constructed, would we expect to see anything that is not a reservation? If only reservations are expected then could drop the filter and let `FateReservation.from` throw an exception if it seems something that is not a reservation.
```suggestion
.map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672854693
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
> Thinking about this again, it would probably be better to just delete/create the column as needed instead of storing a not reserved value. What do you think?
Either way seems like it could be correct. Personally I would prefer deleting. If going with the NOT_RESERVED value approach it would be good to make the validation of the column value more strict, will make a comment about that elsewhere.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672878600
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
>If going with the NOT_RESERVED value approach it would be good to make the validation of the column value more strict, will make a comment about that elsewhere.
Nevermind the code does not work exactly the way I thought it did. I was thinking when reading reservations could do something like the following
```java
if(value look like <lock id>:<uuid) {
// process reserved
} else if (value is NOT_RESERVED) {
//process not reserved
} else {
throw new IllegalArgumentException("invalid reseved column value");
}
```
However there is not a place in the code that reads reservations in this way. Was thinking the code should fail if the reserved column value was something like `1234` because it does not look like a reservation and it also does not look like NOT_RESERVED
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1597514630
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -150,6 +150,39 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // TODO 4131 should this throw an exception if the id doesn't exist (status = UNKNOWN)?
+ FateMutator.Status status = newMutator(fateId).putReservedTx(fateId).tryMutate();
Review Comment:
The manager lock is [here](https://github.com/apache/accumulo/blob/0edc48f910f25e5273d62b1d91a7993bf9529b8e/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java#L217) and from that can access [ServiceLock.getLockID()](https://github.com/apache/accumulo/blob/0edc48f910f25e5273d62b1d91a7993bf9529b8e/core/src/main/java/org/apache/accumulo/core/lock/ServiceLock.java#L594) and [ServiceLock.isLockHeld()](https://github.com/apache/accumulo/blob/0edc48f910f25e5273d62b1d91a7993bf9529b8e/core/src/main/java/org/apache/accumulo/core/lock/ServiceLock.java#L654). So can persist the LockId w/ reservations, although not sure of the best way to do that. Then the periodic test that looks for reservations held by dead processes it can call ServiceLock.isLockHeld.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1662972726
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -313,6 +317,26 @@ private void undo(FateId fateId, Repo<T> op) {
}
+ /**
+ * A thread that finds reservations held by dead processes and unreserves them. Only one thread
+ * runs per store type across all Fate instances (one to clean up dead reservations for
+ * {@link org.apache.accumulo.core.fate.user.UserFateStore UserFateStore} and one to clean up dead
+ * reservations for {@link MetaFateStore}).
+ */
+ private class DeadReservationCleaner implements Runnable {
+ // TODO 4131 periodic check runs every 30 seconds
+ // Should this be longer? Shorter? A configurable Property? A function of something?
+ private static final long INTERVAL_MILLIS = 30_000;
+
+ @Override
+ public void run() {
+ while (keepRunning.get()) {
+ store.deleteDeadReservations();
Review Comment:
That would be good to open a follow on issue about making all of the fate threads critical.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688549674
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
I attempted to refactor UserFateStore to create/delete the RESERVATION column on reserve/unreserve, however, I ran into an issue that I could not figure out, so I ended up reverting this change. Multiple threads would occassionally reserve the same fate id. I could not figure out how/why this was occuring. I think the switch to deleting the column should be made, but maybe as a follow on or after discussion about the potential cause of this. Maybe I could put up a new commit with just this change to easily see the difference and what the issue might be.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688547995
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -73,9 +75,12 @@ public class Fate<T> {
private final FateStore<T> store;
private final T environment;
private final ScheduledThreadPoolExecutor fatePoolWatcher;
- private final ExecutorService executor;
+ private final ExecutorService transactionExecutor;
+ private final ExecutorService deadResCleanerExecutor;
private static final EnumSet<TStatus> FINISHED_STATES = EnumSet.of(FAILED, SUCCESSFUL, UNKNOWN);
+ private static boolean userDeadReservationCleanerRunning = false;
+ private static boolean metaDeadReservationCleanerRunning = false;
Review Comment:
Left as-is until further clarification
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700589603
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -120,35 +127,161 @@ public FateId getFateId() {
}
@Override
- protected void create(FateId fateId, FateKey fateKey) {
- final int maxAttempts = 5;
-
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+ Optional<FateTxStore<T>> txStore = Optional.empty();
+ int maxAttempts = 5;
+ FateMutator.Status status = null;
+
+ // We first need to write the initial/unreserved value for the reservation column
+ // Only need to retry if it is UNKNOWN
for (int attempt = 0; attempt < maxAttempts; attempt++) {
-
- if (attempt >= 1) {
- log.debug("Failed to create transaction with fateId {} and fateKey {}, trying again",
- fateId, fateKey);
- UtilWaitThread.sleep(100);
+ status = newMutator(fateId).putInitReservationVal().tryMutate();
+ if (status != FateMutator.Status.UNKNOWN) {
+ break;
}
+ UtilWaitThread.sleep(100);
+ }
+ for (int attempt = 0; attempt < maxAttempts; attempt++) {
+ status = newMutator(fateId).requireStatus().putStatus(TStatus.NEW).putKey(fateKey)
+ .putReservedTx(reservation).putCreateTime(System.currentTimeMillis()).tryMutate();
+ if (status != FateMutator.Status.UNKNOWN) {
+ break;
+ }
+ UtilWaitThread.sleep(100);
+ }
- var status = newMutator(fateId).requireStatus().putStatus(TStatus.NEW).putKey(fateKey)
- .putCreateTime(System.currentTimeMillis()).tryMutate();
+ switch (status) {
+ case ACCEPTED:
+ txStore = Optional.of(new FateTxStoreImpl(fateId, reservation));
+ break;
+ case REJECTED:
+ // If the status is REJECTED, we need to check what about the mutation was REJECTED:
+ // 1) Possible something like the following occurred:
+ // the first attempt was UNKNOWN but written, the next attempt would be rejected
+ // We return the FateTxStore in this case.
+ // 2) If there is a collision with existing fate id, throw error
+ // 3) If the fate id is already reserved, return an empty optional
+ // 4) If the fate id is still NEW/unseeded and unreserved, we can try to reserve it
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.STATUS_COLUMN.getColumnFamily(),
+ TxColumnFamily.STATUS_COLUMN.getColumnQualifier());
+ scanner.fetchColumn(TxColumnFamily.TX_KEY_COLUMN.getColumnFamily(),
+ TxColumnFamily.TX_KEY_COLUMN.getColumnQualifier());
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ TStatus statusSeen = TStatus.UNKNOWN;
+ Optional<FateKey> fateKeySeen = Optional.empty();
+ Optional<FateReservation> reservationSeen = Optional.empty();
+
+ for (Entry<Key,Value> entry : scanner.stream().collect(Collectors.toList())) {
+ Text colf = entry.getKey().getColumnFamily();
+ Text colq = entry.getKey().getColumnQualifier();
+ Value val = entry.getValue();
+
+ switch (colq.toString()) {
+ case TxColumnFamily.STATUS:
+ statusSeen = TStatus.valueOf(val.toString());
+ break;
+ case TxColumnFamily.TX_KEY:
+ fateKeySeen = Optional.of(FateKey.deserialize(val.get()));
+ break;
+ case TxColumnFamily.RESERVATION:
+ if (FateReservation.isFateReservation(val.get())) {
+ reservationSeen = Optional.of(FateReservation.deserialize(val.get()));
+ }
+ break;
+ default:
+ throw new IllegalStateException("Unexpected column seen: " + colf + ":" + colq);
+ }
+ }
- switch (status) {
- case ACCEPTED:
- return;
- case UNKNOWN:
- continue;
- case REJECTED:
- throw new IllegalStateException("Attempt to create transaction with fateId " + fateId
- + " and fateKey " + fateKey + " was rejected");
- default:
- throw new IllegalStateException("Unknown status " + status);
+ // This will be the case if the mutation status is REJECTED but the mutation was written
+ if (statusSeen == TStatus.NEW && reservationSeen.isPresent()
+ && reservationSeen.orElseThrow().equals(reservation)) {
+ verifyFateKey(fateId, fateKeySeen, fateKey);
+ txStore = Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (statusSeen == TStatus.NEW && reservationSeen.isEmpty()) {
+ verifyFateKey(fateId, fateKeySeen, fateKey);
+ // NEW/unseeded transaction and not reserved, so we can allow it to be reserved
+ // we tryReserve() since another thread may have reserved it since the scan
+ txStore = tryReserve(fateId);
Review Comment:
Added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1692342340
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -67,27 +76,36 @@ public FateId fromTypeAndKey(FateInstanceType instanceType, FateKey fateKey) {
}
};
- protected final Set<FateId> reserved;
+ // The ZooKeeper lock for the process that's running this store instance
+ protected final ZooUtil.LockID lockID;
+ protected final Predicate<ZooUtil.LockID> isLockHeld;
protected final Map<FateId,NanoTime> deferred;
+ protected final FateIdGenerator fateIdGenerator;
private final int maxDeferred;
private final AtomicBoolean deferredOverflow = new AtomicBoolean();
- private final FateIdGenerator fateIdGenerator;
-
- // This is incremented each time a transaction was unreserved that was non new
- protected final SignalCount unreservedNonNewCount = new SignalCount();
// This is incremented each time a transaction is unreserved that was runnable
- protected final SignalCount unreservedRunnableCount = new SignalCount();
+ private final SignalCount unreservedRunnableCount = new SignalCount();
+
+ // Keeps track of the number of concurrent callers to waitForStatusChange()
+ private final AtomicInteger concurrentStatusChangeCallers = new AtomicInteger(0);
public AbstractFateStore() {
- this(DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ this(null, null, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
}
- public AbstractFateStore(int maxDeferred, FateIdGenerator fateIdGenerator) {
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld) {
+ this(lockID, isLockHeld, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ }
+
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld,
+ int maxDeferred, FateIdGenerator fateIdGenerator) {
this.maxDeferred = maxDeferred;
this.fateIdGenerator = Objects.requireNonNull(fateIdGenerator);
- this.reserved = new HashSet<>();
- this.deferred = new HashMap<>();
+ this.deferred = Collections.synchronizedMap(new HashMap<>());
+ this.lockID = Objects.requireNonNullElseGet(lockID, AbstractFateStore::createDummyLockID);
Review Comment:
I did not know this function existed. Went looked at what other new methods exists in Objects and found checkFromIndexSize which could be useful in other places in the code base for efficiently checking bounds. Opened https://github.com/apache/accumulo-access/pull/80 as a result of finding these new methods. I think we can use these index bounds checking methods in other places.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1690323502
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -298,7 +299,13 @@ private Optional<FateId> create(FateKey fateKey) {
@Override
public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
- FateId fateId = fateIdGenerator.fromTypeAndKey(getInstanceType(), fateKey);
+ // TODO 4131 not confident about this new implementation of createAndReserve.
Review Comment:
As it is now, UFS.createAndReserve is pretty lengthy and both implementations could probably share common code. Was mostly interested if the impls are good. Can do code cleanup after.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1690323502
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -298,7 +299,13 @@ private Optional<FateId> create(FateKey fateKey) {
@Override
public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
- FateId fateId = fateIdGenerator.fromTypeAndKey(getInstanceType(), fateKey);
+ // TODO 4131 not confident about this new implementation of createAndReserve.
Review Comment:
As it is now, UFS.createAndReserve is pretty lengthy and both implementations could probably share common code. Was mostly interested if the impls are good. Can do code cleanup after.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2246036407
@keith-turner, @DomGarguilo
I have addressed all of the review changes, and this PR is ready for re-review. However, some tests are no longer passing (e.g., ComprehensiveIT). I believe this is an issue with my new implementation of createAndReserve. I believe the issue is with the UserFateStore implementation, but could be both. Posting this new commit now to get the other changes out, and to potentially receive feedback on what might be wrong with createAndReserve(). I will continue to look into what the issue might be in the meantime. Please take a look at my new review comments and my new commit message:
- Stricter check of the column value for fate reservations. If anything unexpected is seen, an error is now thrown.
- Simplified MetaFateStore.getActiveReservations() to only read from ZooKeeper once.
- Combined the two scans that were done in AbstractFateStore.runnable() into one. This meant adding FateReservation to FateIdStatus and refactoring Meta/UserFateStore.getTransactions().
- No longer use/store a string representation of the FateReservation (was used in UserFateStore). Now, only the serialized value is used. This keeps the usage of FateReservation consistent across Meta and UserFateStore. It was also unneccessary, so simplifies code.
- Moved AbstractFateStore.createAndReserve() implementation into Meta and UserFateStore, and rewrote the impl for each to work with the new way reservations are stored. This also allowed me to delete methods that were only used by AFS.createAndReserve(): create(FateId, FateKey), getStatusAndKey(FateId), create(FateKey).
- Fixed how concurrentStatusChangeCallers was decremented in AbstractFateStore.waitForStatusChange()
- Small change to MetaFateStore.deleteDeadReservations() to avoid reading from ZK unnecessarily
- Added isReservedBy() method to MetaFateStore.NodeValue to avoid code duplication and make the code more clear.
- Since the FateIdStatus now has the FateReservation, realized Meta and UserFateStore.getActiveReservations() could now be simplified to just call list(). This also made the impls the same, so moved to AbstractFateStore. This also made me realize that I had put getActiveReservations() method signature in FateStore, but would be better suited for ReadOnlyFateStore, so moved it there.
- Deleted FateStore.isReserved(FateId)... No longer needed/used
- Moved UNKNOWN status check in AbstractFateStore.reserve() into waiting loop
- Now log when a dead reservation is detected and deleted
- Minor change to Fate: no longer create the executor for the dead reservation cleaner if it's not going to be used
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688548367
##########
core/src/main/java/org/apache/accumulo/core/fate/FateStore.java:
##########
@@ -107,11 +118,155 @@ interface FateTxStore<T> extends ReadOnlyFateTxStore<T> {
void unreserve(Duration deferTime);
}
+ /**
+ * The value stored to indicate a FATE transaction ID ({@link FateId}) has been reserved
+ */
+ class FateReservation {
+
+ // The LockID (provided by the Manager running the FATE which uses this store) which is used for
+ // identifying dead Managers, so their reservations can be deleted and picked up again since
+ // they can no longer be worked on.
+ private final ZooUtil.LockID lockID; // TODO 4131 not sure if this is the best type for this
+ // The UUID generated on a reservation attempt (tryReserve()) used to uniquely identify that
+ // attempt. This is useful for the edge case where the reservation is sent to the server
+ // (Tablet Server for UserFateStore and the ZooKeeper Server for MetaFateStore), but the server
+ // dies before the store receives the response. It allows us to determine if the reservation
+ // was successful and was written by this reservation attempt (could have been successfully
+ // reserved by another attempt or not reserved at all, in which case, we wouldn't want to
+ // expose a FateTxStore).
+ private final UUID reservationUUID;
+ private final byte[] serialized;
+ private static final Pattern UUID_PATTERN =
+ Pattern.compile("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$");
+ private static final Pattern LOCKID_PATTERN = Pattern.compile("^.+/.+\\$[0-9a-fA-F]+$");
+
+ private FateReservation(ZooUtil.LockID lockID, UUID reservationUUID) {
+ this.lockID = Objects.requireNonNull(lockID);
+ this.reservationUUID = Objects.requireNonNull(reservationUUID);
+ this.serialized = serialize(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(ZooUtil.LockID lockID, UUID reservationUUID) {
+ return new FateReservation(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(byte[] serialized) {
+ try (DataInputBuffer buffer = new DataInputBuffer()) {
+ buffer.reset(serialized, serialized.length);
+ ZooUtil.LockID lockID = new ZooUtil.LockID("", buffer.readUTF());
+ UUID reservationUUID = UUID.fromString(buffer.readUTF());
+ return new FateReservation(lockID, reservationUUID);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static FateReservation from(String fateReservationStr) {
+ if (isFateReservation(fateReservationStr)) {
+ String[] fields = fateReservationStr.split(":");
+ ZooUtil.LockID lockId = new ZooUtil.LockID("", fields[0]);
+ UUID reservationUUID = UUID.fromString(fields[1]);
+ return new FateReservation(lockId, reservationUUID);
+ } else {
+ throw new IllegalArgumentException(
+ "Tried to create a FateReservation from an invalid string: " + fateReservationStr);
+ }
+ }
+
+ /**
+ *
+ * @param fateReservationStr the string from a call to {@link FateReservation#toString()}
+ * @return true if the string represents a valid FateReservation object, false otherwise
+ */
+ public static boolean isFateReservation(String fateReservationStr) {
+ if (fateReservationStr != null) {
+ String[] fields = fateReservationStr.split(":");
+ if (fields.length == 2) {
+ return LOCKID_PATTERN.matcher(fields[0]).matches()
+ && UUID_PATTERN.matcher(fields[1]).matches();
+ }
+ }
+ return false;
+ }
+
+ public ZooUtil.LockID getLockID() {
+ return lockID;
+ }
+
+ public UUID getReservationUUID() {
+ return reservationUUID;
+ }
+
+ public byte[] getSerialized() {
+ return serialized;
+ }
+
+ private static byte[] serialize(ZooUtil.LockID lockID, UUID reservationUUID) {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos)) {
+ dos.writeUTF(lockID.serialize("/"));
+ dos.writeUTF(reservationUUID.toString());
+ dos.close();
Review Comment:
Looking at existing code, this was done elsewhere (see FateKey or CompactionConfigStorage for example), and is what I based my code off of since I was doing the same thing. I also didn't understand why the close() was done early, but I left it in to be safe.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688555925
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
+ .orElse(null);
+ if (persistedRes != null && persistedRes.equals(reservation)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ }
+ } catch (TableNotFoundException e) {
+ throw new IllegalStateException(tableName + " not found!", e);
+ }
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ return scanner.stream()
+ .map(entry -> FateReservation.isFateReservation(entry.getValue().toString())).findFirst()
+ .orElse(false);
+ } catch (TableNotFoundException e) {
+ throw new IllegalStateException(tableName + " not found!", e);
+ }
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ Map<FateId,FateReservation> activeReservations = new HashMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(new Range());
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
Review Comment:
Addressed. Now, isFateReservation() throws an error if it's passed anything unnexpected (only expects a NOT_RESERVED value or a valid FateReservation serialization).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688551230
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -114,38 +131,26 @@ public static Object deserialize(byte[] ser) {
}
}
- /**
- * Attempt to reserve the fate transaction.
- *
- * @param fateId The FateId
- * @return An Optional containing the FateTxStore if the transaction was successfully reserved, or
- * an empty Optional if the transaction was already reserved.
- */
- @Override
- public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
- synchronized (this) {
- if (!reserved.contains(fateId)) {
- return Optional.of(reserve(fateId));
- }
- return Optional.empty();
- }
- }
-
@Override
public FateTxStore<T> reserve(FateId fateId) {
- synchronized (AbstractFateStore.this) {
- while (reserved.contains(fateId)) {
- try {
- AbstractFateStore.this.wait(100);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new IllegalStateException(e);
- }
+ Preconditions.checkState(!_getStatus(fateId).equals(TStatus.UNKNOWN),
+ "Attempted to reserve a tx that does not exist: " + fateId);
Review Comment:
Good catch, fixed.
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -361,90 +356,108 @@ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
protected abstract Optional<FateKey> getKey(FateId fateId);
- protected abstract FateTxStore<T> newFateTxStore(FateId fateId, boolean isReserved);
-
- protected abstract FateInstanceType getInstanceType();
+ protected abstract FateTxStore<T> newUnreservedFateTxStore(FateId fateId);
protected abstract class AbstractFateTxStoreImpl<T> implements FateTxStore<T> {
protected final FateId fateId;
- protected final boolean isReserved;
+ protected boolean deleted;
+ protected FateReservation reservation;
protected TStatus observedStatus = null;
- protected AbstractFateTxStoreImpl(FateId fateId, boolean isReserved) {
+ protected AbstractFateTxStoreImpl(FateId fateId) {
+ this.fateId = fateId;
+ this.deleted = false;
+ this.reservation = null;
+ }
+
+ protected AbstractFateTxStoreImpl(FateId fateId, FateReservation reservation) {
this.fateId = fateId;
- this.isReserved = isReserved;
+ this.deleted = false;
+ this.reservation = Objects.requireNonNull(reservation);
+ }
+
+ protected boolean isReserved() {
+ return this.reservation != null;
}
@Override
public TStatus waitForStatusChange(EnumSet<TStatus> expected) {
- Preconditions.checkState(!isReserved,
- "Attempted to wait for status change while reserved " + fateId);
- while (true) {
+ Preconditions.checkState(!isReserved(),
+ "Attempted to wait for status change while reserved: " + fateId);
+ verifyReserved(false);
- long countBefore = unreservedNonNewCount.getCount();
+ int currNumCallers = concurrentStatusChangeCallers.incrementAndGet();
Review Comment:
Good suggestion, fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688553945
##########
core/src/main/java/org/apache/accumulo/core/fate/user/FateMutatorImpl.java:
##########
@@ -79,6 +82,33 @@ public FateMutator<T> putCreateTime(long ctime) {
return this;
}
+ @Override
+ public FateMutator<T> putReservedTx(FateStore.FateReservation reservation) {
+ Condition condition = new Condition(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier()).setValue(NOT_RESERVED);
+ mutation.addCondition(condition);
+ TxColumnFamily.RESERVATION_COLUMN.put(mutation, new Value(reservation.toString()));
Review Comment:
Yes, good call. I now only use the serialized value
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688555387
##########
core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java:
##########
@@ -175,6 +176,21 @@ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
}
return txStore;
}
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return store.isReserved(fateId);
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ return store.getActiveReservations();
+ }
+
+ @Override
+ public void deleteDeadReservations() {
+ store.deleteDeadReservations();
Review Comment:
Yes, nice suggestion. For simplicity in this new commit, I just added the same log message for both impls.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1692342340
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -67,27 +76,36 @@ public FateId fromTypeAndKey(FateInstanceType instanceType, FateKey fateKey) {
}
};
- protected final Set<FateId> reserved;
+ // The ZooKeeper lock for the process that's running this store instance
+ protected final ZooUtil.LockID lockID;
+ protected final Predicate<ZooUtil.LockID> isLockHeld;
protected final Map<FateId,NanoTime> deferred;
+ protected final FateIdGenerator fateIdGenerator;
private final int maxDeferred;
private final AtomicBoolean deferredOverflow = new AtomicBoolean();
- private final FateIdGenerator fateIdGenerator;
-
- // This is incremented each time a transaction was unreserved that was non new
- protected final SignalCount unreservedNonNewCount = new SignalCount();
// This is incremented each time a transaction is unreserved that was runnable
- protected final SignalCount unreservedRunnableCount = new SignalCount();
+ private final SignalCount unreservedRunnableCount = new SignalCount();
+
+ // Keeps track of the number of concurrent callers to waitForStatusChange()
+ private final AtomicInteger concurrentStatusChangeCallers = new AtomicInteger(0);
public AbstractFateStore() {
- this(DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ this(null, null, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
}
- public AbstractFateStore(int maxDeferred, FateIdGenerator fateIdGenerator) {
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld) {
+ this(lockID, isLockHeld, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ }
+
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld,
+ int maxDeferred, FateIdGenerator fateIdGenerator) {
this.maxDeferred = maxDeferred;
this.fateIdGenerator = Objects.requireNonNull(fateIdGenerator);
- this.reserved = new HashSet<>();
- this.deferred = new HashMap<>();
+ this.deferred = Collections.synchronizedMap(new HashMap<>());
+ this.lockID = Objects.requireNonNullElseGet(lockID, AbstractFateStore::createDummyLockID);
Review Comment:
I did not know this function existed. Looked at what other new methods exists in Objects and found checkFromIndexSize which could be useful in other places in the code base for efficiently checking bounds. Opened https://github.com/apache/accumulo-access/pull/80 as a result of finding these new methods. I think we can use these index bounds checking methods in other places.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1692199252
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
> Multiple threads would occassionally reserve the same fate id.
Not sure what the problem was, but its kinda neat you detected that bug. Where you running an IT when you saw multiple threads reserve a transaction?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1702267741
##########
server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/meta/MetaFateMetrics.java:
##########
@@ -59,7 +60,8 @@ public void registerMetrics(MeterRegistry registry) {
@Override
protected ReadOnlyFateStore<FateMetrics<MetaFateMetricValues>> buildStore(ServerContext context) {
try {
- return new MetaFateStore<>(getFateRootPath(context), context.getZooReaderWriter());
+ return new MetaFateStore<>(getFateRootPath(context), context.getZooReaderWriter(),
+ AbstractFateStore.createDummyLockID(), null);
Review Comment:
This store is being created for metrics, so it may be using it in a readonly way and may never reserve. If it never reserves then it does not need a lock, not even a dummy lock. Not something for this PR, but for this case I think we want to somehow create a store obj where attempts to reserve will be non functional and throw an exception.
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -67,27 +76,36 @@ public FateId fromTypeAndKey(FateInstanceType instanceType, FateKey fateKey) {
}
};
- protected final Set<FateId> reserved;
+ // The ZooKeeper lock for the process that's running this store instance
+ protected final ZooUtil.LockID lockID;
+ protected final Predicate<ZooUtil.LockID> isLockHeld;
protected final Map<FateId,NanoTime> deferred;
+ protected final FateIdGenerator fateIdGenerator;
private final int maxDeferred;
private final AtomicBoolean deferredOverflow = new AtomicBoolean();
- private final FateIdGenerator fateIdGenerator;
-
- // This is incremented each time a transaction was unreserved that was non new
- protected final SignalCount unreservedNonNewCount = new SignalCount();
// This is incremented each time a transaction is unreserved that was runnable
- protected final SignalCount unreservedRunnableCount = new SignalCount();
+ private final SignalCount unreservedRunnableCount = new SignalCount();
+
+ // Keeps track of the number of concurrent callers to waitForStatusChange()
+ private final AtomicInteger concurrentStatusChangeCallers = new AtomicInteger(0);
public AbstractFateStore() {
- this(DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ this(null, null, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
}
- public AbstractFateStore(int maxDeferred, FateIdGenerator fateIdGenerator) {
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld) {
+ this(lockID, isLockHeld, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ }
+
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld,
+ int maxDeferred, FateIdGenerator fateIdGenerator) {
this.maxDeferred = maxDeferred;
this.fateIdGenerator = Objects.requireNonNull(fateIdGenerator);
- this.reserved = new HashSet<>();
- this.deferred = new HashMap<>();
+ this.deferred = Collections.synchronizedMap(new HashMap<>());
+ this.lockID = Objects.requireNonNullElseGet(lockID, AbstractFateStore::createDummyLockID);
Review Comment:
ok sounds good. I am working though comments and resolving them. Left some comments open that seemed like they needed a follow on issue. Once issues are opened can resolve those comments. We may need an issue about making the fate utils get a lock.
##########
core/src/main/java/org/apache/accumulo/core/fate/user/RowFateStatusFilter.java:
##########
@@ -53,19 +56,28 @@ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> op
}
@Override
- public boolean accept(Key k, Value v) {
- var tstatus = ReadOnlyFateStore.TStatus.valueOf(v.toString());
- return valuesToAccept.contains(tstatus);
+ protected boolean filter(Text currentRow, List<Key> keys, List<Value> values) {
+ for (int i = 0; i < keys.size(); i++) {
+ Key key = keys.get(i);
+ if (key.getColumnQualifier()
+ .equals(FateSchema.TxColumnFamily.STATUS_COLUMN.getColumnQualifier())
Review Comment:
```suggestion
if (FateSchema.TxColumnFamily.STATUS_COLUMN.hasColumns(key)
```
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -57,8 +57,8 @@
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
// TODO 4131 should probably add support to AbstractFateStore, MetaFateStore,
-// and UserFateStore to accept null lockID and zooCache (maybe make these fields
-// Optional<>). This could replace the current createDummyLockID(). This support
+// and UserFateStore to accept null lockID (maybe make this field Optional<>).
Review Comment:
When I have a PR w/ a lot of TODOs sometimes I will try to open an issue for each TODO and remove it before merging. Could do that for this PR but do not have to, if not doing it then it would be good to use a more unique TODO which I think is being done here? Do all of the TODOs have the 4131 suffix? If so then it will be easy to find them with grep. Then we just need an open issue that records the fact that all `TODO 4131` need to be done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700586932
##########
core/src/main/java/org/apache/accumulo/core/fate/user/FateStatusFilter.java:
##########
@@ -54,8 +55,15 @@ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> op
@Override
public boolean accept(Key k, Value v) {
- var tstatus = ReadOnlyFateStore.TStatus.valueOf(v.toString());
- return valuesToAccept.contains(tstatus);
+ // We may see TStatus values or FateReservation values with how this filter is used,
+ // only accept TStatus values, return false on FateReservation values, error otherwise
+ try {
+ var tstatus = ReadOnlyFateStore.TStatus.valueOf(v.toString());
+ return valuesToAccept.contains(tstatus);
+ } catch (IllegalArgumentException e) {
Review Comment:
Reworked iterator. Thanks for the info, this is much nicer than what I had
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2302639172
Resolved the merge conflicts with `AbstractFateStore` using the new `CountDownTimer`.
Also noticed a sync block that should have been removed so removed that.
`AbstractFateStore` should probably be looked at again before this is merged.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672963994
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
I'm not sure I follow how the reservation could be incorrectly written. I could just be missing/misunderstanding something but for MetaFateStore where I do this check, it checks if a FateReservation object is present and that can only take on the form `<lock id>:<uuid>`. Creating the FateReservation object handles these checks of making sure only a valid FateReservation of the form `<lock id>:<uuid>` is created.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672963994
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
I'm not sure I follow how the reservation could be incorrectly written. I could just be missing/misunderstanding something but for MetaFateStore where I do this check, it checks if a FateReservation object is present and that can only take on the form <lock id>:<uuid>. Creating the FateReservation object handles these checks of making sure only a valid FateReservation of the form <lock id>:<uuid> is created.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1696006961
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -104,19 +109,108 @@ public FateId create() {
}
@Override
- protected void create(FateId fateId, FateKey key) {
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
- NodeExistsPolicy.FAIL);
- } catch (KeeperException | InterruptedException e) {
+ byte[] nodeVal = zk.mutateOrCreate(getTXPath(fateId),
+ new NodeValue(TStatus.NEW, reservation, fateKey).serialize(), currSerNodeVal -> {
+ // We are only returning a non-null value for the following cases:
+ // 1) The existing NodeValue for fateId is exactly the same as the value set for the
+ // node if it doesn't yet exist:
+ // TStatus = TStatus.NEW, FateReservation = reservation, FateKey = fateKey
+ // This might occur if there was a ZK server fault and the same write is running a 2nd
+ // time
+ // 2) The existing NodeValue for fateId has:
+ // TStatus = TStatus.NEW, no FateReservation present, FateKey = fateKey
+ // The fateId is NEW/unseeded and not reserved, so we can allow it to be reserved
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ if (currNodeVal.status == TStatus.NEW && currNodeVal.isReservedBy(reservation)) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ return currSerNodeVal;
+ } else if (currNodeVal.status == TStatus.NEW && !currNodeVal.isReserved()) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ // NEW/unseeded transaction and not reserved, so we can allow it to be reserved
+ return new NodeValue(TStatus.NEW, reservation, fateKey).serialize();
+ } else {
+ log.trace(
+ "fate id {} tstatus {} fate key {} is reserved {} is either currently reserved "
+ + "or has already been seeded with work (non-NEW status), or both",
+ fateId, currNodeVal.status, currNodeVal.fateKey.orElse(null),
+ currNodeVal.isReserved());
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (nodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
Review Comment:
Its possible nothing exists for this fateId in zookeeper. Looking at the impl of `mutateExisting` it calls [Zookeeper.getData()](https://zookeeper.apache.org/doc/r3.9.2/apidocs/zookeeper-server/org/apache/zookeeper/ZooKeeper.html#getData(java.lang.String,boolean,org.apache.zookeeper.data.Stat)) which will throw a NoNodeException for this case. Looking at the usage of `mutateExisting` seems like its currently only used in places where we expect the node to always exists. This case is a bit different, because the fate id could be deleted. So could add the following to the try/catch.
```java
}catch(KeeperException.NoNodeException nne) {
//TODO log trace?
return Optional.empty();
```
Also maybe can test this case of attempting to reserve something that does not exists in the ITs.
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -104,19 +109,108 @@ public FateId create() {
}
@Override
- protected void create(FateId fateId, FateKey key) {
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
- NodeExistsPolicy.FAIL);
- } catch (KeeperException | InterruptedException e) {
+ byte[] nodeVal = zk.mutateOrCreate(getTXPath(fateId),
+ new NodeValue(TStatus.NEW, reservation, fateKey).serialize(), currSerNodeVal -> {
+ // We are only returning a non-null value for the following cases:
+ // 1) The existing NodeValue for fateId is exactly the same as the value set for the
+ // node if it doesn't yet exist:
+ // TStatus = TStatus.NEW, FateReservation = reservation, FateKey = fateKey
+ // This might occur if there was a ZK server fault and the same write is running a 2nd
+ // time
+ // 2) The existing NodeValue for fateId has:
+ // TStatus = TStatus.NEW, no FateReservation present, FateKey = fateKey
+ // The fateId is NEW/unseeded and not reserved, so we can allow it to be reserved
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ if (currNodeVal.status == TStatus.NEW && currNodeVal.isReservedBy(reservation)) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ return currSerNodeVal;
+ } else if (currNodeVal.status == TStatus.NEW && !currNodeVal.isReserved()) {
+ verifyFateKey(fateId, currNodeVal.fateKey, fateKey);
+ // NEW/unseeded transaction and not reserved, so we can allow it to be reserved
+ return new NodeValue(TStatus.NEW, reservation, fateKey).serialize();
+ } else {
+ log.trace(
+ "fate id {} tstatus {} fate key {} is reserved {} is either currently reserved "
+ + "or has already been seeded with work (non-NEW status), or both",
+ fateId, currNodeVal.status, currNodeVal.fateKey.orElse(null),
+ currNodeVal.isReserved());
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (nodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ // The uuid handles the case where there was a ZK server fault and the write for this thread
+ // went through but that was not acknowledged, and we are reading our own write for 2nd
+ // time.
+ if (!currNodeVal.isReserved() || currNodeVal.isReservedBy(reservation)) {
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ // Add the FateReservation to the node to reserve
+ return new NodeValue(currNodeVal.status, reservation, currFateKey).serialize();
+ } else {
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (newSerNodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
throw new IllegalStateException(e);
}
}
@Override
- protected Pair<TStatus,Optional<FateKey>> getStatusAndKey(FateId fateId) {
- final NodeValue node = getNode(fateId);
- return new Pair<>(node.status, node.fateKey);
+ public void deleteDeadReservations() {
+ for (Map.Entry<FateId,FateReservation> entry : getActiveReservations().entrySet()) {
+ FateId fateId = entry.getKey();
+ FateReservation reservation = entry.getValue();
+ if (isLockHeld.test(reservation.getLockID())) {
+ continue;
+ }
+ try {
+ zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ // Make sure the current node is still reserved and reserved with the expected reservation
+ // and it is dead
+ if (currNodeVal.isReservedBy(reservation)
+ && !isLockHeld.test(currNodeVal.reservation.orElseThrow().getLockID())) {
+ // Delete the reservation
+ log.trace("Deleted the dead reservation {} for fate id {}", reservation, fateId);
+ return new NodeValue(currNodeVal.status, null, currNodeVal.fateKey.orElse(null))
+ .serialize();
+ } else {
+ // No change
+ return null;
+ }
+ });
+ } catch (KeeperException | InterruptedException | AcceptableThriftTableOperationException e) {
Review Comment:
We could catch NoNodeException here and just ignore it, means it was deleted between when we read it and tried to act on it.
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -363,13 +498,23 @@ protected Stream<FateIdStatus> getTransactions(Set<TStatus> statuses) {
Stream<FateIdStatus> stream = zk.getChildren(path).stream().map(strTxid -> {
String txUUIDStr = strTxid.split("_")[1];
FateId fateId = FateId.from(fateInstanceType, txUUIDStr);
- // Memoizing for two reasons. First the status may never be requested, so in that case avoid
- // the lookup. Second, if its requested multiple times the result will always be consistent.
- Supplier<TStatus> statusSupplier = Suppliers.memoize(() -> _getStatus(fateId));
+ // Memoizing for two reasons. First the status or reservation may never be requested, so
+ // in that case avoid the lookup. Second, if it's requested multiple times the result will
+ // always be consistent.
+ Supplier<Pair<TStatus,Optional<FateReservation>>> statusAndResSupplier =
+ Suppliers.memoize(() -> {
+ NodeValue zkNode = getNode(fateId);
+ return new Pair<>(zkNode.status, zkNode.reservation);
+ });
Review Comment:
Seems like the Pair could be dropped and this could be simplified to return the NodeValue
```suggestion
Supplier<NodeValue> statusAndResSupplier =
Suppliers.memoize(() -> getNode(fateId));
```
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -120,35 +127,161 @@ public FateId getFateId() {
}
@Override
- protected void create(FateId fateId, FateKey fateKey) {
- final int maxAttempts = 5;
-
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+ Optional<FateTxStore<T>> txStore = Optional.empty();
+ int maxAttempts = 5;
+ FateMutator.Status status = null;
+
+ // We first need to write the initial/unreserved value for the reservation column
+ // Only need to retry if it is UNKNOWN
for (int attempt = 0; attempt < maxAttempts; attempt++) {
-
- if (attempt >= 1) {
- log.debug("Failed to create transaction with fateId {} and fateKey {}, trying again",
- fateId, fateKey);
- UtilWaitThread.sleep(100);
+ status = newMutator(fateId).putInitReservationVal().tryMutate();
+ if (status != FateMutator.Status.UNKNOWN) {
+ break;
}
+ UtilWaitThread.sleep(100);
+ }
+ for (int attempt = 0; attempt < maxAttempts; attempt++) {
Review Comment:
The way this code is structured its writing two mutations. Wondering if it could write everything in a single conditional mutation. The conditional mutation would have the following
* condition that reservation column is absent
* condition that status column is absent
* set status column to NEW
* set reservation column to expected reservation
* set key column
* set time column
I suspect the way it structured has to do with the NOT_RESERVED, but I do not understand why it needs to be this way.
##########
core/src/main/java/org/apache/accumulo/core/fate/user/FateStatusFilter.java:
##########
@@ -54,8 +55,15 @@ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> op
@Override
public boolean accept(Key k, Value v) {
- var tstatus = ReadOnlyFateStore.TStatus.valueOf(v.toString());
- return valuesToAccept.contains(tstatus);
+ // We may see TStatus values or FateReservation values with how this filter is used,
+ // only accept TStatus values, return false on FateReservation values, error otherwise
+ try {
+ var tstatus = ReadOnlyFateStore.TStatus.valueOf(v.toString());
+ return valuesToAccept.contains(tstatus);
+ } catch (IllegalArgumentException e) {
Review Comment:
This iterator may need to be reworked. The iterator should probably return a the status and reservation column for row only if the status matches. This iterator is looking at individual key values, so it can not consider the higher level of the row. Could do the following
* Make this class instead extent WholeRowIterator
* Override the `boolean filter(Text currentRow, List<Key> keys, List<Value> values)` method in WholeRowIterator and make it look for the status column and check its value to determine wether to keep the row.
* Change the `configureScanner` method on this class to either setup this filter or just a WholeRowIterator w/o filtering when all statuses are requested.
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -224,9 +322,20 @@ public void setStatus(TStatus status) {
verifyReserved(true);
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(status).serialize(),
- NodeExistsPolicy.OVERWRITE);
- } catch (KeeperException | InterruptedException e) {
+ zk.mutateExisting(getTXPath(fateId), currSerializedData -> {
+ NodeValue currNodeVal = new NodeValue(currSerializedData);
+ // Ensure the FateId is reserved in ZK, and it is reserved with the expected reservation
+ if (currNodeVal.isReservedBy(this.reservation)) {
+ FateReservation currFateReservation = currNodeVal.reservation.orElseThrow();
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ NodeValue newNodeValue = new NodeValue(status, currFateReservation, currFateKey);
+ return newNodeValue.serialize();
+ } else {
+ throw new IllegalStateException("Either the FateId is not reserved in ZK, or it is"
+ + " but the reservation in ZK differs from that in the store.");
+ }
+ });
+ } catch (KeeperException | InterruptedException | AcceptableThriftTableOperationException e) {
Review Comment:
For this one we expect the node to exist in zookeeper, so if we see a NoNodeException here its ok throw an exception.
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -120,35 +127,161 @@ public FateId getFateId() {
}
@Override
- protected void create(FateId fateId, FateKey fateKey) {
- final int maxAttempts = 5;
-
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+ Optional<FateTxStore<T>> txStore = Optional.empty();
+ int maxAttempts = 5;
+ FateMutator.Status status = null;
+
+ // We first need to write the initial/unreserved value for the reservation column
+ // Only need to retry if it is UNKNOWN
for (int attempt = 0; attempt < maxAttempts; attempt++) {
-
- if (attempt >= 1) {
- log.debug("Failed to create transaction with fateId {} and fateKey {}, trying again",
- fateId, fateKey);
- UtilWaitThread.sleep(100);
+ status = newMutator(fateId).putInitReservationVal().tryMutate();
+ if (status != FateMutator.Status.UNKNOWN) {
+ break;
}
+ UtilWaitThread.sleep(100);
+ }
+ for (int attempt = 0; attempt < maxAttempts; attempt++) {
+ status = newMutator(fateId).requireStatus().putStatus(TStatus.NEW).putKey(fateKey)
+ .putReservedTx(reservation).putCreateTime(System.currentTimeMillis()).tryMutate();
+ if (status != FateMutator.Status.UNKNOWN) {
+ break;
+ }
+ UtilWaitThread.sleep(100);
+ }
- var status = newMutator(fateId).requireStatus().putStatus(TStatus.NEW).putKey(fateKey)
- .putCreateTime(System.currentTimeMillis()).tryMutate();
+ switch (status) {
+ case ACCEPTED:
+ txStore = Optional.of(new FateTxStoreImpl(fateId, reservation));
+ break;
+ case REJECTED:
+ // If the status is REJECTED, we need to check what about the mutation was REJECTED:
+ // 1) Possible something like the following occurred:
+ // the first attempt was UNKNOWN but written, the next attempt would be rejected
+ // We return the FateTxStore in this case.
+ // 2) If there is a collision with existing fate id, throw error
+ // 3) If the fate id is already reserved, return an empty optional
+ // 4) If the fate id is still NEW/unseeded and unreserved, we can try to reserve it
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.STATUS_COLUMN.getColumnFamily(),
+ TxColumnFamily.STATUS_COLUMN.getColumnQualifier());
+ scanner.fetchColumn(TxColumnFamily.TX_KEY_COLUMN.getColumnFamily(),
+ TxColumnFamily.TX_KEY_COLUMN.getColumnQualifier());
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ TStatus statusSeen = TStatus.UNKNOWN;
+ Optional<FateKey> fateKeySeen = Optional.empty();
+ Optional<FateReservation> reservationSeen = Optional.empty();
+
+ for (Entry<Key,Value> entry : scanner.stream().collect(Collectors.toList())) {
Review Comment:
```suggestion
for (Entry<Key,Value> entry : scanner) {
```
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -67,27 +76,36 @@ public FateId fromTypeAndKey(FateInstanceType instanceType, FateKey fateKey) {
}
};
- protected final Set<FateId> reserved;
+ // The ZooKeeper lock for the process that's running this store instance
+ protected final ZooUtil.LockID lockID;
+ protected final Predicate<ZooUtil.LockID> isLockHeld;
protected final Map<FateId,NanoTime> deferred;
+ protected final FateIdGenerator fateIdGenerator;
private final int maxDeferred;
private final AtomicBoolean deferredOverflow = new AtomicBoolean();
- private final FateIdGenerator fateIdGenerator;
-
- // This is incremented each time a transaction was unreserved that was non new
- protected final SignalCount unreservedNonNewCount = new SignalCount();
// This is incremented each time a transaction is unreserved that was runnable
- protected final SignalCount unreservedRunnableCount = new SignalCount();
+ private final SignalCount unreservedRunnableCount = new SignalCount();
+
+ // Keeps track of the number of concurrent callers to waitForStatusChange()
+ private final AtomicInteger concurrentStatusChangeCallers = new AtomicInteger(0);
public AbstractFateStore() {
- this(DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ this(null, null, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
}
- public AbstractFateStore(int maxDeferred, FateIdGenerator fateIdGenerator) {
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld) {
+ this(lockID, isLockHeld, DEFAULT_MAX_DEFERRED, DEFAULT_FATE_ID_GENERATOR);
+ }
+
+ public AbstractFateStore(ZooUtil.LockID lockID, Predicate<ZooUtil.LockID> isLockHeld,
+ int maxDeferred, FateIdGenerator fateIdGenerator) {
this.maxDeferred = maxDeferred;
this.fateIdGenerator = Objects.requireNonNull(fateIdGenerator);
- this.reserved = new HashSet<>();
- this.deferred = new HashMap<>();
+ this.deferred = Collections.synchronizedMap(new HashMap<>());
+ this.lockID = Objects.requireNonNullElseGet(lockID, AbstractFateStore::createDummyLockID);
Review Comment:
Falling back to calling createDummyLockID here could hide bugs. Could instead have the caller of this constructor that does not have a lockid call `AbstractFateStore.createDummyLockID()`. This removes the ambiguity of whether null was intentional or unintentional. This could be a follow on issue.
```suggestion
this.lockID = Objects.requireNonNull(lockID);
```
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -421,14 +569,30 @@ private Optional<FateKey> deserializeFateKey(DataInputBuffer buffer) throws IOEx
return Optional.empty();
}
+ private Optional<FateReservation> deserializeFateReservation(DataInputBuffer buffer)
+ throws IOException {
+ int length = buffer.readInt();
+ if (length > 0) {
Review Comment:
If negative values are not expected, could add a check for that.
```suggestion
int length = buffer.readInt();
Preconditions.checkArgument(length >=0);
if (length > 0) {
```
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -120,35 +127,161 @@ public FateId getFateId() {
}
@Override
- protected void create(FateId fateId, FateKey fateKey) {
- final int maxAttempts = 5;
-
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+ Optional<FateTxStore<T>> txStore = Optional.empty();
+ int maxAttempts = 5;
+ FateMutator.Status status = null;
+
+ // We first need to write the initial/unreserved value for the reservation column
+ // Only need to retry if it is UNKNOWN
for (int attempt = 0; attempt < maxAttempts; attempt++) {
-
- if (attempt >= 1) {
- log.debug("Failed to create transaction with fateId {} and fateKey {}, trying again",
- fateId, fateKey);
- UtilWaitThread.sleep(100);
+ status = newMutator(fateId).putInitReservationVal().tryMutate();
+ if (status != FateMutator.Status.UNKNOWN) {
+ break;
}
+ UtilWaitThread.sleep(100);
+ }
+ for (int attempt = 0; attempt < maxAttempts; attempt++) {
+ status = newMutator(fateId).requireStatus().putStatus(TStatus.NEW).putKey(fateKey)
+ .putReservedTx(reservation).putCreateTime(System.currentTimeMillis()).tryMutate();
+ if (status != FateMutator.Status.UNKNOWN) {
+ break;
+ }
+ UtilWaitThread.sleep(100);
+ }
- var status = newMutator(fateId).requireStatus().putStatus(TStatus.NEW).putKey(fateKey)
- .putCreateTime(System.currentTimeMillis()).tryMutate();
+ switch (status) {
+ case ACCEPTED:
+ txStore = Optional.of(new FateTxStoreImpl(fateId, reservation));
+ break;
+ case REJECTED:
+ // If the status is REJECTED, we need to check what about the mutation was REJECTED:
+ // 1) Possible something like the following occurred:
+ // the first attempt was UNKNOWN but written, the next attempt would be rejected
+ // We return the FateTxStore in this case.
+ // 2) If there is a collision with existing fate id, throw error
+ // 3) If the fate id is already reserved, return an empty optional
+ // 4) If the fate id is still NEW/unseeded and unreserved, we can try to reserve it
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.STATUS_COLUMN.getColumnFamily(),
+ TxColumnFamily.STATUS_COLUMN.getColumnQualifier());
+ scanner.fetchColumn(TxColumnFamily.TX_KEY_COLUMN.getColumnFamily(),
+ TxColumnFamily.TX_KEY_COLUMN.getColumnQualifier());
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ TStatus statusSeen = TStatus.UNKNOWN;
+ Optional<FateKey> fateKeySeen = Optional.empty();
+ Optional<FateReservation> reservationSeen = Optional.empty();
+
+ for (Entry<Key,Value> entry : scanner.stream().collect(Collectors.toList())) {
+ Text colf = entry.getKey().getColumnFamily();
+ Text colq = entry.getKey().getColumnQualifier();
+ Value val = entry.getValue();
+
+ switch (colq.toString()) {
+ case TxColumnFamily.STATUS:
+ statusSeen = TStatus.valueOf(val.toString());
+ break;
+ case TxColumnFamily.TX_KEY:
+ fateKeySeen = Optional.of(FateKey.deserialize(val.get()));
+ break;
+ case TxColumnFamily.RESERVATION:
+ if (FateReservation.isFateReservation(val.get())) {
+ reservationSeen = Optional.of(FateReservation.deserialize(val.get()));
+ }
+ break;
+ default:
+ throw new IllegalStateException("Unexpected column seen: " + colf + ":" + colq);
+ }
+ }
- switch (status) {
- case ACCEPTED:
- return;
- case UNKNOWN:
- continue;
- case REJECTED:
- throw new IllegalStateException("Attempt to create transaction with fateId " + fateId
- + " and fateKey " + fateKey + " was rejected");
- default:
- throw new IllegalStateException("Unknown status " + status);
+ // This will be the case if the mutation status is REJECTED but the mutation was written
+ if (statusSeen == TStatus.NEW && reservationSeen.isPresent()
+ && reservationSeen.orElseThrow().equals(reservation)) {
+ verifyFateKey(fateId, fateKeySeen, fateKey);
+ txStore = Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (statusSeen == TStatus.NEW && reservationSeen.isEmpty()) {
+ verifyFateKey(fateId, fateKeySeen, fateKey);
+ // NEW/unseeded transaction and not reserved, so we can allow it to be reserved
+ // we tryReserve() since another thread may have reserved it since the scan
+ txStore = tryReserve(fateId);
Review Comment:
```suggestion
txStore = tryReserve(fateId);
// the status was known before reserving to be NEW, however it could change so check after reserving to avoid race conditions.
var statusAfterReserve = txStore.map(ReadOnlyFateTxStore::getStatus).orElse(TStatus.UNKNOWN);
if (statusAfterReserve != TStatus.NEW) {
txStore.ifPresent(txs->txs.unreserve(Duration.ZERO));
txStore = Optional.empty();
}
```
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -224,9 +322,20 @@ public void setStatus(TStatus status) {
verifyReserved(true);
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(status).serialize(),
- NodeExistsPolicy.OVERWRITE);
- } catch (KeeperException | InterruptedException e) {
+ zk.mutateExisting(getTXPath(fateId), currSerializedData -> {
+ NodeValue currNodeVal = new NodeValue(currSerializedData);
+ // Ensure the FateId is reserved in ZK, and it is reserved with the expected reservation
+ if (currNodeVal.isReservedBy(this.reservation)) {
+ FateReservation currFateReservation = currNodeVal.reservation.orElseThrow();
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ NodeValue newNodeValue = new NodeValue(status, currFateReservation, currFateKey);
+ return newNodeValue.serialize();
+ } else {
+ throw new IllegalStateException("Either the FateId is not reserved in ZK, or it is"
Review Comment:
For this exception it would be useful to include the fateId. In general any exception including the fate id in the message could be very helpful for debugging problems. This could be a follow on issue.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688550936
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -298,7 +299,13 @@ private Optional<FateId> create(FateKey fateKey) {
@Override
public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
- FateId fateId = fateIdGenerator.fromTypeAndKey(getInstanceType(), fateKey);
+ // TODO 4131 not confident about this new implementation of createAndReserve.
Review Comment:
Implemented in User/MetaFateStore.createAndReserve(). Thank you for this detailed description.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688556368
##########
core/src/main/java/org/apache/accumulo/core/fate/FateStore.java:
##########
@@ -107,11 +118,155 @@ interface FateTxStore<T> extends ReadOnlyFateTxStore<T> {
void unreserve(Duration deferTime);
}
+ /**
+ * The value stored to indicate a FATE transaction ID ({@link FateId}) has been reserved
+ */
+ class FateReservation {
+
+ // The LockID (provided by the Manager running the FATE which uses this store) which is used for
+ // identifying dead Managers, so their reservations can be deleted and picked up again since
+ // they can no longer be worked on.
+ private final ZooUtil.LockID lockID; // TODO 4131 not sure if this is the best type for this
+ // The UUID generated on a reservation attempt (tryReserve()) used to uniquely identify that
+ // attempt. This is useful for the edge case where the reservation is sent to the server
+ // (Tablet Server for UserFateStore and the ZooKeeper Server for MetaFateStore), but the server
+ // dies before the store receives the response. It allows us to determine if the reservation
+ // was successful and was written by this reservation attempt (could have been successfully
+ // reserved by another attempt or not reserved at all, in which case, we wouldn't want to
+ // expose a FateTxStore).
+ private final UUID reservationUUID;
+ private final byte[] serialized;
+ private static final Pattern UUID_PATTERN =
+ Pattern.compile("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$");
+ private static final Pattern LOCKID_PATTERN = Pattern.compile("^.+/.+\\$[0-9a-fA-F]+$");
+
+ private FateReservation(ZooUtil.LockID lockID, UUID reservationUUID) {
+ this.lockID = Objects.requireNonNull(lockID);
+ this.reservationUUID = Objects.requireNonNull(reservationUUID);
+ this.serialized = serialize(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(ZooUtil.LockID lockID, UUID reservationUUID) {
+ return new FateReservation(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(byte[] serialized) {
+ try (DataInputBuffer buffer = new DataInputBuffer()) {
+ buffer.reset(serialized, serialized.length);
+ ZooUtil.LockID lockID = new ZooUtil.LockID("", buffer.readUTF());
+ UUID reservationUUID = UUID.fromString(buffer.readUTF());
+ return new FateReservation(lockID, reservationUUID);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static FateReservation from(String fateReservationStr) {
+ if (isFateReservation(fateReservationStr)) {
+ String[] fields = fateReservationStr.split(":");
+ ZooUtil.LockID lockId = new ZooUtil.LockID("", fields[0]);
+ UUID reservationUUID = UUID.fromString(fields[1]);
+ return new FateReservation(lockId, reservationUUID);
+ } else {
+ throw new IllegalArgumentException(
+ "Tried to create a FateReservation from an invalid string: " + fateReservationStr);
+ }
+ }
+
+ /**
+ *
+ * @param fateReservationStr the string from a call to {@link FateReservation#toString()}
+ * @return true if the string represents a valid FateReservation object, false otherwise
+ */
+ public static boolean isFateReservation(String fateReservationStr) {
+ if (fateReservationStr != null) {
+ String[] fields = fateReservationStr.split(":");
+ if (fields.length == 2) {
+ return LOCKID_PATTERN.matcher(fields[0]).matches()
+ && UUID_PATTERN.matcher(fields[1]).matches();
+ }
+ }
+ return false;
+ }
+
+ public ZooUtil.LockID getLockID() {
+ return lockID;
+ }
+
+ public UUID getReservationUUID() {
+ return reservationUUID;
+ }
+
+ public byte[] getSerialized() {
+ return serialized;
+ }
+
+ private static byte[] serialize(ZooUtil.LockID lockID, UUID reservationUUID) {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos)) {
+ dos.writeUTF(lockID.serialize("/"));
+ dos.writeUTF(reservationUUID.toString());
+ dos.close();
Review Comment:
I suspect close is called here to ensure all data is flushed in the DataOutputStream before calling `baos.toByteArray()` which is under the DataOutputStream. Seems like calling `flush()` would be ok.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688547495
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -360,7 +380,27 @@ public Fate(T environment, FateStore<T> store, Function<Repo<T>,String> toLogStr
}
}
}, 3, SECONDS));
- this.executor = pool;
+ this.transactionExecutor = pool;
+
+ // Create a dead reservation cleaner for this store that will periodically (every 30 seconds)
+ // clean up reservations held by dead processes, if they exist. Only created if a dead
+ // reservation cleaner is not already running for the given store type.
+ // TODO 4131 periodic cleanup runs every 30 seconds
+ // Should this be longer? Shorter? A configurable Property? A function of something?
+ ScheduledExecutorService deadResCleanerExecutor = ThreadPools.getServerThreadPools()
+ .createScheduledExecutorService(1, store.type() + "-dead-reservation-cleaner-pool");
+ if ((store.type() == FateInstanceType.USER && !userDeadReservationCleanerRunning)
+ || (store.type() == FateInstanceType.META && !metaDeadReservationCleanerRunning)) {
+ ScheduledFuture<?> deadReservationCleaner = deadResCleanerExecutor
+ .scheduleWithFixedDelay(new DeadReservationCleaner(), 3, 30, SECONDS);
+ ThreadPools.watchCriticalScheduledTask(deadReservationCleaner);
+ if (store.type() == FateInstanceType.USER) {
+ userDeadReservationCleanerRunning = true;
+ } else if (store.type() == FateInstanceType.META) {
+ metaDeadReservationCleanerRunning = true;
+ }
+ }
Review Comment:
Added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1670934639
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -298,7 +299,13 @@ private Optional<FateId> create(FateKey fateKey) {
@Override
public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
- FateId fateId = fateIdGenerator.fromTypeAndKey(getInstanceType(), fateKey);
+ // TODO 4131 not confident about this new implementation of createAndReserve.
Review Comment:
The way this method previously worked and the way the code was structured it was driven by the lock object being in AbstractFateStore. Now that there is no longer a lock we want to implement the following algorithm for both stores using conditional updates (@cshannon you added the fatekey feature, does the following algorithm seem ok to you?).
1. var fateId = FateId fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
2. in the store conditionally create fateId setting the status, reservation and fateKey atomically at creation time. The condition should check that nothing currently exist for the fateId, its expected to be absent.
3. If the conditional update fails for some reason, read the fateId and see if it already has the expected status, fateKey and reservation. If it does, then probably running for a second time and already reserved. If it does not then need to take the following actions.
1. If the fateId exists and has the same fateKey, but different status (should have NEW status) or reservation than expected then return Optional.empty(). This is the case where another thread has already created and reserved using the fateKey so there is nothing to do.
2. If the fateId exists and has different fateKey or no fateKey then this represents an unexpected collision ( this unexpected with the 128 bit fate id) so throw an exception in this case.
5. If the conditional update succeed , then we can return a non empty Optional w/ a FateTxStore obj.
Now that the lock object is no longer needed in AbstractFateStore it may make sense to completely push the full implementation of the `createAndReserve` method into MetaFateStore and UserFateStore and have nothing in AbstractFateStore for the method. The reason I am thinking this may be better is because the algorithm above is based on conditional updates now and how ZK and accumulo do conditional updates is very different. However there may still be opportunities to share common code, so not sure of the best way to structure the code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672878600
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
>If going with the NOT_RESERVED value approach it would be good to make the validation of the column value more strict, will make a comment about that elsewhere.
Nevermind the code does not work exactly the way I thought it did. I was thinking when reading reservations could do something like the following
```java
if(value look like <lock id>:<uuid) {
// process reserved
} else if (value is NOT_RESERVED) {
//process not reserved
} else {
throw new IllegalArgumentException("invalid reseved column value");
}
```
However the is not a place in the code that reads reservations in this way. Was thinking the code should fail if the reserved column value was something like '`1234` because it does not look like a reservation and it also does not look like NOT_RESERVED
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1674032301
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
Ah I understand now. Yes that should (but currently doesn't) throw an error if the entry is not NOT_RESERVED and not a valid FateReservation str. Since I will be changing the column to only ever include a valid FateReservation (column created and deleted as needed), I will just change this to throw an error if it sees anything but a FateReservation string. Thanks for the clarification
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672972761
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
+ .orElse(null);
+ if (persistedRes != null && persistedRes.equals(reservation)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ }
+ } catch (TableNotFoundException e) {
+ throw new IllegalStateException(tableName + " not found!", e);
+ }
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ return scanner.stream()
+ .map(entry -> FateReservation.isFateReservation(entry.getValue().toString())).findFirst()
+ .orElse(false);
+ } catch (TableNotFoundException e) {
+ throw new IllegalStateException(tableName + " not found!", e);
+ }
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ Map<FateId,FateReservation> activeReservations = new HashMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(new Range());
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
Review Comment:
If we had buggy code that was writing out a reservation that looked like `<lockid>:<uuid>:<uuid>` I think `isFateReservation` would return false for this (not completley sure, hard to follow the curly braces in the diff). I think it will split the string on `:` and return false if does not have two fields. Would be better to make this more strict and throw an exception when unexpected data like this is seen.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672826477
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -73,9 +75,12 @@ public class Fate<T> {
private final FateStore<T> store;
private final T environment;
private final ScheduledThreadPoolExecutor fatePoolWatcher;
- private final ExecutorService executor;
+ private final ExecutorService transactionExecutor;
+ private final ExecutorService deadResCleanerExecutor;
private static final EnumSet<TStatus> FINISHED_STATES = EnumSet.of(FAILED, SUCCESSFUL, UNKNOWN);
+ private static boolean userDeadReservationCleanerRunning = false;
+ private static boolean metaDeadReservationCleanerRunning = false;
Review Comment:
> Should these be Atomic? Or is it fine to keep them as-is?
Right now, there is only one Manager which runs, so these do not need to be Atomic, but the goal is to have multiple managers in the future, so not sure if this should be changed now or in the future.
Edit for further clarification: Only one thread will be accessing these at a time (the Manager). The only places these are accessed or used are in the constructor for Fate, and in Fate.shutdown(), neither of which have multiple threads executing. This will change when/if more than 1 Manager is running in the future.
> Can these static variables could be removed? Do these help testing in someway? In the manager we can assume that only a single instance of each type is created. If we wanted to guard against multiple instances per type, then a more general check could be done that is not limited to dead reservation cleaners.
The purpose of these was to avoid creating more than 1 DeadReservationCleaner running per store type. As I currently have this, the first Fate object to be created per store type will be running a DeadReservationCleaner (in the future, this probably should be changed as running on the first Fate object created may not be the best approach). In the Manager, only one Fate object will be created per store type, so this isn't a necessary check now, but isn't the plan in the future to have multiple Fate instance running per store type, so in that case we'd only ever want 1 DeadReservationCleaner running? I was premtively handling this case. What do you have in mind regarding a more general check? Or should I not be handling this case yet?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2272200690
I resolved a few of the comments.
> 4. There was a TODO regarding whether the ZooUtil.LockID was the best type for the lock. Should I make a follow on issue regarding this or is ZooUtil.LockID okay?
That type seems like the best exiting type to used, so I do not think an issue is needed.
> The DeadReservationCleaner runs every 30 seconds. Is this okay? Should this be longer? Shorter?
30 seconds seems too short to me. Would be doing a lot of work and usually never finding anything, should probably be longer maybe 2 or 3 minutes.
> Could you elaborate on what you mean by "all of the Fate data into a single node". By "all" do you mean the Repos, TxInfo, Status, Reservation, and FateKey would be in one ZK node? Writing the issue, I want to be sure I'm understanding correctly.
I was thinking of putting all data that is related to a single fate id into a single zk node. This will allow atomic updates. It will be slower (always have to rewrite all repos when adding/removing). We are only using it for the metadata table. So thinking it will make things more correct at the expense of speed but that tradeoff is good for metadata related fates.
> Can these be safely deleted without creating follow on issue(s)? If follow on issues should be made, perhaps it would be best for you to write these or explain it here because I am not sure what should be done.
Yeah you can delete those. I will look into it and open an issue if needed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700579912
##########
core/src/main/java/org/apache/accumulo/core/fate/FateStore.java:
##########
@@ -107,11 +118,155 @@ interface FateTxStore<T> extends ReadOnlyFateTxStore<T> {
void unreserve(Duration deferTime);
}
+ /**
+ * The value stored to indicate a FATE transaction ID ({@link FateId}) has been reserved
+ */
+ class FateReservation {
+
+ // The LockID (provided by the Manager running the FATE which uses this store) which is used for
+ // identifying dead Managers, so their reservations can be deleted and picked up again since
+ // they can no longer be worked on.
+ private final ZooUtil.LockID lockID; // TODO 4131 not sure if this is the best type for this
+ // The UUID generated on a reservation attempt (tryReserve()) used to uniquely identify that
+ // attempt. This is useful for the edge case where the reservation is sent to the server
+ // (Tablet Server for UserFateStore and the ZooKeeper Server for MetaFateStore), but the server
+ // dies before the store receives the response. It allows us to determine if the reservation
+ // was successful and was written by this reservation attempt (could have been successfully
+ // reserved by another attempt or not reserved at all, in which case, we wouldn't want to
+ // expose a FateTxStore).
+ private final UUID reservationUUID;
+ private final byte[] serialized;
+ private static final Pattern UUID_PATTERN =
+ Pattern.compile("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$");
+ private static final Pattern LOCKID_PATTERN = Pattern.compile("^.+/.+\\$[0-9a-fA-F]+$");
+
+ private FateReservation(ZooUtil.LockID lockID, UUID reservationUUID) {
+ this.lockID = Objects.requireNonNull(lockID);
+ this.reservationUUID = Objects.requireNonNull(reservationUUID);
+ this.serialized = serialize(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(ZooUtil.LockID lockID, UUID reservationUUID) {
+ return new FateReservation(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(byte[] serialized) {
+ try (DataInputBuffer buffer = new DataInputBuffer()) {
+ buffer.reset(serialized, serialized.length);
+ ZooUtil.LockID lockID = new ZooUtil.LockID("", buffer.readUTF());
+ UUID reservationUUID = UUID.fromString(buffer.readUTF());
+ return new FateReservation(lockID, reservationUUID);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static FateReservation from(String fateReservationStr) {
+ if (isFateReservation(fateReservationStr)) {
+ String[] fields = fateReservationStr.split(":");
+ ZooUtil.LockID lockId = new ZooUtil.LockID("", fields[0]);
+ UUID reservationUUID = UUID.fromString(fields[1]);
+ return new FateReservation(lockId, reservationUUID);
+ } else {
+ throw new IllegalArgumentException(
+ "Tried to create a FateReservation from an invalid string: " + fateReservationStr);
+ }
+ }
+
+ /**
+ *
+ * @param fateReservationStr the string from a call to {@link FateReservation#toString()}
+ * @return true if the string represents a valid FateReservation object, false otherwise
+ */
+ public static boolean isFateReservation(String fateReservationStr) {
+ if (fateReservationStr != null) {
+ String[] fields = fateReservationStr.split(":");
+ if (fields.length == 2) {
+ return LOCKID_PATTERN.matcher(fields[0]).matches()
+ && UUID_PATTERN.matcher(fields[1]).matches();
+ }
+ }
+ return false;
+ }
+
+ public ZooUtil.LockID getLockID() {
+ return lockID;
+ }
+
+ public UUID getReservationUUID() {
+ return reservationUUID;
+ }
+
+ public byte[] getSerialized() {
+ return serialized;
+ }
+
+ private static byte[] serialize(ZooUtil.LockID lockID, UUID reservationUUID) {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos)) {
+ dos.writeUTF(lockID.serialize("/"));
+ dos.writeUTF(reservationUUID.toString());
+ dos.close();
Review Comment:
I'll leave it as-is unless there is a significant difference between the two. If it's changed here, it should probably be changed everywhere this is done, which shouldn't be done in this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700589058
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -120,35 +127,161 @@ public FateId getFateId() {
}
@Override
- protected void create(FateId fateId, FateKey fateKey) {
- final int maxAttempts = 5;
-
+ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
+ final var reservation = FateReservation.from(lockID, UUID.randomUUID());
+ final var fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
+ Optional<FateTxStore<T>> txStore = Optional.empty();
+ int maxAttempts = 5;
+ FateMutator.Status status = null;
+
+ // We first need to write the initial/unreserved value for the reservation column
+ // Only need to retry if it is UNKNOWN
for (int attempt = 0; attempt < maxAttempts; attempt++) {
-
- if (attempt >= 1) {
- log.debug("Failed to create transaction with fateId {} and fateKey {}, trying again",
- fateId, fateKey);
- UtilWaitThread.sleep(100);
+ status = newMutator(fateId).putInitReservationVal().tryMutate();
+ if (status != FateMutator.Status.UNKNOWN) {
+ break;
}
+ UtilWaitThread.sleep(100);
+ }
+ for (int attempt = 0; attempt < maxAttempts; attempt++) {
+ status = newMutator(fateId).requireStatus().putStatus(TStatus.NEW).putKey(fateKey)
+ .putReservedTx(reservation).putCreateTime(System.currentTimeMillis()).tryMutate();
+ if (status != FateMutator.Status.UNKNOWN) {
+ break;
+ }
+ UtilWaitThread.sleep(100);
+ }
- var status = newMutator(fateId).requireStatus().putStatus(TStatus.NEW).putKey(fateKey)
- .putCreateTime(System.currentTimeMillis()).tryMutate();
+ switch (status) {
+ case ACCEPTED:
+ txStore = Optional.of(new FateTxStoreImpl(fateId, reservation));
+ break;
+ case REJECTED:
+ // If the status is REJECTED, we need to check what about the mutation was REJECTED:
+ // 1) Possible something like the following occurred:
+ // the first attempt was UNKNOWN but written, the next attempt would be rejected
+ // We return the FateTxStore in this case.
+ // 2) If there is a collision with existing fate id, throw error
+ // 3) If the fate id is already reserved, return an empty optional
+ // 4) If the fate id is still NEW/unseeded and unreserved, we can try to reserve it
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.STATUS_COLUMN.getColumnFamily(),
+ TxColumnFamily.STATUS_COLUMN.getColumnQualifier());
+ scanner.fetchColumn(TxColumnFamily.TX_KEY_COLUMN.getColumnFamily(),
+ TxColumnFamily.TX_KEY_COLUMN.getColumnQualifier());
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ TStatus statusSeen = TStatus.UNKNOWN;
+ Optional<FateKey> fateKeySeen = Optional.empty();
+ Optional<FateReservation> reservationSeen = Optional.empty();
+
+ for (Entry<Key,Value> entry : scanner.stream().collect(Collectors.toList())) {
Review Comment:
haha fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1700585082
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -224,9 +322,20 @@ public void setStatus(TStatus status) {
verifyReserved(true);
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(status).serialize(),
- NodeExistsPolicy.OVERWRITE);
- } catch (KeeperException | InterruptedException e) {
+ zk.mutateExisting(getTXPath(fateId), currSerializedData -> {
+ NodeValue currNodeVal = new NodeValue(currSerializedData);
+ // Ensure the FateId is reserved in ZK, and it is reserved with the expected reservation
+ if (currNodeVal.isReservedBy(this.reservation)) {
+ FateReservation currFateReservation = currNodeVal.reservation.orElseThrow();
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ NodeValue newNodeValue = new NodeValue(status, currFateReservation, currFateKey);
+ return newNodeValue.serialize();
+ } else {
+ throw new IllegalStateException("Either the FateId is not reserved in ZK, or it is"
Review Comment:
Yes, I must have forgotten it here. Added and included the reservation in the log as well. Double checked AFS, UFS, and MFS that everywhere something is logged or thrown regarding a fate id that the fate id is included in the message, so no follow on issue needed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "DomGarguilo (via GitHub)" <gi...@apache.org>.
DomGarguilo commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1663115683
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
+ .orElse(null);
+ if (persistedRes != null && persistedRes.equals(reservation)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ }
Review Comment:
```suggestion
return scanner.stream().map(entry -> entry.getValue().toString())
.filter(FateReservation::isFateReservation).map(FateReservation::from)
.filter(persistedRes -> persistedRes.equals(reservation)).findFirst()
.map(persistedRes -> new FateTxStoreImpl(fateId, reservation));
```
These could be combined
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -360,7 +380,27 @@ public Fate(T environment, FateStore<T> store, Function<Repo<T>,String> toLogStr
}
}
}, 3, SECONDS));
- this.executor = pool;
+ this.transactionExecutor = pool;
+
+ // Create a dead reservation cleaner for this store that will periodically (every 30 seconds)
+ // clean up reservations held by dead processes, if they exist. Only created if a dead
+ // reservation cleaner is not already running for the given store type.
+ // TODO 4131 periodic cleanup runs every 30 seconds
+ // Should this be longer? Shorter? A configurable Property? A function of something?
+ ScheduledExecutorService deadResCleanerExecutor = ThreadPools.getServerThreadPools()
+ .createScheduledExecutorService(1, store.type() + "-dead-reservation-cleaner-pool");
+ if ((store.type() == FateInstanceType.USER && !userDeadReservationCleanerRunning)
+ || (store.type() == FateInstanceType.META && !metaDeadReservationCleanerRunning)) {
+ ScheduledFuture<?> deadReservationCleaner = deadResCleanerExecutor
+ .scheduleWithFixedDelay(new DeadReservationCleaner(), 3, 30, SECONDS);
+ ThreadPools.watchCriticalScheduledTask(deadReservationCleaner);
+ if (store.type() == FateInstanceType.USER) {
+ userDeadReservationCleanerRunning = true;
+ } else if (store.type() == FateInstanceType.META) {
+ metaDeadReservationCleanerRunning = true;
+ }
+ }
Review Comment:
```suggestion
boolean isUserStore = store.type() == FateInstanceType.USER;
boolean isMetaStore = store.type() == FateInstanceType.META;
if ((isUserStore && !userDeadReservationCleanerRunning)
|| (isMetaStore && !metaDeadReservationCleanerRunning)) {
ScheduledFuture<?> deadReservationCleaner = deadResCleanerExecutor
.scheduleWithFixedDelay(new DeadReservationCleaner(), 3, 30, SECONDS);
ThreadPools.watchCriticalScheduledTask(deadReservationCleaner);
if (isUserStore) {
userDeadReservationCleanerRunning = true;
} else {
metaDeadReservationCleanerRunning = true;
}
}
```
Refactoring out these conditions might help with readability a tiny bit.
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -73,9 +75,12 @@ public class Fate<T> {
private final FateStore<T> store;
private final T environment;
private final ScheduledThreadPoolExecutor fatePoolWatcher;
- private final ExecutorService executor;
+ private final ExecutorService transactionExecutor;
+ private final ExecutorService deadResCleanerExecutor;
private static final EnumSet<TStatus> FINISHED_STATES = EnumSet.of(FAILED, SUCCESSFUL, UNKNOWN);
+ private static boolean userDeadReservationCleanerRunning = false;
+ private static boolean metaDeadReservationCleanerRunning = false;
Review Comment:
Should these be Atomic? Or is it fine to keep them as-is?
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -360,7 +380,27 @@ public Fate(T environment, FateStore<T> store, Function<Repo<T>,String> toLogStr
}
}
}, 3, SECONDS));
- this.executor = pool;
+ this.transactionExecutor = pool;
+
+ // Create a dead reservation cleaner for this store that will periodically (every 30 seconds)
+ // clean up reservations held by dead processes, if they exist. Only created if a dead
+ // reservation cleaner is not already running for the given store type.
+ // TODO 4131 periodic cleanup runs every 30 seconds
+ // Should this be longer? Shorter? A configurable Property? A function of something?
+ ScheduledExecutorService deadResCleanerExecutor = ThreadPools.getServerThreadPools()
+ .createScheduledExecutorService(1, store.type() + "-dead-reservation-cleaner-pool");
+ if ((store.type() == FateInstanceType.USER && !userDeadReservationCleanerRunning)
+ || (store.type() == FateInstanceType.META && !metaDeadReservationCleanerRunning)) {
+ ScheduledFuture<?> deadReservationCleaner = deadResCleanerExecutor
+ .scheduleWithFixedDelay(new DeadReservationCleaner(), 3, 30, SECONDS);
+ ThreadPools.watchCriticalScheduledTask(deadReservationCleaner);
+ if (store.type() == FateInstanceType.USER) {
+ userDeadReservationCleanerRunning = true;
+ } else if (store.type() == FateInstanceType.META) {
+ metaDeadReservationCleanerRunning = true;
+ }
+ }
+ this.deadResCleanerExecutor = deadResCleanerExecutor;
Review Comment:
If the if statement above is never entered, will this executor ever be used? If not I wonder if we should avoid creating it. This is just a question, not sure what should be done here.
##########
core/src/main/java/org/apache/accumulo/core/fate/FateStore.java:
##########
@@ -107,11 +118,155 @@ interface FateTxStore<T> extends ReadOnlyFateTxStore<T> {
void unreserve(Duration deferTime);
}
+ /**
+ * The value stored to indicate a FATE transaction ID ({@link FateId}) has been reserved
+ */
+ class FateReservation {
+
+ // The LockID (provided by the Manager running the FATE which uses this store) which is used for
+ // identifying dead Managers, so their reservations can be deleted and picked up again since
+ // they can no longer be worked on.
+ private final ZooUtil.LockID lockID; // TODO 4131 not sure if this is the best type for this
+ // The UUID generated on a reservation attempt (tryReserve()) used to uniquely identify that
+ // attempt. This is useful for the edge case where the reservation is sent to the server
+ // (Tablet Server for UserFateStore and the ZooKeeper Server for MetaFateStore), but the server
+ // dies before the store receives the response. It allows us to determine if the reservation
+ // was successful and was written by this reservation attempt (could have been successfully
+ // reserved by another attempt or not reserved at all, in which case, we wouldn't want to
+ // expose a FateTxStore).
+ private final UUID reservationUUID;
+ private final byte[] serialized;
+ private static final Pattern UUID_PATTERN =
+ Pattern.compile("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$");
+ private static final Pattern LOCKID_PATTERN = Pattern.compile("^.+/.+\\$[0-9a-fA-F]+$");
+
+ private FateReservation(ZooUtil.LockID lockID, UUID reservationUUID) {
+ this.lockID = Objects.requireNonNull(lockID);
+ this.reservationUUID = Objects.requireNonNull(reservationUUID);
+ this.serialized = serialize(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(ZooUtil.LockID lockID, UUID reservationUUID) {
+ return new FateReservation(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(byte[] serialized) {
+ try (DataInputBuffer buffer = new DataInputBuffer()) {
+ buffer.reset(serialized, serialized.length);
+ ZooUtil.LockID lockID = new ZooUtil.LockID("", buffer.readUTF());
+ UUID reservationUUID = UUID.fromString(buffer.readUTF());
+ return new FateReservation(lockID, reservationUUID);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static FateReservation from(String fateReservationStr) {
+ if (isFateReservation(fateReservationStr)) {
+ String[] fields = fateReservationStr.split(":");
+ ZooUtil.LockID lockId = new ZooUtil.LockID("", fields[0]);
+ UUID reservationUUID = UUID.fromString(fields[1]);
+ return new FateReservation(lockId, reservationUUID);
+ } else {
+ throw new IllegalArgumentException(
+ "Tried to create a FateReservation from an invalid string: " + fateReservationStr);
+ }
+ }
+
+ /**
+ *
+ * @param fateReservationStr the string from a call to {@link FateReservation#toString()}
+ * @return true if the string represents a valid FateReservation object, false otherwise
+ */
+ public static boolean isFateReservation(String fateReservationStr) {
+ if (fateReservationStr != null) {
+ String[] fields = fateReservationStr.split(":");
+ if (fields.length == 2) {
+ return LOCKID_PATTERN.matcher(fields[0]).matches()
+ && UUID_PATTERN.matcher(fields[1]).matches();
+ }
+ }
+ return false;
+ }
+
+ public ZooUtil.LockID getLockID() {
+ return lockID;
+ }
+
+ public UUID getReservationUUID() {
+ return reservationUUID;
+ }
+
+ public byte[] getSerialized() {
+ return serialized;
+ }
+
+ private static byte[] serialize(ZooUtil.LockID lockID, UUID reservationUUID) {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos)) {
+ dos.writeUTF(lockID.serialize("/"));
+ dos.writeUTF(reservationUUID.toString());
+ dos.close();
Review Comment:
Is there a reason you are calling `close()` early? Could just call `flush()` instead if that is the intent and let the `try-with-resources` close things automatically.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1624536328
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -150,6 +150,39 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // TODO 4131 should this throw an exception if the id doesn't exist (status = UNKNOWN)?
+ FateMutator.Status status = newMutator(fateId).putReservedTx(fateId).tryMutate();
Review Comment:
Thank you for this detailed explanation. This was really helpful for https://github.com/apache/accumulo/pull/4524/commits/e6671fbb903da53014bd64e53ad125351add6a04
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1631616426
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -315,6 +420,30 @@ public List<ReadOnlyRepo<T>> getStack() {
return dops;
}
}
+
+ @Override
+ protected void unreserve() {
+ try {
+ if (!this.deleted) {
+ zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ if ((currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(this.reservation))) {
+ // Remove the FateReservation from the NodeValue to unreserve
+ return new NodeValue(currNodeVal.status, null, currFateKey).serialize();
+ } else {
+ // possible this is running a 2nd time in zk server fault conditions and its first
Review Comment:
Could add some logging here. If the reservation in ZK is not set, then could log something at trace. If there is a reservation in ZK that differs, it could be log something at debug. These may be useful later for debugging problems when initially using the elasticity changes.
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -421,14 +548,30 @@ private Optional<FateKey> deserializeFateKey(DataInputBuffer buffer) throws IOEx
return Optional.empty();
}
+ private Optional<FateReservation> deserializeFateReservation(DataInputBuffer buffer)
+ throws IOException {
+ int length = buffer.readInt();
+ if (length > 0) {
+ return Optional.of(FateReservation.deserialize(buffer.readNBytes(length)));
+ }
+ return Optional.empty();
+ }
+
byte[] serialize() {
try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos)) {
dos.writeUTF(status.name());
+ if (isReserved()) {
Review Comment:
Not a change to make in this PR. Thinking if we push all of the Fate data into a single node it would be nice to use JSON. That would make the data stored in ZK more human readable and it would make it easier to serialize and de-serialize.
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -224,9 +323,14 @@ public void setStatus(TStatus status) {
verifyReserved(true);
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(status).serialize(),
- NodeExistsPolicy.OVERWRITE);
- } catch (KeeperException | InterruptedException e) {
+ zk.mutateExisting(getTXPath(fateId), currSerializedData -> {
+ NodeValue currNodeVal = new NodeValue(currSerializedData);
Review Comment:
Somewhere in this lamba it should check a reservation exist for the ZK node and that the reservation on the ZK node is equal to AbstractFateTxStoreImpl.reserveration, if those conditions are not met then throw an exception. This check ensures that the expected state we have in memory aligns with what is in the persistent store.
Ideally these kinds of checks would be done for all updates, however it is not possible to atomically do these checks for data stored in other ZK nodes. The way FATE stores data in ZK it uses multiple nodes. The schema could be changed to store all data in a single node which would allow all updates to verify the reservation atomically, that is not a change for this PR. Would be a follow on issue.
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -106,13 +112,102 @@ public FateId create() {
@Override
protected void create(FateId fateId, FateKey key) {
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
+ zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, null, key).serialize(),
NodeExistsPolicy.FAIL);
} catch (KeeperException | InterruptedException e) {
throw new IllegalStateException(e);
}
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ // The uuid handles the case where there was a ZK server fault and the write for this thread
+ // went through but that was not acknowledged, and we are reading our own write for 2nd
+ // time.
+ if (!currNodeVal.isReserved() || (currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(reservation))) {
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ // Add the FateReservation to the node to reserve
+ return new NodeValue(currNodeVal.status, reservation, currFateKey).serialize();
+ } else {
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (newSerNodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ boolean isReserved;
+ try {
+ isReserved = getNode(fateId).isReserved();
+ } catch (Exception e) {
+ // Exception thrown, so node doesn't exist, so it is not reserved
+ isReserved = false;
+ }
Review Comment:
If possible this catch should be more narrow. Only want to ignore exceptions that would be expected for the case of it not existing. Other exceptions should bubble up so bugs can be detected. Also nice to log something when ignoring an exception.
```suggestion
} catch (NoNodeException e) {
log.trace("Saw {} when checking reservation of {}", e.getMessage(), fateId);
isReserved = false;
} catch (Exception e) {
throw new RuntimeException(e);
}
```
##########
test/src/main/java/org/apache/accumulo/test/fate/MultipleStoresIT.java:
##########
@@ -0,0 +1,571 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.fate;
+
+import static org.apache.accumulo.test.fate.user.UserFateStoreIT.createFateTable;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.fate.Fate;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.FateStore;
+import org.apache.accumulo.core.fate.MetaFateStore;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore;
+import org.apache.accumulo.core.fate.Repo;
+import org.apache.accumulo.core.fate.user.UserFateStore;
+import org.apache.accumulo.core.fate.zookeeper.ZooCache;
+import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.test.util.Wait;
+import org.apache.accumulo.test.zookeeper.ZooKeeperTestingServer;
+import org.easymock.EasyMock;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+// TODO 4131 could potentially have separate classes for testing MetaFateStore and UserFateStore
+// similar to how FateTestRunner is used, however that interface doesn't work as nicely here
+// since we are using multiple stores instead of just one. Can do something similar to
+// FateTestRunner here if desired
+public class MultipleStoresIT extends SharedMiniClusterBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MultipleStoresIT.class);
+ @TempDir
+ private static File tempDir;
+ private static ZooKeeperTestingServer szk = null;
+ private static ZooReaderWriter zk;
+ private static final String FATE_DIR = "/fate";
+ private ClientContext client;
+
+ @BeforeEach
+ public void beforeEachSetup() {
+ client = (ClientContext) Accumulo.newClient().from(getClientProps()).build();
+ }
+
+ @AfterEach
+ public void afterEachTeardown() {
+ client.close();
+ }
+
+ @BeforeAll
+ public static void beforeAllSetup() throws Exception {
+ SharedMiniClusterBase.startMiniCluster();
+ szk = new ZooKeeperTestingServer(tempDir);
+ zk = szk.getZooReaderWriter();
+ }
+
+ @AfterAll
+ public static void afterAllTeardown() throws Exception {
+ SharedMiniClusterBase.stopMiniCluster();
+ szk.close();
+ }
+
+ @Test
+ public void testReserveUnreserve() throws Exception {
+ testReserveUnreserve(FateInstanceType.META);
+ testReserveUnreserve(FateInstanceType.USER);
+ }
+
+ private void testReserveUnreserve(FateInstanceType storeType) throws Exception {
+ // reserving/unreserving a FateId should be reflected across instances of the stores
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> activeReservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1);
+ store2 = new UserFateStore<>(client, tableName, lock2);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock1);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock2);
+ }
+
+ // Create the fate ids using store1
+ for (int i = 0; i < numFateIds; i++) {
+ assertTrue(allIds.add(store1.create()));
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Reserve half the fate ids using store1 and rest using store2, after reserving a fate id in
+ // one, should not be able to reserve the same in the other. Should also not matter that all the
+ // ids were created using store1
+ int count = 0;
+ for (FateId fateId : allIds) {
+ if (count % 2 == 0) {
+ reservations.add(store1.reserve(fateId));
+ assertTrue(store2.tryReserve(fateId).isEmpty());
+ } else {
+ reservations.add(store2.reserve(fateId));
+ assertTrue(store1.tryReserve(fateId).isEmpty());
+ }
+ count++;
+ }
+ // Both stores should return the same reserved transactions
+ activeReservations = store1.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+ activeReservations = store2.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+
+ // Test setting/getting the TStatus and unreserving the transactions
+ for (int i = 0; i < allIds.size(); i++) {
+ var reservation = reservations.get(i);
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, reservation.getStatus());
+ reservation.setStatus(ReadOnlyFateStore.TStatus.SUBMITTED);
+ assertEquals(ReadOnlyFateStore.TStatus.SUBMITTED, reservation.getStatus());
+ reservation.delete();
+ reservation.unreserve(Duration.ofMillis(0));
+ // Attempt to set a status on a tx that has been unreserved (should throw exception)
+ assertThrows(IllegalStateException.class,
+ () -> reservation.setStatus(ReadOnlyFateStore.TStatus.NEW));
+ }
+ assertTrue(store1.getActiveReservations().isEmpty());
+ assertTrue(store2.getActiveReservations().isEmpty());
+ }
+
+ @Test
+ public void testReserveNonExistentTxn() throws Exception {
+ testReserveNonExistentTxn(FateInstanceType.META);
+ testReserveNonExistentTxn(FateInstanceType.USER);
+ }
+
+ private void testReserveNonExistentTxn(FateInstanceType storeType) throws Exception {
+ // Tests that reserve() doesn't hang indefinitely and instead throws an error
+ // on reserve() a non-existent transaction.
+ final FateStore<SleepingTestEnv> store;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final String tableName = getUniqueNames(1)[0];
+ final FateId fakeFateId = FateId.from(storeType, UUID.randomUUID());
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ assertThrows(IllegalStateException.class, () -> store.reserve(fakeFateId));
+ }
+
+ @Test
+ public void testReserveReservedAndUnreserveUnreserved() throws Exception {
+ testReserveReservedAndUnreserveUnreserved(FateInstanceType.META);
+ testReserveReservedAndUnreserveUnreserved(FateInstanceType.USER);
+ }
+
+ private void testReserveReservedAndUnreserveUnreserved(FateInstanceType storeType)
+ throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final FateStore<SleepingTestEnv> store;
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ // Create some FateIds and ensure that they can be reserved
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId = store.create();
+ assertTrue(allIds.add(fateId));
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservations.add(reservation.orElseThrow());
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Try to reserve again, should not reserve
+ for (FateId fateId : allIds) {
+ assertTrue(store.tryReserve(fateId).isEmpty());
+ }
+
+ // Unreserve all the FateIds
+ for (var reservation : reservations) {
+ reservation.unreserve(Duration.ofMillis(0));
+ }
+ // Try to unreserve again (should throw exception)
+ for (var reservation : reservations) {
+ assertThrows(IllegalStateException.class, () -> reservation.unreserve(Duration.ofMillis(0)));
+ }
+ }
+
+ @Test
+ public void testReserveAfterUnreserveAndReserveAfterDeleted() throws Exception {
+ testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType.META);
+ testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType.USER);
+ }
+
+ private void testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType storeType)
+ throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final FateStore<SleepingTestEnv> store;
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ // Create some FateIds and ensure that they can be reserved
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId = store.create();
+ assertTrue(allIds.add(fateId));
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservations.add(reservation.orElseThrow());
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Unreserve all
+ for (var reservation : reservations) {
+ reservation.unreserve(Duration.ofMillis(0));
+ }
+
+ // Ensure they can be reserved again, and delete and unreserve this time
+ for (FateId fateId : allIds) {
+ // Verify that the tx status is still NEW after unreserving since it hasn't been deleted
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, store.read(fateId).getStatus());
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservation.orElseThrow().delete();
+ reservation.orElseThrow().unreserve(Duration.ofMillis(0));
+ }
+
+ for (FateId fateId : allIds) {
+ // Verify that the tx is now unknown since it has been deleted
+ assertEquals(ReadOnlyFateStore.TStatus.UNKNOWN, store.read(fateId).getStatus());
+ // Attempt to reserve a deleted txn, should throw an exception and not wait indefinitely
+ assertThrows(IllegalStateException.class, () -> store.reserve(fateId));
+ }
+ }
+
+ @Test
+ public void testMultipleFateInstances() throws Exception {
+ testMultipleFateInstances(FateInstanceType.META);
+ testMultipleFateInstances(FateInstanceType.USER);
+ }
+
+ private void testMultipleFateInstances(FateInstanceType storeType) throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final SleepingTestEnv testEnv1 = new SleepingTestEnv(50);
+ final SleepingTestEnv testEnv2 = new SleepingTestEnv(50);
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1);
+ store2 = new UserFateStore<>(client, tableName, lock2);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock1);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock2);
+ }
+
+ Fate<SleepingTestEnv> fate1 =
+ new Fate<>(testEnv1, store1, Object::toString, DefaultConfiguration.getInstance());
+ fate1.startDeadReservationCleaner();
+ Fate<SleepingTestEnv> fate2 =
+ new Fate<>(testEnv2, store2, Object::toString, DefaultConfiguration.getInstance());
+ fate2.startDeadReservationCleaner();
+
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId;
+ // Start half the txns using fate1, and the other half using fate2
+ if (i % 2 == 0) {
+ fateId = fate1.startTransaction();
+ fate1.seedTransaction("op" + i, fateId, new SleepingTestRepo(), true, "test");
+ } else {
+ fateId = fate2.startTransaction();
+ fate2.seedTransaction("op" + i, fateId, new SleepingTestRepo(), true, "test");
+ }
+ allIds.add(fateId);
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Should be able to wait for completion on any fate instance
+ for (FateId fateId : allIds) {
+ fate2.waitForCompletion(fateId);
+ }
+ // Ensure that all txns have been executed and have only been executed once
+ assertTrue(Collections.disjoint(testEnv1.executedOps, testEnv2.executedOps));
+ assertEquals(allIds, Sets.union(testEnv1.executedOps, testEnv2.executedOps));
+
+ fate1.shutdown(1, TimeUnit.MINUTES);
+ fate2.shutdown(1, TimeUnit.MINUTES);
+ }
+
+ @Test
+ public void testDeadReservationsCleanup() throws Exception {
+ testDeadReservationsCleanup(FateInstanceType.META);
+ testDeadReservationsCleanup(FateInstanceType.USER);
+ }
+
+ private void testDeadReservationsCleanup(FateInstanceType storeType) throws Exception {
+ // Tests reserving some transactions, then simulating that the Manager died by creating
+ // a new Fate instance and store with a new LockID. The transactions which were
+ // reserved using the old LockID should be cleaned up by Fate's DeadReservationCleaner,
+ // then picked up by the new Fate/store.
+
+ final String tableName = getUniqueNames(1)[0];
+ // One transaction for each FATE worker thread
+ final int numFateIds =
+ Integer.parseInt(Property.MANAGER_FATE_THREADPOOL_SIZE.getDefaultValue());
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<LatchTestEnv> mockedStore1, store2;
+ final LatchTestEnv testEnv1 = new LatchTestEnv();
+ final LatchTestEnv testEnv2 = new LatchTestEnv();
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> reservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ mockedStore1 = EasyMock.createMockBuilder(UserFateStore.class)
+ .withConstructor(ClientContext.class, String.class, ZooUtil.LockID.class)
+ .withArgs(client, tableName, lock1).addMockedMethod("isDeadReservation").createMock();
+ } else {
+ mockedStore1 = EasyMock.createMockBuilder(MetaFateStore.class)
+ .withConstructor(String.class, ZooReaderWriter.class, ZooCache.class,
+ ZooUtil.LockID.class)
+ .withArgs(FATE_DIR, zk, client.getZooCache(), lock1).addMockedMethod("isDeadReservation")
+ .createMock();
Review Comment:
Not sure about this, but the following pattern may simplify this test over mocking where liveLocks is a `Set<ZooUtil.LockID>` that the test can add and remove things to as needed.
```suggestion
mockedStore1 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock1) {
@Override
public boolean isDeadReservation(FateReservation reservation) {
return !liveLocks.contains(reservation.getLockID());
}
};
```
Another thing to consider is passing in a `Predicate<ZooUtil.LockID>` to the Fate store constructors that can be used to check if a lock is live. Then the `isDeadReservation()` method could be dropped an stores could use this predicate internally. Test could pass in a predicate that checks a set. The real code could pass in a predicate taht calls the ServiceLock code.
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +154,76 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ status = newMutator(fateId).requireReserved(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ }
Review Comment:
Its possible that this status could be unknown, but I don't think we need to loop and try writing again in this case. I think we can avoid attempting another write here altogether (because this is a tryReserve method) and just scan and read the reservation column. If its set then we can return the operation w/ the reservation and if its not set then we can return emptty(). So that avoids looping or retrying.
```suggestion
var persistedReservation = // todo scan and read reservation
if(reservation.equals(persistedReservation)){
return Optional.of(new FateTxStoreImpl(fateId, reservation));
} // else fall through and return empty()
```
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -502,4 +520,14 @@ protected Serializable deserializeTxInfo(TxInfo txInfo, byte[] data) {
throw new IllegalStateException("Bad node data " + txInfo);
}
}
+
+ /**
+ * TODO 4131 this is a temporary method used to create a dummy lock when using a FateStore outside
Review Comment:
This would be a follow on issue. We can have a path in ZK for utilities to get a ZK lock. Currently I think some of the fate admin utilities that change the fate store require that no manager process is running. These utilities could be changed to get a lock at this utility path in ZK and reserve fate txs they operate on instead of requiring no manager process.
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +154,76 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ status = newMutator(fateId).requireReserved(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ }
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return newMutator(fateId).requireReserved().tryMutate().equals(FateMutator.Status.ACCEPTED);
Review Comment:
This should not write a mutation to check if something is reseved, it should scan and read the reserved column. The server side lock of a conditional mutation does not offer benefit here because after this method returns and something uses the returned value, the lock is no longer held. So may as well scan in that case.
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +154,76 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ status = newMutator(fateId).requireReserved(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ }
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return newMutator(fateId).requireReserved().tryMutate().equals(FateMutator.Status.ACCEPTED);
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ Map<FateId,FateReservation> activeReservations = new HashMap<>();
+
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(new Range());
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .forEach(entry -> {
+ String reservationColVal = entry.getValue().toString();
+ FateId fateId = FateId.from(fateInstanceType, entry.getKey().getRow().toString());
+ FateReservation reservation = FateReservation.from(reservationColVal);
+ activeReservations.put(fateId, reservation);
+ });
+ } catch (TableNotFoundException e) {
+ throw new IllegalStateException(tableName + " not found!", e);
+ }
+
+ return activeReservations;
+ }
+
+ @Override
+ public void deleteDeadReservations() {
+ for (Entry<FateId,FateReservation> entry : getActiveReservations().entrySet()) {
+ FateId fateId = entry.getKey();
+ FateReservation reservation = entry.getValue();
+ if (isDeadReservation(reservation)) {
+ newMutator(fateId).putUnreserveTx(reservation).tryMutate();
+ // No need to check the status... If it is ACCEPTED, we have successfully unreserved
Review Comment:
Nice description
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -313,6 +317,26 @@ private void undo(FateId fateId, Repo<T> op) {
}
+ /**
+ * A thread that finds reservations held by dead processes and unreserves them. Only one thread
+ * runs per store type across all Fate instances (one to clean up dead reservations for
+ * {@link org.apache.accumulo.core.fate.user.UserFateStore UserFateStore} and one to clean up dead
+ * reservations for {@link MetaFateStore}).
+ */
+ private class DeadReservationCleaner implements Runnable {
+ // TODO 4131 periodic check runs every 30 seconds
+ // Should this be longer? Shorter? A configurable Property? A function of something?
+ private static final long INTERVAL_MILLIS = 30_000;
+
+ @Override
+ public void run() {
+ while (keepRunning.get()) {
+ store.deleteDeadReservations();
Review Comment:
I have not looked into it, but need to understand what happens if this thread dies/throws an exception. Accumulo servers processes can designate some threads as critical and that would essentially cause the manager process to die when that is done. May want to do that for this thread, if its not already done.
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -272,8 +340,12 @@ public FateInstanceType type() {
private class FateTxStoreImpl extends AbstractFateTxStoreImpl<T> {
- private FateTxStoreImpl(FateId fateId, boolean isReserved) {
- super(fateId, isReserved);
+ private FateTxStoreImpl(FateId fateId) {
+ super(fateId);
+ }
+
+ private FateTxStoreImpl(FateId fateId, FateReservation reservation) {
Review Comment:
This would be a follow on issue, not something for this PR. For all of the writes made for a Fate tx we can now add a condition requiring the reservation to be set.
##########
test/src/main/java/org/apache/accumulo/test/fate/MultipleStoresIT.java:
##########
@@ -0,0 +1,571 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.fate;
+
+import static org.apache.accumulo.test.fate.user.UserFateStoreIT.createFateTable;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.fate.Fate;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.FateStore;
+import org.apache.accumulo.core.fate.MetaFateStore;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore;
+import org.apache.accumulo.core.fate.Repo;
+import org.apache.accumulo.core.fate.user.UserFateStore;
+import org.apache.accumulo.core.fate.zookeeper.ZooCache;
+import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.test.util.Wait;
+import org.apache.accumulo.test.zookeeper.ZooKeeperTestingServer;
+import org.easymock.EasyMock;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+// TODO 4131 could potentially have separate classes for testing MetaFateStore and UserFateStore
+// similar to how FateTestRunner is used, however that interface doesn't work as nicely here
+// since we are using multiple stores instead of just one. Can do something similar to
+// FateTestRunner here if desired
+public class MultipleStoresIT extends SharedMiniClusterBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MultipleStoresIT.class);
+ @TempDir
+ private static File tempDir;
+ private static ZooKeeperTestingServer szk = null;
+ private static ZooReaderWriter zk;
+ private static final String FATE_DIR = "/fate";
+ private ClientContext client;
+
+ @BeforeEach
+ public void beforeEachSetup() {
+ client = (ClientContext) Accumulo.newClient().from(getClientProps()).build();
+ }
+
+ @AfterEach
+ public void afterEachTeardown() {
+ client.close();
+ }
+
+ @BeforeAll
+ public static void beforeAllSetup() throws Exception {
+ SharedMiniClusterBase.startMiniCluster();
+ szk = new ZooKeeperTestingServer(tempDir);
+ zk = szk.getZooReaderWriter();
+ }
+
+ @AfterAll
+ public static void afterAllTeardown() throws Exception {
+ SharedMiniClusterBase.stopMiniCluster();
+ szk.close();
+ }
+
+ @Test
+ public void testReserveUnreserve() throws Exception {
+ testReserveUnreserve(FateInstanceType.META);
+ testReserveUnreserve(FateInstanceType.USER);
+ }
+
+ private void testReserveUnreserve(FateInstanceType storeType) throws Exception {
+ // reserving/unreserving a FateId should be reflected across instances of the stores
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> activeReservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1);
+ store2 = new UserFateStore<>(client, tableName, lock2);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock1);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock2);
+ }
+
+ // Create the fate ids using store1
+ for (int i = 0; i < numFateIds; i++) {
+ assertTrue(allIds.add(store1.create()));
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Reserve half the fate ids using store1 and rest using store2, after reserving a fate id in
+ // one, should not be able to reserve the same in the other. Should also not matter that all the
+ // ids were created using store1
+ int count = 0;
+ for (FateId fateId : allIds) {
+ if (count % 2 == 0) {
+ reservations.add(store1.reserve(fateId));
+ assertTrue(store2.tryReserve(fateId).isEmpty());
+ } else {
+ reservations.add(store2.reserve(fateId));
+ assertTrue(store1.tryReserve(fateId).isEmpty());
+ }
+ count++;
+ }
+ // Both stores should return the same reserved transactions
+ activeReservations = store1.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+ activeReservations = store2.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+
+ // Test setting/getting the TStatus and unreserving the transactions
+ for (int i = 0; i < allIds.size(); i++) {
+ var reservation = reservations.get(i);
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, reservation.getStatus());
+ reservation.setStatus(ReadOnlyFateStore.TStatus.SUBMITTED);
+ assertEquals(ReadOnlyFateStore.TStatus.SUBMITTED, reservation.getStatus());
+ reservation.delete();
+ reservation.unreserve(Duration.ofMillis(0));
+ // Attempt to set a status on a tx that has been unreserved (should throw exception)
+ assertThrows(IllegalStateException.class,
+ () -> reservation.setStatus(ReadOnlyFateStore.TStatus.NEW));
+ }
+ assertTrue(store1.getActiveReservations().isEmpty());
+ assertTrue(store2.getActiveReservations().isEmpty());
+ }
+
+ @Test
+ public void testReserveNonExistentTxn() throws Exception {
+ testReserveNonExistentTxn(FateInstanceType.META);
+ testReserveNonExistentTxn(FateInstanceType.USER);
+ }
+
+ private void testReserveNonExistentTxn(FateInstanceType storeType) throws Exception {
+ // Tests that reserve() doesn't hang indefinitely and instead throws an error
+ // on reserve() a non-existent transaction.
+ final FateStore<SleepingTestEnv> store;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final String tableName = getUniqueNames(1)[0];
+ final FateId fakeFateId = FateId.from(storeType, UUID.randomUUID());
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ assertThrows(IllegalStateException.class, () -> store.reserve(fakeFateId));
+ }
+
+ @Test
+ public void testReserveReservedAndUnreserveUnreserved() throws Exception {
+ testReserveReservedAndUnreserveUnreserved(FateInstanceType.META);
+ testReserveReservedAndUnreserveUnreserved(FateInstanceType.USER);
+ }
+
+ private void testReserveReservedAndUnreserveUnreserved(FateInstanceType storeType)
+ throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final FateStore<SleepingTestEnv> store;
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ // Create some FateIds and ensure that they can be reserved
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId = store.create();
+ assertTrue(allIds.add(fateId));
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservations.add(reservation.orElseThrow());
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Try to reserve again, should not reserve
+ for (FateId fateId : allIds) {
+ assertTrue(store.tryReserve(fateId).isEmpty());
+ }
+
+ // Unreserve all the FateIds
+ for (var reservation : reservations) {
+ reservation.unreserve(Duration.ofMillis(0));
+ }
+ // Try to unreserve again (should throw exception)
+ for (var reservation : reservations) {
+ assertThrows(IllegalStateException.class, () -> reservation.unreserve(Duration.ofMillis(0)));
+ }
+ }
+
+ @Test
+ public void testReserveAfterUnreserveAndReserveAfterDeleted() throws Exception {
+ testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType.META);
+ testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType.USER);
+ }
+
+ private void testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType storeType)
+ throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final FateStore<SleepingTestEnv> store;
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ // Create some FateIds and ensure that they can be reserved
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId = store.create();
+ assertTrue(allIds.add(fateId));
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservations.add(reservation.orElseThrow());
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Unreserve all
+ for (var reservation : reservations) {
+ reservation.unreserve(Duration.ofMillis(0));
+ }
+
+ // Ensure they can be reserved again, and delete and unreserve this time
+ for (FateId fateId : allIds) {
+ // Verify that the tx status is still NEW after unreserving since it hasn't been deleted
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, store.read(fateId).getStatus());
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservation.orElseThrow().delete();
+ reservation.orElseThrow().unreserve(Duration.ofMillis(0));
+ }
+
+ for (FateId fateId : allIds) {
+ // Verify that the tx is now unknown since it has been deleted
+ assertEquals(ReadOnlyFateStore.TStatus.UNKNOWN, store.read(fateId).getStatus());
+ // Attempt to reserve a deleted txn, should throw an exception and not wait indefinitely
+ assertThrows(IllegalStateException.class, () -> store.reserve(fateId));
+ }
+ }
+
+ @Test
+ public void testMultipleFateInstances() throws Exception {
+ testMultipleFateInstances(FateInstanceType.META);
+ testMultipleFateInstances(FateInstanceType.USER);
+ }
+
+ private void testMultipleFateInstances(FateInstanceType storeType) throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final SleepingTestEnv testEnv1 = new SleepingTestEnv(50);
+ final SleepingTestEnv testEnv2 = new SleepingTestEnv(50);
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1);
+ store2 = new UserFateStore<>(client, tableName, lock2);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock1);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock2);
+ }
+
+ Fate<SleepingTestEnv> fate1 =
+ new Fate<>(testEnv1, store1, Object::toString, DefaultConfiguration.getInstance());
+ fate1.startDeadReservationCleaner();
+ Fate<SleepingTestEnv> fate2 =
+ new Fate<>(testEnv2, store2, Object::toString, DefaultConfiguration.getInstance());
+ fate2.startDeadReservationCleaner();
+
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId;
+ // Start half the txns using fate1, and the other half using fate2
+ if (i % 2 == 0) {
+ fateId = fate1.startTransaction();
+ fate1.seedTransaction("op" + i, fateId, new SleepingTestRepo(), true, "test");
+ } else {
+ fateId = fate2.startTransaction();
+ fate2.seedTransaction("op" + i, fateId, new SleepingTestRepo(), true, "test");
+ }
+ allIds.add(fateId);
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Should be able to wait for completion on any fate instance
+ for (FateId fateId : allIds) {
+ fate2.waitForCompletion(fateId);
+ }
+ // Ensure that all txns have been executed and have only been executed once
+ assertTrue(Collections.disjoint(testEnv1.executedOps, testEnv2.executedOps));
+ assertEquals(allIds, Sets.union(testEnv1.executedOps, testEnv2.executedOps));
+
+ fate1.shutdown(1, TimeUnit.MINUTES);
+ fate2.shutdown(1, TimeUnit.MINUTES);
+ }
+
+ @Test
+ public void testDeadReservationsCleanup() throws Exception {
+ testDeadReservationsCleanup(FateInstanceType.META);
+ testDeadReservationsCleanup(FateInstanceType.USER);
+ }
+
+ private void testDeadReservationsCleanup(FateInstanceType storeType) throws Exception {
+ // Tests reserving some transactions, then simulating that the Manager died by creating
+ // a new Fate instance and store with a new LockID. The transactions which were
+ // reserved using the old LockID should be cleaned up by Fate's DeadReservationCleaner,
+ // then picked up by the new Fate/store.
+
+ final String tableName = getUniqueNames(1)[0];
+ // One transaction for each FATE worker thread
+ final int numFateIds =
+ Integer.parseInt(Property.MANAGER_FATE_THREADPOOL_SIZE.getDefaultValue());
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<LatchTestEnv> mockedStore1, store2;
+ final LatchTestEnv testEnv1 = new LatchTestEnv();
+ final LatchTestEnv testEnv2 = new LatchTestEnv();
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> reservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ mockedStore1 = EasyMock.createMockBuilder(UserFateStore.class)
+ .withConstructor(ClientContext.class, String.class, ZooUtil.LockID.class)
+ .withArgs(client, tableName, lock1).addMockedMethod("isDeadReservation").createMock();
Review Comment:
Wonder if instead of mocking that this method could create real locks in ZK since its running in an IT. This would make the test be a bit more end to end. Looked a bit at ServiceLockIT trying to see what was needed to create a lock and does not look like a few lines of code. Maybe some utility code could be shared with that class, but not sure. If its even workable, could be a follow on improvement. Not too sure about this suggestion, just posting for discussion.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1631616426
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -315,6 +420,30 @@ public List<ReadOnlyRepo<T>> getStack() {
return dops;
}
}
+
+ @Override
+ protected void unreserve() {
+ try {
+ if (!this.deleted) {
+ zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ if ((currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(this.reservation))) {
+ // Remove the FateReservation from the NodeValue to unreserve
+ return new NodeValue(currNodeVal.status, null, currFateKey).serialize();
+ } else {
+ // possible this is running a 2nd time in zk server fault conditions and its first
Review Comment:
Could add some logging here. If the reservation in ZK is not set, then could log something at trace. If there is a reservation in ZK that differs, it could log something at debug. These logs may be useful later for debugging problems when initially using the elasticity changes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2266058704
I am finally looking through the test changes right now. Everything else looks good.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1702286669
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -57,8 +57,8 @@
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
// TODO 4131 should probably add support to AbstractFateStore, MetaFateStore,
-// and UserFateStore to accept null lockID and zooCache (maybe make these fields
-// Optional<>). This could replace the current createDummyLockID(). This support
+// and UserFateStore to accept null lockID (maybe make this field Optional<>).
Review Comment:
Yes, I have all the TODOs marked `TODO 4131`. I was planning on creating new issues and deleting all the `TODO 4131` when this PR is ready to be merged.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1704410617
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
+ .orElse(null);
+ if (persistedRes != null && persistedRes.equals(reservation)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ }
Review Comment:
Marking resolved as this is more a style choice and doesn't affect functionality
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1704409055
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -360,7 +380,27 @@ public Fate(T environment, FateStore<T> store, Function<Repo<T>,String> toLogStr
}
}
}, 3, SECONDS));
- this.executor = pool;
+ this.transactionExecutor = pool;
+
+ // Create a dead reservation cleaner for this store that will periodically (every 30 seconds)
+ // clean up reservations held by dead processes, if they exist. Only created if a dead
+ // reservation cleaner is not already running for the given store type.
+ // TODO 4131 periodic cleanup runs every 30 seconds
+ // Should this be longer? Shorter? A configurable Property? A function of something?
+ ScheduledExecutorService deadResCleanerExecutor = ThreadPools.getServerThreadPools()
+ .createScheduledExecutorService(1, store.type() + "-dead-reservation-cleaner-pool");
+ if ((store.type() == FateInstanceType.USER && !userDeadReservationCleanerRunning)
+ || (store.type() == FateInstanceType.META && !metaDeadReservationCleanerRunning)) {
+ ScheduledFuture<?> deadReservationCleaner = deadResCleanerExecutor
+ .scheduleWithFixedDelay(new DeadReservationCleaner(), 3, 30, SECONDS);
+ ThreadPools.watchCriticalScheduledTask(deadReservationCleaner);
+ if (store.type() == FateInstanceType.USER) {
+ userDeadReservationCleanerRunning = true;
+ } else if (store.type() == FateInstanceType.META) {
+ metaDeadReservationCleanerRunning = true;
+ }
+ }
Review Comment:
Resolved as no longer applicable: no longer check the store type
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "DomGarguilo (via GitHub)" <gi...@apache.org>.
DomGarguilo commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1707419235
##########
core/src/main/java/org/apache/accumulo/core/fate/FateStore.java:
##########
@@ -107,11 +118,155 @@ interface FateTxStore<T> extends ReadOnlyFateTxStore<T> {
void unreserve(Duration deferTime);
}
+ /**
+ * The value stored to indicate a FATE transaction ID ({@link FateId}) has been reserved
+ */
+ class FateReservation {
+
+ // The LockID (provided by the Manager running the FATE which uses this store) which is used for
+ // identifying dead Managers, so their reservations can be deleted and picked up again since
+ // they can no longer be worked on.
+ private final ZooUtil.LockID lockID; // TODO 4131 not sure if this is the best type for this
+ // The UUID generated on a reservation attempt (tryReserve()) used to uniquely identify that
+ // attempt. This is useful for the edge case where the reservation is sent to the server
+ // (Tablet Server for UserFateStore and the ZooKeeper Server for MetaFateStore), but the server
+ // dies before the store receives the response. It allows us to determine if the reservation
+ // was successful and was written by this reservation attempt (could have been successfully
+ // reserved by another attempt or not reserved at all, in which case, we wouldn't want to
+ // expose a FateTxStore).
+ private final UUID reservationUUID;
+ private final byte[] serialized;
+ private static final Pattern UUID_PATTERN =
+ Pattern.compile("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$");
+ private static final Pattern LOCKID_PATTERN = Pattern.compile("^.+/.+\\$[0-9a-fA-F]+$");
+
+ private FateReservation(ZooUtil.LockID lockID, UUID reservationUUID) {
+ this.lockID = Objects.requireNonNull(lockID);
+ this.reservationUUID = Objects.requireNonNull(reservationUUID);
+ this.serialized = serialize(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(ZooUtil.LockID lockID, UUID reservationUUID) {
+ return new FateReservation(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(byte[] serialized) {
+ try (DataInputBuffer buffer = new DataInputBuffer()) {
+ buffer.reset(serialized, serialized.length);
+ ZooUtil.LockID lockID = new ZooUtil.LockID("", buffer.readUTF());
+ UUID reservationUUID = UUID.fromString(buffer.readUTF());
+ return new FateReservation(lockID, reservationUUID);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static FateReservation from(String fateReservationStr) {
+ if (isFateReservation(fateReservationStr)) {
+ String[] fields = fateReservationStr.split(":");
+ ZooUtil.LockID lockId = new ZooUtil.LockID("", fields[0]);
+ UUID reservationUUID = UUID.fromString(fields[1]);
+ return new FateReservation(lockId, reservationUUID);
+ } else {
+ throw new IllegalArgumentException(
+ "Tried to create a FateReservation from an invalid string: " + fateReservationStr);
+ }
+ }
+
+ /**
+ *
+ * @param fateReservationStr the string from a call to {@link FateReservation#toString()}
+ * @return true if the string represents a valid FateReservation object, false otherwise
+ */
+ public static boolean isFateReservation(String fateReservationStr) {
+ if (fateReservationStr != null) {
+ String[] fields = fateReservationStr.split(":");
+ if (fields.length == 2) {
+ return LOCKID_PATTERN.matcher(fields[0]).matches()
+ && UUID_PATTERN.matcher(fields[1]).matches();
+ }
+ }
+ return false;
+ }
+
+ public ZooUtil.LockID getLockID() {
+ return lockID;
+ }
+
+ public UUID getReservationUUID() {
+ return reservationUUID;
+ }
+
+ public byte[] getSerialized() {
+ return serialized;
+ }
+
+ private static byte[] serialize(ZooUtil.LockID lockID, UUID reservationUUID) {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos)) {
+ dos.writeUTF(lockID.serialize("/"));
+ dos.writeUTF(reservationUUID.toString());
+ dos.close();
Review Comment:
Since the code functions correctly and its not a big deal. I'll just resolve this without any further changes needing to be made
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1670934639
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -298,7 +299,13 @@ private Optional<FateId> create(FateKey fateKey) {
@Override
public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
- FateId fateId = fateIdGenerator.fromTypeAndKey(getInstanceType(), fateKey);
+ // TODO 4131 not confident about this new implementation of createAndReserve.
Review Comment:
The way this method previously worked and the way the code was structured it was driven by the lock object being in AbstractFateStore. Now that there is no longer a lock we want to implement the following algorithm for both stores using conditional updates (@cshannon you added the fatekey feature, does the following algorithm seem ok to you?).
1. var fateId = FateId fateId = fateIdGenerator.fromTypeAndKey(type(), fateKey);
2. in the store conditionally create fateId setting the status, reservation and fateKey atomically at creation time. The condition should check that nothing currently exist for the fateId, its expected to be absent.
3. If the conditional update fails for some reason, read the fateId and see if it already has the expected status, fateKey and reservation. If it does, then probably running for a second time and already reserved. If it does not then need to take the following actions.
1. If the fateId exists and has the same fateKey, but different status (should have NEW status) or reservation than expected then return Optional.empty(). This is the case where another thread has already created and reserved using the fateKey so there is nothing to do.
2. If the fateId exists and has different fateKey or no fateKey then this represents an unexpected collision (and this unexpected with the 128 bit keys) so throw an exception in this case.
5. If the conditional update succeed , then we can return a non empty Optional w/ a FateTxStore obj.
Now that the lock object is no longer needed in AbstractFateStore it may make sense to completely push the full implementation of the `createAndReserve` method into MetaFateStore and UserFateStore and have nothing in AbstractFateStore for the method. The reason I am thinking this may be better is because the algorithm above is based on conditional updates now and how ZK and accumulo do conditional updates is very different. However there may still be opportunities to share common code, so not sure of the best way to structure the code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1671308191
##########
core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java:
##########
@@ -175,6 +176,21 @@ public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
}
return txStore;
}
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return store.isReserved(fateId);
+ }
+
+ @Override
+ public Map<FateId,FateReservation> getActiveReservations() {
+ return store.getActiveReservations();
+ }
+
+ @Override
+ public void deleteDeadReservations() {
+ store.deleteDeadReservations();
Review Comment:
Not sure if this is workable, maybe the method could take a consumer that is passes found dead reservations to like : `deleteDeadReservations(Consumer<FateReservation> foundDead)` then this could add logging for those. Alternatively could just put the same log stmts in each impl of the store.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1672930975
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
You're referring to MetaFateStore where I have that pattern correct? I don't think UserFateStore would have this issue but I could be wrong
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "keith-turner (via GitHub)" <gi...@apache.org>.
keith-turner commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1674580318
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +157,95 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ try (Scanner scanner = context.createScanner(tableName, Authorizations.EMPTY)) {
+ scanner.setRange(getRow(fateId));
+ scanner.fetchColumn(TxColumnFamily.RESERVATION_COLUMN.getColumnFamily(),
+ TxColumnFamily.RESERVATION_COLUMN.getColumnQualifier());
+ FateReservation persistedRes = scanner.stream()
+ .filter(entry -> FateReservation.isFateReservation(entry.getValue().toString()))
+ .map(entry -> FateReservation.from(entry.getValue().toString())).findFirst()
Review Comment:
I had made a comment elsewhere about avoiding using `toString()` for serialization and suggested using `getSerialized()` instead When I made that comment I thought the impl of `toString()` and `getSerialized()` were the same. Now I realize they are different. Maybe part of the solution to this is converging UserFateStore and MetaFateStore on the same serialization impl and making the de-serialization as strict as possible. Would that work? If NOT_RESERVED is dropped seems like that makes it easier for them to have same ser/deser impl.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688551699
##########
core/src/main/java/org/apache/accumulo/core/fate/FateStore.java:
##########
@@ -107,11 +118,155 @@ interface FateTxStore<T> extends ReadOnlyFateTxStore<T> {
void unreserve(Duration deferTime);
}
+ /**
+ * The value stored to indicate a FATE transaction ID ({@link FateId}) has been reserved
+ */
+ class FateReservation {
+
+ // The LockID (provided by the Manager running the FATE which uses this store) which is used for
+ // identifying dead Managers, so their reservations can be deleted and picked up again since
+ // they can no longer be worked on.
+ private final ZooUtil.LockID lockID; // TODO 4131 not sure if this is the best type for this
+ // The UUID generated on a reservation attempt (tryReserve()) used to uniquely identify that
+ // attempt. This is useful for the edge case where the reservation is sent to the server
+ // (Tablet Server for UserFateStore and the ZooKeeper Server for MetaFateStore), but the server
+ // dies before the store receives the response. It allows us to determine if the reservation
+ // was successful and was written by this reservation attempt (could have been successfully
+ // reserved by another attempt or not reserved at all, in which case, we wouldn't want to
+ // expose a FateTxStore).
+ private final UUID reservationUUID;
+ private final byte[] serialized;
+ private static final Pattern UUID_PATTERN =
+ Pattern.compile("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$");
+ private static final Pattern LOCKID_PATTERN = Pattern.compile("^.+/.+\\$[0-9a-fA-F]+$");
+
+ private FateReservation(ZooUtil.LockID lockID, UUID reservationUUID) {
+ this.lockID = Objects.requireNonNull(lockID);
+ this.reservationUUID = Objects.requireNonNull(reservationUUID);
+ this.serialized = serialize(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(ZooUtil.LockID lockID, UUID reservationUUID) {
+ return new FateReservation(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(byte[] serialized) {
+ try (DataInputBuffer buffer = new DataInputBuffer()) {
+ buffer.reset(serialized, serialized.length);
+ ZooUtil.LockID lockID = new ZooUtil.LockID("", buffer.readUTF());
+ UUID reservationUUID = UUID.fromString(buffer.readUTF());
+ return new FateReservation(lockID, reservationUUID);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static FateReservation from(String fateReservationStr) {
+ if (isFateReservation(fateReservationStr)) {
+ String[] fields = fateReservationStr.split(":");
+ ZooUtil.LockID lockId = new ZooUtil.LockID("", fields[0]);
+ UUID reservationUUID = UUID.fromString(fields[1]);
+ return new FateReservation(lockId, reservationUUID);
+ } else {
+ throw new IllegalArgumentException(
+ "Tried to create a FateReservation from an invalid string: " + fateReservationStr);
+ }
+ }
+
+ /**
+ *
+ * @param fateReservationStr the string from a call to {@link FateReservation#toString()}
+ * @return true if the string represents a valid FateReservation object, false otherwise
+ */
+ public static boolean isFateReservation(String fateReservationStr) {
+ if (fateReservationStr != null) {
+ String[] fields = fateReservationStr.split(":");
+ if (fields.length == 2) {
+ return LOCKID_PATTERN.matcher(fields[0]).matches()
+ && UUID_PATTERN.matcher(fields[1]).matches();
Review Comment:
No longer applicablevvvvvvvvv
##########
core/src/main/java/org/apache/accumulo/core/fate/FateStore.java:
##########
@@ -107,11 +118,155 @@ interface FateTxStore<T> extends ReadOnlyFateTxStore<T> {
void unreserve(Duration deferTime);
}
+ /**
+ * The value stored to indicate a FATE transaction ID ({@link FateId}) has been reserved
+ */
+ class FateReservation {
+
+ // The LockID (provided by the Manager running the FATE which uses this store) which is used for
+ // identifying dead Managers, so their reservations can be deleted and picked up again since
+ // they can no longer be worked on.
+ private final ZooUtil.LockID lockID; // TODO 4131 not sure if this is the best type for this
+ // The UUID generated on a reservation attempt (tryReserve()) used to uniquely identify that
+ // attempt. This is useful for the edge case where the reservation is sent to the server
+ // (Tablet Server for UserFateStore and the ZooKeeper Server for MetaFateStore), but the server
+ // dies before the store receives the response. It allows us to determine if the reservation
+ // was successful and was written by this reservation attempt (could have been successfully
+ // reserved by another attempt or not reserved at all, in which case, we wouldn't want to
+ // expose a FateTxStore).
+ private final UUID reservationUUID;
+ private final byte[] serialized;
+ private static final Pattern UUID_PATTERN =
+ Pattern.compile("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$");
+ private static final Pattern LOCKID_PATTERN = Pattern.compile("^.+/.+\\$[0-9a-fA-F]+$");
+
+ private FateReservation(ZooUtil.LockID lockID, UUID reservationUUID) {
+ this.lockID = Objects.requireNonNull(lockID);
+ this.reservationUUID = Objects.requireNonNull(reservationUUID);
+ this.serialized = serialize(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(ZooUtil.LockID lockID, UUID reservationUUID) {
+ return new FateReservation(lockID, reservationUUID);
+ }
+
+ public static FateReservation from(byte[] serialized) {
+ try (DataInputBuffer buffer = new DataInputBuffer()) {
+ buffer.reset(serialized, serialized.length);
+ ZooUtil.LockID lockID = new ZooUtil.LockID("", buffer.readUTF());
+ UUID reservationUUID = UUID.fromString(buffer.readUTF());
+ return new FateReservation(lockID, reservationUUID);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ public static FateReservation from(String fateReservationStr) {
+ if (isFateReservation(fateReservationStr)) {
+ String[] fields = fateReservationStr.split(":");
+ ZooUtil.LockID lockId = new ZooUtil.LockID("", fields[0]);
+ UUID reservationUUID = UUID.fromString(fields[1]);
+ return new FateReservation(lockId, reservationUUID);
+ } else {
+ throw new IllegalArgumentException(
+ "Tried to create a FateReservation from an invalid string: " + fateReservationStr);
+ }
+ }
+
+ /**
+ *
+ * @param fateReservationStr the string from a call to {@link FateReservation#toString()}
+ * @return true if the string represents a valid FateReservation object, false otherwise
+ */
+ public static boolean isFateReservation(String fateReservationStr) {
+ if (fateReservationStr != null) {
+ String[] fields = fateReservationStr.split(":");
+ if (fields.length == 2) {
+ return LOCKID_PATTERN.matcher(fields[0]).matches()
+ && UUID_PATTERN.matcher(fields[1]).matches();
Review Comment:
No longer applicable
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688550936
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -298,7 +299,13 @@ private Optional<FateId> create(FateKey fateKey) {
@Override
public Optional<FateTxStore<T>> createAndReserve(FateKey fateKey) {
- FateId fateId = fateIdGenerator.fromTypeAndKey(getInstanceType(), fateKey);
+ // TODO 4131 not confident about this new implementation of createAndReserve.
Review Comment:
Implemented in User/MetaFateStore.createAndReserve(). Thank you for this detailed description.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1688553038
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -55,7 +60,7 @@
//TODO use zoocache? - ACCUMULO-1297
//TODO handle zookeeper being down gracefully - ACCUMULO-1297
-
+// TODO 4131 noticed this class is not in the fate.zookeeper package. Should it be?
Review Comment:
Yeah that's what I thought. This can be done in a follow on issue/PR. This can be kept unresolved until then
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#issuecomment-2248733902
I have found and addressed the bug in my most recent commit. This PR is now ready for full review.
The tests verified to still pass: the sunny day tests, all fate tests (ZooMutatorIT, FateMutatorImplIT, UserFateStoreIT, FateInterleavingIT, FateIT, FateOpsCommandsIT, ComprehensiveFlakyFateIT, DeleteRowsFlakyFateIT, ManagerRepoIT, and MultipleStoresIT (new test added)) except FateStoreIT. FateStoreIT reflects a method that has been removed in this PR, and want to receive feedback about the removal of this method before changes are made to this test. Have a "TODO 4131" marking this so this is not forgotten.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fate reservations moved out of memory [accumulo]
Posted by "kevinrr888 (via GitHub)" <gi...@apache.org>.
kevinrr888 commented on code in PR #4524:
URL: https://github.com/apache/accumulo/pull/4524#discussion_r1662598974
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -106,13 +112,102 @@ public FateId create() {
@Override
protected void create(FateId fateId, FateKey key) {
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, key).serialize(),
+ zk.putPersistentData(getTXPath(fateId), new NodeValue(TStatus.NEW, null, key).serialize(),
NodeExistsPolicy.FAIL);
} catch (KeeperException | InterruptedException e) {
throw new IllegalStateException(e);
}
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // uniquely identify this attempt to reserve the fate operation data
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ try {
+ byte[] newSerNodeVal = zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ // The uuid handles the case where there was a ZK server fault and the write for this thread
+ // went through but that was not acknowledged, and we are reading our own write for 2nd
+ // time.
+ if (!currNodeVal.isReserved() || (currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(reservation))) {
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ // Add the FateReservation to the node to reserve
+ return new NodeValue(currNodeVal.status, reservation, currFateKey).serialize();
+ } else {
+ // This will not change the value to null but will return null
+ return null;
+ }
+ });
+ if (newSerNodeVal != null) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else {
+ return Optional.empty();
+ }
+ } catch (InterruptedException | KeeperException | AcceptableThriftTableOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ boolean isReserved;
+ try {
+ isReserved = getNode(fateId).isReserved();
+ } catch (Exception e) {
+ // Exception thrown, so node doesn't exist, so it is not reserved
+ isReserved = false;
+ }
Review Comment:
Looking at getNode(), the NNE is handled in that method, so I simplified isReserved()
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -224,9 +323,14 @@ public void setStatus(TStatus status) {
verifyReserved(true);
try {
- zk.putPersistentData(getTXPath(fateId), new NodeValue(status).serialize(),
- NodeExistsPolicy.OVERWRITE);
- } catch (KeeperException | InterruptedException e) {
+ zk.mutateExisting(getTXPath(fateId), currSerializedData -> {
+ NodeValue currNodeVal = new NodeValue(currSerializedData);
Review Comment:
Fixed
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -315,6 +420,30 @@ public List<ReadOnlyRepo<T>> getStack() {
return dops;
}
}
+
+ @Override
+ protected void unreserve() {
+ try {
+ if (!this.deleted) {
+ zk.mutateExisting(getTXPath(fateId), currSerNodeVal -> {
+ NodeValue currNodeVal = new NodeValue(currSerNodeVal);
+ FateKey currFateKey = currNodeVal.fateKey.orElse(null);
+ if ((currNodeVal.isReserved()
+ && currNodeVal.reservation.orElseThrow().equals(this.reservation))) {
+ // Remove the FateReservation from the NodeValue to unreserve
+ return new NodeValue(currNodeVal.status, null, currFateKey).serialize();
+ } else {
+ // possible this is running a 2nd time in zk server fault conditions and its first
Review Comment:
Added
##########
core/src/main/java/org/apache/accumulo/core/fate/MetaFateStore.java:
##########
@@ -421,14 +548,30 @@ private Optional<FateKey> deserializeFateKey(DataInputBuffer buffer) throws IOEx
return Optional.empty();
}
+ private Optional<FateReservation> deserializeFateReservation(DataInputBuffer buffer)
+ throws IOException {
+ int length = buffer.readInt();
+ if (length > 0) {
+ return Optional.of(FateReservation.deserialize(buffer.readNBytes(length)));
+ }
+ return Optional.empty();
+ }
+
byte[] serialize() {
try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos)) {
dos.writeUTF(status.name());
+ if (isReserved()) {
Review Comment:
That sounds like a good idea
##########
core/src/main/java/org/apache/accumulo/core/fate/AbstractFateStore.java:
##########
@@ -502,4 +520,14 @@ protected Serializable deserializeTxInfo(TxInfo txInfo, byte[] data) {
throw new IllegalStateException("Bad node data " + txInfo);
}
}
+
+ /**
+ * TODO 4131 this is a temporary method used to create a dummy lock when using a FateStore outside
Review Comment:
Yes, that sounds like a great change. And yes you are correct about some of the fate admin utils requiring the Manager to be down, so that would be helpful for those as well.
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +154,76 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ status = newMutator(fateId).requireReserved(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ }
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public boolean isReserved(FateId fateId) {
+ return newMutator(fateId).requireReserved().tryMutate().equals(FateMutator.Status.ACCEPTED);
Review Comment:
Understood, fixed
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -272,8 +340,12 @@ public FateInstanceType type() {
private class FateTxStoreImpl extends AbstractFateTxStoreImpl<T> {
- private FateTxStoreImpl(FateId fateId, boolean isReserved) {
- super(fateId, isReserved);
+ private FateTxStoreImpl(FateId fateId) {
+ super(fateId);
+ }
+
+ private FateTxStoreImpl(FateId fateId, FateReservation reservation) {
Review Comment:
That sounds like a good addition
##########
core/src/main/java/org/apache/accumulo/core/fate/user/UserFateStore.java:
##########
@@ -151,6 +154,76 @@ protected void create(FateId fateId, FateKey fateKey) {
+ " and fateKey " + fateKey + " after " + maxAttempts + " attempts");
}
+ @Override
+ public Optional<FateTxStore<T>> tryReserve(FateId fateId) {
+ // Create a unique FateReservation for this reservation attempt
+ FateReservation reservation = FateReservation.from(lockID, UUID.randomUUID());
+
+ FateMutator.Status status = newMutator(fateId).putReservedTx(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ } else if (status.equals(FateMutator.Status.UNKNOWN)) {
+ // If the status is UNKNOWN, this means an error occurred after the mutation was
+ // sent to the TabletServer, and it is unknown if the mutation was written. We
+ // need to check if the mutation was written and if it was written by this
+ // attempt at reservation. If it was written by this reservation attempt,
+ // we can return the FateTxStore since it was successfully reserved in this
+ // attempt, otherwise we return empty (was written by another reservation
+ // attempt or was not written at all).
+ status = newMutator(fateId).requireReserved(reservation).tryMutate();
+ if (status.equals(FateMutator.Status.ACCEPTED)) {
+ return Optional.of(new FateTxStoreImpl(fateId, reservation));
+ }
Review Comment:
Yeah, I wasn't sure about this mutation. This is a much better way to do this; fixed.
##########
core/src/main/java/org/apache/accumulo/core/fate/Fate.java:
##########
@@ -313,6 +317,26 @@ private void undo(FateId fateId, Repo<T> op) {
}
+ /**
+ * A thread that finds reservations held by dead processes and unreserves them. Only one thread
+ * runs per store type across all Fate instances (one to clean up dead reservations for
+ * {@link org.apache.accumulo.core.fate.user.UserFateStore UserFateStore} and one to clean up dead
+ * reservations for {@link MetaFateStore}).
+ */
+ private class DeadReservationCleaner implements Runnable {
+ // TODO 4131 periodic check runs every 30 seconds
+ // Should this be longer? Shorter? A configurable Property? A function of something?
+ private static final long INTERVAL_MILLIS = 30_000;
+
+ @Override
+ public void run() {
+ while (keepRunning.get()) {
+ store.deleteDeadReservations();
Review Comment:
After the DeadReservationCleaner dies, the Manager continues to run normally. I changed it to be a critical thread. When looking at this, I noticed that none of the other Fate threads are critical. It seems that maybe they should be? I can create a follow on issue if so. If the WorkFinder or any of the actual workers (TransactionRunner) die, Manager will continue to run normally.
##########
test/src/main/java/org/apache/accumulo/test/fate/MultipleStoresIT.java:
##########
@@ -0,0 +1,571 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.fate;
+
+import static org.apache.accumulo.test.fate.user.UserFateStoreIT.createFateTable;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.fate.Fate;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.FateStore;
+import org.apache.accumulo.core.fate.MetaFateStore;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore;
+import org.apache.accumulo.core.fate.Repo;
+import org.apache.accumulo.core.fate.user.UserFateStore;
+import org.apache.accumulo.core.fate.zookeeper.ZooCache;
+import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.test.util.Wait;
+import org.apache.accumulo.test.zookeeper.ZooKeeperTestingServer;
+import org.easymock.EasyMock;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+// TODO 4131 could potentially have separate classes for testing MetaFateStore and UserFateStore
+// similar to how FateTestRunner is used, however that interface doesn't work as nicely here
+// since we are using multiple stores instead of just one. Can do something similar to
+// FateTestRunner here if desired
+public class MultipleStoresIT extends SharedMiniClusterBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MultipleStoresIT.class);
+ @TempDir
+ private static File tempDir;
+ private static ZooKeeperTestingServer szk = null;
+ private static ZooReaderWriter zk;
+ private static final String FATE_DIR = "/fate";
+ private ClientContext client;
+
+ @BeforeEach
+ public void beforeEachSetup() {
+ client = (ClientContext) Accumulo.newClient().from(getClientProps()).build();
+ }
+
+ @AfterEach
+ public void afterEachTeardown() {
+ client.close();
+ }
+
+ @BeforeAll
+ public static void beforeAllSetup() throws Exception {
+ SharedMiniClusterBase.startMiniCluster();
+ szk = new ZooKeeperTestingServer(tempDir);
+ zk = szk.getZooReaderWriter();
+ }
+
+ @AfterAll
+ public static void afterAllTeardown() throws Exception {
+ SharedMiniClusterBase.stopMiniCluster();
+ szk.close();
+ }
+
+ @Test
+ public void testReserveUnreserve() throws Exception {
+ testReserveUnreserve(FateInstanceType.META);
+ testReserveUnreserve(FateInstanceType.USER);
+ }
+
+ private void testReserveUnreserve(FateInstanceType storeType) throws Exception {
+ // reserving/unreserving a FateId should be reflected across instances of the stores
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> activeReservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1);
+ store2 = new UserFateStore<>(client, tableName, lock2);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock1);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock2);
+ }
+
+ // Create the fate ids using store1
+ for (int i = 0; i < numFateIds; i++) {
+ assertTrue(allIds.add(store1.create()));
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Reserve half the fate ids using store1 and rest using store2, after reserving a fate id in
+ // one, should not be able to reserve the same in the other. Should also not matter that all the
+ // ids were created using store1
+ int count = 0;
+ for (FateId fateId : allIds) {
+ if (count % 2 == 0) {
+ reservations.add(store1.reserve(fateId));
+ assertTrue(store2.tryReserve(fateId).isEmpty());
+ } else {
+ reservations.add(store2.reserve(fateId));
+ assertTrue(store1.tryReserve(fateId).isEmpty());
+ }
+ count++;
+ }
+ // Both stores should return the same reserved transactions
+ activeReservations = store1.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+ activeReservations = store2.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+
+ // Test setting/getting the TStatus and unreserving the transactions
+ for (int i = 0; i < allIds.size(); i++) {
+ var reservation = reservations.get(i);
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, reservation.getStatus());
+ reservation.setStatus(ReadOnlyFateStore.TStatus.SUBMITTED);
+ assertEquals(ReadOnlyFateStore.TStatus.SUBMITTED, reservation.getStatus());
+ reservation.delete();
+ reservation.unreserve(Duration.ofMillis(0));
+ // Attempt to set a status on a tx that has been unreserved (should throw exception)
+ assertThrows(IllegalStateException.class,
+ () -> reservation.setStatus(ReadOnlyFateStore.TStatus.NEW));
+ }
+ assertTrue(store1.getActiveReservations().isEmpty());
+ assertTrue(store2.getActiveReservations().isEmpty());
+ }
+
+ @Test
+ public void testReserveNonExistentTxn() throws Exception {
+ testReserveNonExistentTxn(FateInstanceType.META);
+ testReserveNonExistentTxn(FateInstanceType.USER);
+ }
+
+ private void testReserveNonExistentTxn(FateInstanceType storeType) throws Exception {
+ // Tests that reserve() doesn't hang indefinitely and instead throws an error
+ // on reserve() a non-existent transaction.
+ final FateStore<SleepingTestEnv> store;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final String tableName = getUniqueNames(1)[0];
+ final FateId fakeFateId = FateId.from(storeType, UUID.randomUUID());
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ assertThrows(IllegalStateException.class, () -> store.reserve(fakeFateId));
+ }
+
+ @Test
+ public void testReserveReservedAndUnreserveUnreserved() throws Exception {
+ testReserveReservedAndUnreserveUnreserved(FateInstanceType.META);
+ testReserveReservedAndUnreserveUnreserved(FateInstanceType.USER);
+ }
+
+ private void testReserveReservedAndUnreserveUnreserved(FateInstanceType storeType)
+ throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final FateStore<SleepingTestEnv> store;
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ // Create some FateIds and ensure that they can be reserved
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId = store.create();
+ assertTrue(allIds.add(fateId));
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservations.add(reservation.orElseThrow());
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Try to reserve again, should not reserve
+ for (FateId fateId : allIds) {
+ assertTrue(store.tryReserve(fateId).isEmpty());
+ }
+
+ // Unreserve all the FateIds
+ for (var reservation : reservations) {
+ reservation.unreserve(Duration.ofMillis(0));
+ }
+ // Try to unreserve again (should throw exception)
+ for (var reservation : reservations) {
+ assertThrows(IllegalStateException.class, () -> reservation.unreserve(Duration.ofMillis(0)));
+ }
+ }
+
+ @Test
+ public void testReserveAfterUnreserveAndReserveAfterDeleted() throws Exception {
+ testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType.META);
+ testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType.USER);
+ }
+
+ private void testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType storeType)
+ throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final FateStore<SleepingTestEnv> store;
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ // Create some FateIds and ensure that they can be reserved
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId = store.create();
+ assertTrue(allIds.add(fateId));
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservations.add(reservation.orElseThrow());
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Unreserve all
+ for (var reservation : reservations) {
+ reservation.unreserve(Duration.ofMillis(0));
+ }
+
+ // Ensure they can be reserved again, and delete and unreserve this time
+ for (FateId fateId : allIds) {
+ // Verify that the tx status is still NEW after unreserving since it hasn't been deleted
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, store.read(fateId).getStatus());
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservation.orElseThrow().delete();
+ reservation.orElseThrow().unreserve(Duration.ofMillis(0));
+ }
+
+ for (FateId fateId : allIds) {
+ // Verify that the tx is now unknown since it has been deleted
+ assertEquals(ReadOnlyFateStore.TStatus.UNKNOWN, store.read(fateId).getStatus());
+ // Attempt to reserve a deleted txn, should throw an exception and not wait indefinitely
+ assertThrows(IllegalStateException.class, () -> store.reserve(fateId));
+ }
+ }
+
+ @Test
+ public void testMultipleFateInstances() throws Exception {
+ testMultipleFateInstances(FateInstanceType.META);
+ testMultipleFateInstances(FateInstanceType.USER);
+ }
+
+ private void testMultipleFateInstances(FateInstanceType storeType) throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final SleepingTestEnv testEnv1 = new SleepingTestEnv(50);
+ final SleepingTestEnv testEnv2 = new SleepingTestEnv(50);
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1);
+ store2 = new UserFateStore<>(client, tableName, lock2);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock1);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock2);
+ }
+
+ Fate<SleepingTestEnv> fate1 =
+ new Fate<>(testEnv1, store1, Object::toString, DefaultConfiguration.getInstance());
+ fate1.startDeadReservationCleaner();
+ Fate<SleepingTestEnv> fate2 =
+ new Fate<>(testEnv2, store2, Object::toString, DefaultConfiguration.getInstance());
+ fate2.startDeadReservationCleaner();
+
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId;
+ // Start half the txns using fate1, and the other half using fate2
+ if (i % 2 == 0) {
+ fateId = fate1.startTransaction();
+ fate1.seedTransaction("op" + i, fateId, new SleepingTestRepo(), true, "test");
+ } else {
+ fateId = fate2.startTransaction();
+ fate2.seedTransaction("op" + i, fateId, new SleepingTestRepo(), true, "test");
+ }
+ allIds.add(fateId);
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Should be able to wait for completion on any fate instance
+ for (FateId fateId : allIds) {
+ fate2.waitForCompletion(fateId);
+ }
+ // Ensure that all txns have been executed and have only been executed once
+ assertTrue(Collections.disjoint(testEnv1.executedOps, testEnv2.executedOps));
+ assertEquals(allIds, Sets.union(testEnv1.executedOps, testEnv2.executedOps));
+
+ fate1.shutdown(1, TimeUnit.MINUTES);
+ fate2.shutdown(1, TimeUnit.MINUTES);
+ }
+
+ @Test
+ public void testDeadReservationsCleanup() throws Exception {
+ testDeadReservationsCleanup(FateInstanceType.META);
+ testDeadReservationsCleanup(FateInstanceType.USER);
+ }
+
+ private void testDeadReservationsCleanup(FateInstanceType storeType) throws Exception {
+ // Tests reserving some transactions, then simulating that the Manager died by creating
+ // a new Fate instance and store with a new LockID. The transactions which were
+ // reserved using the old LockID should be cleaned up by Fate's DeadReservationCleaner,
+ // then picked up by the new Fate/store.
+
+ final String tableName = getUniqueNames(1)[0];
+ // One transaction for each FATE worker thread
+ final int numFateIds =
+ Integer.parseInt(Property.MANAGER_FATE_THREADPOOL_SIZE.getDefaultValue());
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<LatchTestEnv> mockedStore1, store2;
+ final LatchTestEnv testEnv1 = new LatchTestEnv();
+ final LatchTestEnv testEnv2 = new LatchTestEnv();
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> reservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ mockedStore1 = EasyMock.createMockBuilder(UserFateStore.class)
+ .withConstructor(ClientContext.class, String.class, ZooUtil.LockID.class)
+ .withArgs(client, tableName, lock1).addMockedMethod("isDeadReservation").createMock();
Review Comment:
For now, I went with your next suggestion with passing the predicate to the stores. This is not as end-to-end because the lock is not actually in ZK but may be good enough for now. Can be done in follow on issue if desired.
##########
test/src/main/java/org/apache/accumulo/test/fate/MultipleStoresIT.java:
##########
@@ -0,0 +1,571 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.fate;
+
+import static org.apache.accumulo.test.fate.user.UserFateStoreIT.createFateTable;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.fate.Fate;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.FateStore;
+import org.apache.accumulo.core.fate.MetaFateStore;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore;
+import org.apache.accumulo.core.fate.Repo;
+import org.apache.accumulo.core.fate.user.UserFateStore;
+import org.apache.accumulo.core.fate.zookeeper.ZooCache;
+import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.test.util.Wait;
+import org.apache.accumulo.test.zookeeper.ZooKeeperTestingServer;
+import org.easymock.EasyMock;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+// TODO 4131 could potentially have separate classes for testing MetaFateStore and UserFateStore
+// similar to how FateTestRunner is used, however that interface doesn't work as nicely here
+// since we are using multiple stores instead of just one. Can do something similar to
+// FateTestRunner here if desired
+public class MultipleStoresIT extends SharedMiniClusterBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MultipleStoresIT.class);
+ @TempDir
+ private static File tempDir;
+ private static ZooKeeperTestingServer szk = null;
+ private static ZooReaderWriter zk;
+ private static final String FATE_DIR = "/fate";
+ private ClientContext client;
+
+ @BeforeEach
+ public void beforeEachSetup() {
+ client = (ClientContext) Accumulo.newClient().from(getClientProps()).build();
+ }
+
+ @AfterEach
+ public void afterEachTeardown() {
+ client.close();
+ }
+
+ @BeforeAll
+ public static void beforeAllSetup() throws Exception {
+ SharedMiniClusterBase.startMiniCluster();
+ szk = new ZooKeeperTestingServer(tempDir);
+ zk = szk.getZooReaderWriter();
+ }
+
+ @AfterAll
+ public static void afterAllTeardown() throws Exception {
+ SharedMiniClusterBase.stopMiniCluster();
+ szk.close();
+ }
+
+ @Test
+ public void testReserveUnreserve() throws Exception {
+ testReserveUnreserve(FateInstanceType.META);
+ testReserveUnreserve(FateInstanceType.USER);
+ }
+
+ private void testReserveUnreserve(FateInstanceType storeType) throws Exception {
+ // reserving/unreserving a FateId should be reflected across instances of the stores
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> activeReservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1);
+ store2 = new UserFateStore<>(client, tableName, lock2);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock1);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock2);
+ }
+
+ // Create the fate ids using store1
+ for (int i = 0; i < numFateIds; i++) {
+ assertTrue(allIds.add(store1.create()));
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Reserve half the fate ids using store1 and rest using store2, after reserving a fate id in
+ // one, should not be able to reserve the same in the other. Should also not matter that all the
+ // ids were created using store1
+ int count = 0;
+ for (FateId fateId : allIds) {
+ if (count % 2 == 0) {
+ reservations.add(store1.reserve(fateId));
+ assertTrue(store2.tryReserve(fateId).isEmpty());
+ } else {
+ reservations.add(store2.reserve(fateId));
+ assertTrue(store1.tryReserve(fateId).isEmpty());
+ }
+ count++;
+ }
+ // Both stores should return the same reserved transactions
+ activeReservations = store1.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+ activeReservations = store2.getActiveReservations();
+ assertEquals(allIds, activeReservations.keySet());
+
+ // Test setting/getting the TStatus and unreserving the transactions
+ for (int i = 0; i < allIds.size(); i++) {
+ var reservation = reservations.get(i);
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, reservation.getStatus());
+ reservation.setStatus(ReadOnlyFateStore.TStatus.SUBMITTED);
+ assertEquals(ReadOnlyFateStore.TStatus.SUBMITTED, reservation.getStatus());
+ reservation.delete();
+ reservation.unreserve(Duration.ofMillis(0));
+ // Attempt to set a status on a tx that has been unreserved (should throw exception)
+ assertThrows(IllegalStateException.class,
+ () -> reservation.setStatus(ReadOnlyFateStore.TStatus.NEW));
+ }
+ assertTrue(store1.getActiveReservations().isEmpty());
+ assertTrue(store2.getActiveReservations().isEmpty());
+ }
+
+ @Test
+ public void testReserveNonExistentTxn() throws Exception {
+ testReserveNonExistentTxn(FateInstanceType.META);
+ testReserveNonExistentTxn(FateInstanceType.USER);
+ }
+
+ private void testReserveNonExistentTxn(FateInstanceType storeType) throws Exception {
+ // Tests that reserve() doesn't hang indefinitely and instead throws an error
+ // on reserve() a non-existent transaction.
+ final FateStore<SleepingTestEnv> store;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final String tableName = getUniqueNames(1)[0];
+ final FateId fakeFateId = FateId.from(storeType, UUID.randomUUID());
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ assertThrows(IllegalStateException.class, () -> store.reserve(fakeFateId));
+ }
+
+ @Test
+ public void testReserveReservedAndUnreserveUnreserved() throws Exception {
+ testReserveReservedAndUnreserveUnreserved(FateInstanceType.META);
+ testReserveReservedAndUnreserveUnreserved(FateInstanceType.USER);
+ }
+
+ private void testReserveReservedAndUnreserveUnreserved(FateInstanceType storeType)
+ throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final FateStore<SleepingTestEnv> store;
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ // Create some FateIds and ensure that they can be reserved
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId = store.create();
+ assertTrue(allIds.add(fateId));
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservations.add(reservation.orElseThrow());
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Try to reserve again, should not reserve
+ for (FateId fateId : allIds) {
+ assertTrue(store.tryReserve(fateId).isEmpty());
+ }
+
+ // Unreserve all the FateIds
+ for (var reservation : reservations) {
+ reservation.unreserve(Duration.ofMillis(0));
+ }
+ // Try to unreserve again (should throw exception)
+ for (var reservation : reservations) {
+ assertThrows(IllegalStateException.class, () -> reservation.unreserve(Duration.ofMillis(0)));
+ }
+ }
+
+ @Test
+ public void testReserveAfterUnreserveAndReserveAfterDeleted() throws Exception {
+ testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType.META);
+ testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType.USER);
+ }
+
+ private void testReserveAfterUnreserveAndReserveAfterDeleted(FateInstanceType storeType)
+ throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final List<FateStore.FateTxStore<SleepingTestEnv>> reservations = new ArrayList<>();
+ final FateStore<SleepingTestEnv> store;
+ final ZooUtil.LockID lock = new ZooUtil.LockID("/locks", "L1", 50);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store = new UserFateStore<>(client, tableName, lock);
+ } else {
+ store = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock);
+ }
+
+ // Create some FateIds and ensure that they can be reserved
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId = store.create();
+ assertTrue(allIds.add(fateId));
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservations.add(reservation.orElseThrow());
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Unreserve all
+ for (var reservation : reservations) {
+ reservation.unreserve(Duration.ofMillis(0));
+ }
+
+ // Ensure they can be reserved again, and delete and unreserve this time
+ for (FateId fateId : allIds) {
+ // Verify that the tx status is still NEW after unreserving since it hasn't been deleted
+ assertEquals(ReadOnlyFateStore.TStatus.NEW, store.read(fateId).getStatus());
+ var reservation = store.tryReserve(fateId);
+ assertFalse(reservation.isEmpty());
+ reservation.orElseThrow().delete();
+ reservation.orElseThrow().unreserve(Duration.ofMillis(0));
+ }
+
+ for (FateId fateId : allIds) {
+ // Verify that the tx is now unknown since it has been deleted
+ assertEquals(ReadOnlyFateStore.TStatus.UNKNOWN, store.read(fateId).getStatus());
+ // Attempt to reserve a deleted txn, should throw an exception and not wait indefinitely
+ assertThrows(IllegalStateException.class, () -> store.reserve(fateId));
+ }
+ }
+
+ @Test
+ public void testMultipleFateInstances() throws Exception {
+ testMultipleFateInstances(FateInstanceType.META);
+ testMultipleFateInstances(FateInstanceType.USER);
+ }
+
+ private void testMultipleFateInstances(FateInstanceType storeType) throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+ final int numFateIds = 500;
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<SleepingTestEnv> store1, store2;
+ final SleepingTestEnv testEnv1 = new SleepingTestEnv(50);
+ final SleepingTestEnv testEnv2 = new SleepingTestEnv(50);
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ store1 = new UserFateStore<>(client, tableName, lock1);
+ store2 = new UserFateStore<>(client, tableName, lock2);
+ } else {
+ store1 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock1);
+ store2 = new MetaFateStore<>(FATE_DIR, zk, client.getZooCache(), lock2);
+ }
+
+ Fate<SleepingTestEnv> fate1 =
+ new Fate<>(testEnv1, store1, Object::toString, DefaultConfiguration.getInstance());
+ fate1.startDeadReservationCleaner();
+ Fate<SleepingTestEnv> fate2 =
+ new Fate<>(testEnv2, store2, Object::toString, DefaultConfiguration.getInstance());
+ fate2.startDeadReservationCleaner();
+
+ for (int i = 0; i < numFateIds; i++) {
+ FateId fateId;
+ // Start half the txns using fate1, and the other half using fate2
+ if (i % 2 == 0) {
+ fateId = fate1.startTransaction();
+ fate1.seedTransaction("op" + i, fateId, new SleepingTestRepo(), true, "test");
+ } else {
+ fateId = fate2.startTransaction();
+ fate2.seedTransaction("op" + i, fateId, new SleepingTestRepo(), true, "test");
+ }
+ allIds.add(fateId);
+ }
+ assertEquals(numFateIds, allIds.size());
+
+ // Should be able to wait for completion on any fate instance
+ for (FateId fateId : allIds) {
+ fate2.waitForCompletion(fateId);
+ }
+ // Ensure that all txns have been executed and have only been executed once
+ assertTrue(Collections.disjoint(testEnv1.executedOps, testEnv2.executedOps));
+ assertEquals(allIds, Sets.union(testEnv1.executedOps, testEnv2.executedOps));
+
+ fate1.shutdown(1, TimeUnit.MINUTES);
+ fate2.shutdown(1, TimeUnit.MINUTES);
+ }
+
+ @Test
+ public void testDeadReservationsCleanup() throws Exception {
+ testDeadReservationsCleanup(FateInstanceType.META);
+ testDeadReservationsCleanup(FateInstanceType.USER);
+ }
+
+ private void testDeadReservationsCleanup(FateInstanceType storeType) throws Exception {
+ // Tests reserving some transactions, then simulating that the Manager died by creating
+ // a new Fate instance and store with a new LockID. The transactions which were
+ // reserved using the old LockID should be cleaned up by Fate's DeadReservationCleaner,
+ // then picked up by the new Fate/store.
+
+ final String tableName = getUniqueNames(1)[0];
+ // One transaction for each FATE worker thread
+ final int numFateIds =
+ Integer.parseInt(Property.MANAGER_FATE_THREADPOOL_SIZE.getDefaultValue());
+ final boolean isUserStore = storeType.equals(FateInstanceType.USER);
+ final Set<FateId> allIds = new HashSet<>();
+ final FateStore<LatchTestEnv> mockedStore1, store2;
+ final LatchTestEnv testEnv1 = new LatchTestEnv();
+ final LatchTestEnv testEnv2 = new LatchTestEnv();
+ final ZooUtil.LockID lock1 = new ZooUtil.LockID("/locks", "L1", 50);
+ final ZooUtil.LockID lock2 = new ZooUtil.LockID("/locks", "L2", 52);
+ Map<FateId,FateStore.FateReservation> reservations;
+
+ if (isUserStore) {
+ createFateTable(client, tableName);
+ mockedStore1 = EasyMock.createMockBuilder(UserFateStore.class)
+ .withConstructor(ClientContext.class, String.class, ZooUtil.LockID.class)
+ .withArgs(client, tableName, lock1).addMockedMethod("isDeadReservation").createMock();
+ } else {
+ mockedStore1 = EasyMock.createMockBuilder(MetaFateStore.class)
+ .withConstructor(String.class, ZooReaderWriter.class, ZooCache.class,
+ ZooUtil.LockID.class)
+ .withArgs(FATE_DIR, zk, client.getZooCache(), lock1).addMockedMethod("isDeadReservation")
+ .createMock();
Review Comment:
The predicate is a great idea and cleans up the code quite a bit; added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@accumulo.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org