You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2021/04/13 18:35:01 UTC
[ignite-3] 04/04: IGNITE-14389 getAll and tests (WIP)
This is an automated email from the ASF dual-hosted git repository.
agura pushed a commit to branch ignite-14389
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit 8ff3571bce120ea41bf5f1123c6c52b51c36a983
Author: Andrey Gura <ag...@apache.org>
AuthorDate: Tue Apr 13 21:34:37 2021 +0300
IGNITE-14389 getAll and tests (WIP)
---
.../metastorage/client/MetaStorageService.java | 4 +-
modules/metastorage-server/pom.xml | 6 +
.../metastorage/server/KeyValueStorage.java | 8 +
.../server/SimpleInMemoryKeyValueStorage.java | 36 +++-
.../server/SimpleInMemoryKeyValueStorageTest.java | 208 +++++++++++++++++++--
5 files changed, 244 insertions(+), 18 deletions(-)
diff --git a/modules/metastorage-client/src/main/java/org/apache/ignite/metastorage/client/MetaStorageService.java b/modules/metastorage-client/src/main/java/org/apache/ignite/metastorage/client/MetaStorageService.java
index dfb4868..d4ded44 100644
--- a/modules/metastorage-client/src/main/java/org/apache/ignite/metastorage/client/MetaStorageService.java
+++ b/modules/metastorage-client/src/main/java/org/apache/ignite/metastorage/client/MetaStorageService.java
@@ -52,7 +52,7 @@ public interface MetaStorageService {
* Retrieves an entry for the given key and the revision upper bound.
*
* @param key The key. Couldn't be {@code null}.
- * @param revUpperBound The upper bound for entry revisions. Must be positive.
+ * @param revUpperBound The upper bound for entry revisions. Must be positive.
* @return An entry for the given key and maximum revision limited by {@code revUpperBound}.
* Couldn't be {@code null}.
* @throws OperationTimeoutException If the operation is timed out. Will be thrown on getting future result.
@@ -82,7 +82,7 @@ public interface MetaStorageService {
*
* @param keys The collection of keys. Couldn't be {@code null} or empty.
* Collection elements couldn't be {@code null}.
- * @param revUpperBound The upper bound for entry revisions. Must be positive.
+ * @param revUpperBound The upper bound for entry revisions. Must be positive.
* @return A map of entries for given keys and maximum revision limited by {@code revUpperBound}.
* Couldn't be {@code null}.
* @throws OperationTimeoutException If the operation is timed out. Will be thrown on getting future result.
diff --git a/modules/metastorage-server/pom.xml b/modules/metastorage-server/pom.xml
index 3c51fc5..d73d080 100644
--- a/modules/metastorage-server/pom.xml
+++ b/modules/metastorage-server/pom.xml
@@ -40,6 +40,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>metastorage-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
</dependency>
diff --git a/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java b/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java
index ead1043..0596c4a 100644
--- a/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java
+++ b/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/KeyValueStorage.java
@@ -2,7 +2,9 @@ package org.apache.ignite.internal.metastorage.server;
import org.jetbrains.annotations.NotNull;
+import java.util.Collection;
import java.util.Iterator;
+import java.util.List;
public interface KeyValueStorage {
@@ -16,6 +18,12 @@ public interface KeyValueStorage {
@NotNull
Entry get(byte[] key, long rev);
+ @NotNull
+ Collection<Entry> getAll(List<byte[]> keys);
+
+ @NotNull
+ Collection<Entry> getAll(List<byte[]> keys, long revUpperBound);
+
void put(byte[] key, byte[] value);
@NotNull
diff --git a/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java b/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java
index 3700f4a..8523f51 100644
--- a/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java
+++ b/modules/metastorage-server/src/main/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorage.java
@@ -2,6 +2,7 @@ package org.apache.ignite.internal.metastorage.server;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
@@ -78,6 +79,16 @@ public class SimpleInMemoryKeyValueStorage implements KeyValueStorage {
}
@Override
+ public @NotNull Collection<Entry> getAll(List<byte[]> keys) {
+ return doGetAll(keys, LATEST_REV);
+ }
+
+ @Override
+ public @NotNull Collection<Entry> getAll(List<byte[]> keys, long revUpperBound) {
+ return doGetAll(keys, revUpperBound);
+ }
+
+ @Override
public void remove(byte[] key) {
synchronized (mux) {
Entry e = doGet(key, LATEST_REV, false);
@@ -197,17 +208,17 @@ public class SimpleInMemoryKeyValueStorage implements KeyValueStorage {
NavigableMap<byte[], List<Long>> compactedKeysIdx,
NavigableMap<Long, NavigableMap<byte[], Value>> compactedRevsIdx
) {
- Long lrev = lastRevision(revs);
+ Long lastRev = lastRevision(revs);
- NavigableMap<byte[], Value> kv = revsIdx.get(lrev);
+ NavigableMap<byte[], Value> kv = revsIdx.get(lastRev);
Value lastVal = kv.get(key);
if (!lastVal.tombstone()) {
- compactedKeysIdx.put(key, listOf(lrev));
+ compactedKeysIdx.put(key, listOf(lastRev));
NavigableMap<byte[], Value> compactedKv = compactedRevsIdx.computeIfAbsent(
- lrev,
+ lastRev,
k -> new TreeMap<>(LEXICOGRAPHIC_COMPARATOR)
);
@@ -215,6 +226,23 @@ public class SimpleInMemoryKeyValueStorage implements KeyValueStorage {
}
}
+ @NotNull
+ private Collection<Entry> doGetAll(List<byte[]> keys, long rev) {
+ assert keys != null : "keys list can't be null.";
+ assert !keys.isEmpty() : "keys list can't be empty.";
+ assert rev > 0 : "Revision must be positive.";
+
+ Collection<Entry> res = new ArrayList<>(keys.size());
+
+ synchronized (mux) {
+ for (byte[] key : keys) {
+ res.add(doGet(key, rev, false));
+ }
+ }
+
+ return res;
+ }
+
/**
* Returns entry for given key.
*
diff --git a/modules/metastorage-server/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java b/modules/metastorage-server/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java
index 5b797fc..fa130e6 100644
--- a/modules/metastorage-server/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java
+++ b/modules/metastorage-server/src/test/java/org/apache/ignite/internal/metastorage/server/SimpleInMemoryKeyValueStorageTest.java
@@ -1,17 +1,18 @@
package org.apache.ignite.internal.metastorage.server;
-import org.jetbrains.annotations.NotNull;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
+import java.util.Collection;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.ignite.metastorage.common.Key;
+import org.jetbrains.annotations.NotNull;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
-import static org.junit.jupiter.api.Assertions.assertArrayEquals;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
class SimpleInMemoryKeyValueStorageTest {
private KeyValueStorage storage;
@@ -56,6 +57,193 @@ class SimpleInMemoryKeyValueStorageTest {
}
@Test
+ void getAll() {
+ byte[] key1 = k(1);
+ byte[] val1 = kv(1, 1);
+
+ byte[] key2 = k(2);
+ byte[] val2_1 = kv(2, 21);
+ byte[] val2_2 = kv(2, 22);
+
+ byte[] key3 = k(3);
+ byte[] val3 = kv(3, 3);
+
+ byte[] key4 = k(4);
+
+ assertEquals(0, storage.revision());
+ assertEquals(0, storage.updateCounter());
+
+ // Regular put.
+ storage.put(key1, val1);
+
+ // Rewrite.
+ storage.put(key2, val2_1);
+ storage.put(key2, val2_2);
+
+ // Remove.
+ storage.put(key3, val3);
+ storage.remove(key3);
+
+ assertEquals(5, storage.revision());
+ assertEquals(5, storage.updateCounter());
+
+ Collection<Entry> entries = storage.getAll(List.of(key1, key2, key3, key4));
+
+ assertEquals(4, entries.size());
+
+ Map<Key, Entry> map = entries.stream().collect(Collectors.toMap(e -> new Key(e.key()), Function.identity()));
+
+ // Test regular put value.
+ Entry e1 = map.get(new Key(key1));
+
+ assertNotNull(e1);
+ assertEquals(1, e1.revision());
+ assertEquals(1, e1.updateCounter());
+ assertFalse(e1.tombstone());
+ assertFalse(e1.empty());
+ assertArrayEquals(val1, e1.value());
+
+ // Test rewritten value.
+ Entry e2 = map.get(new Key(key2));
+
+ assertNotNull(e2);
+ assertEquals(3, e2.revision());
+ assertEquals(3, e2.updateCounter());
+ assertFalse(e2.tombstone());
+ assertFalse(e2.empty());
+ assertArrayEquals(val2_2, e2.value());
+
+ // Test removed value.
+ Entry e3 = map.get(new Key(key3));
+
+ assertNotNull(e3);
+ assertEquals(5, e3.revision());
+ assertEquals(5, e3.updateCounter());
+ assertTrue(e3.tombstone());
+ assertFalse(e3.empty());
+
+ // Test empty value.
+ Entry e4 = map.get(new Key(key4));
+
+ assertNotNull(e4);
+ assertFalse(e4.tombstone());
+ assertTrue(e4.empty());
+ }
+
+ @Test
+ void getAllWithRevisionBound() {
+ byte[] key1 = k(1);
+ byte[] val1 = kv(1, 1);
+
+ byte[] key2 = k(2);
+ byte[] val2_1 = kv(2, 21);
+ byte[] val2_2 = kv(2, 22);
+
+ byte[] key3 = k(3);
+ byte[] val3 = kv(3, 3);
+
+ byte[] key4 = k(4);
+
+ assertEquals(0, storage.revision());
+ assertEquals(0, storage.updateCounter());
+
+ // Regular put.
+ storage.put(key1, val1);
+
+ // Rewrite.
+ storage.put(key2, val2_1);
+ storage.put(key2, val2_2);
+
+ // Remove.
+ storage.put(key3, val3);
+ storage.remove(key3);
+
+ assertEquals(5, storage.revision());
+ assertEquals(5, storage.updateCounter());
+
+ // Bounded by revision 2.
+ Collection<Entry> entries = storage.getAll(List.of(key1, key2, key3, key4), 2);
+
+ assertEquals(4, entries.size());
+
+ Map<Key, Entry> map = entries.stream().collect(Collectors.toMap(e -> new Key(e.key()), Function.identity()));
+
+ // Test regular put value.
+ Entry e1 = map.get(new Key(key1));
+
+ assertNotNull(e1);
+ assertEquals(1, e1.revision());
+ assertEquals(1, e1.updateCounter());
+ assertFalse(e1.tombstone());
+ assertFalse(e1.empty());
+ assertArrayEquals(val1, e1.value());
+
+ // Test while not rewritten value.
+ Entry e2 = map.get(new Key(key2));
+
+ assertNotNull(e2);
+ assertEquals(2, e2.revision());
+ assertEquals(2, e2.updateCounter());
+ assertFalse(e2.tombstone());
+ assertFalse(e2.empty());
+ assertArrayEquals(val2_1, e2.value());
+
+ // Values with larger revision don't exist yet.
+ Entry e3 = map.get(new Key(key3));
+
+ assertNotNull(e3);
+ assertTrue(e3.empty());
+
+ Entry e4 = map.get(new Key(key4));
+
+ assertNotNull(e4);
+ assertTrue(e4.empty());
+
+ // Bounded by revision 4.
+ entries = storage.getAll(List.of(key1, key2, key3, key4), 4);
+
+ assertEquals(4, entries.size());
+
+ map = entries.stream().collect(Collectors.toMap(e -> new Key(e.key()), Function.identity()));
+
+ // Test regular put value.
+ e1 = map.get(new Key(key1));
+
+ assertNotNull(e1);
+ assertEquals(1, e1.revision());
+ assertEquals(1, e1.updateCounter());
+ assertFalse(e1.tombstone());
+ assertFalse(e1.empty());
+ assertArrayEquals(val1, e1.value());
+
+ // Test rewritten value.
+ e2 = map.get(new Key(key2));
+
+ assertNotNull(e2);
+ assertEquals(3, e2.revision());
+ assertEquals(3, e2.updateCounter());
+ assertFalse(e2.tombstone());
+ assertFalse(e2.empty());
+ assertArrayEquals(val2_2, e2.value());
+
+ // Test not removed value.
+ e3 = map.get(new Key(key3));
+
+ assertNotNull(e3);
+ assertEquals(4, e3.revision());
+ assertEquals(4, e3.updateCounter());
+ assertFalse(e3.tombstone());
+ assertFalse(e3.empty());
+ assertArrayEquals(val3, e3.value());
+
+ // Value with larger revision doesn't exist yet.
+ e4 = map.get(new Key(key4));
+
+ assertNotNull(e4);
+ assertTrue(e4.empty());
+ }
+
+ @Test
public void getAndPut() {
byte[] key = k(1);
byte[] val = kv(1, 1);
@@ -208,7 +396,6 @@ class SimpleInMemoryKeyValueStorageTest {
@Test
public void getAndPutAfterRemove() {
byte[] key = k(1);
-
byte[] val = kv(1, 1);
storage.getAndPut(key, val);
@@ -218,11 +405,8 @@ class SimpleInMemoryKeyValueStorageTest {
Entry e = storage.getAndPut(key, val);
assertEquals(3, storage.revision());
-
assertEquals(3, storage.updateCounter());
-
assertEquals(2, e.revision());
-
assertTrue(e.tombstone());
}