You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2024/10/06 18:35:59 UTC
(datasketches-java) 01/01: This fixes some tests
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch bring_6.1.X_to_match_master
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 3cf3643790e4219ddbb0aa16362f8b21f7670553
Author: Lee Rhodes <le...@gmail.com>
AuthorDate: Sun Oct 6 11:35:41 2024 -0700
This fixes some tests
that were not properly checking that the MemoryRequestServer (and the
sketch) were closing intermediate increases in size when off-heap.
---
.../org/apache/datasketches/hll/DirectAuxHashMapTest.java | 13 +++++++++----
.../quantiles/DirectQuantilesMemoryRequestTest.java | 12 ++++++++++++
.../apache/datasketches/quantiles/DoublesSketchTest.java | 6 ++++++
.../datasketches/theta/DirectQuickSelectSketchTest.java | 6 ++++++
.../java/org/apache/datasketches/theta/UnionImplTest.java | 3 +++
5 files changed, 36 insertions(+), 4 deletions(-)
diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
index 71774783..c1ef688c 100644
--- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
+++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
@@ -43,13 +43,16 @@ import org.apache.datasketches.memory.WritableMemory;
public class DirectAuxHashMapTest {
@Test
- public void checkGrow() {
+ public void checkGrow() { //It is very rare, but this forces an HLL_4 to exceed its computed memory size.
int lgConfigK = 4;
TgtHllType tgtHllType = TgtHllType.HLL_4;
int n = 8; //put lgConfigK == 4 into HLL mode
int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
HllSketch hllSketch;
WritableMemory wmem = WritableMemory.allocateDirect(bytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem; //copy of wmem
+ assertTrue(wmemCopy.isDirect()); //original copy of wmem is off-heap
+ assertTrue(wmemCopy.isAlive()); //original copy of wmem is Alive
hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
for (int i = 0; i < n; i++) {
hllSketch.update(i);
@@ -57,7 +60,7 @@ public class DirectAuxHashMapTest {
hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values
hllSketch.couponUpdate(HllUtil.pair(8, 15));
hllSketch.couponUpdate(HllUtil.pair(9, 15));
- //println(hllSketch.toString(true, true, true, true));
+ //println(hllSketch.toString(true, true, true, true)); //
DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl;
assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2);
assertTrue(hllSketch.isMemory());
@@ -75,19 +78,21 @@ public class DirectAuxHashMapTest {
byteArray = hllSketch.toUpdatableByteArray();
WritableMemory wmem2 = WritableMemory.writableWrap(byteArray);
hllSketch2 = HllSketch.writableWrap(wmem2);
- //println(hllSketch2.toString(true, true, true, true));
+ //println(hllSketch2.toString(true, true, true, true)); //
DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl;
assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2);
assertEquals(dha2.getAuxHashMap().getAuxCount(), 3);
//Check grow to on-heap
hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow
- //println(hllSketch.toString(true, true, true, true));
+ //println(hllSketch.toString(true, true, true, true)); //
dha = (DirectHllArray) hllSketch.hllSketchImpl;
assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3);
assertEquals(dha.getAuxHashMap().getAuxCount(), 4);
assertTrue(hllSketch.isMemory());
assertFalse(hllSketch.isOffHeap());
+ assertTrue(wmemCopy.isDirect()); //original copy of wmem was off-heap and still is
+ assertFalse(wmemCopy.isAlive()); //original copy of wmem has been closed
}
@Test
diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
index 2ec2fbe2..cfa14b60 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
@@ -48,6 +48,7 @@ public class DirectQuantilesMemoryRequestTest {
//########## Owning Implementation
// This part would actually be part of the Memory owning implementation so it is faked here
WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
println("Initial mem size: " + wmem.getCapacity());
//########## Receiving Application
@@ -70,6 +71,8 @@ public class DirectQuantilesMemoryRequestTest {
// so the the wmem reference is invalid. Use the sketch to get the last memory reference.
WritableMemory lastMem = usk1.getMemory();
println("Final mem size: " + usk1.getMemory().getCapacity());
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
@@ -79,6 +82,7 @@ public class DirectQuantilesMemoryRequestTest {
final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything
WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
println("Initial mem size: " + wmem.getCapacity());
final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
for (int i = 1; i <= u; i++) {
@@ -88,6 +92,8 @@ public class DirectQuantilesMemoryRequestTest {
println("curCombBufItemCap: " + currentSpace);
assertEquals(currentSpace, 2 * k);
println("last Mem Cap: " + usk1.getMemory().getCapacity());
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
@@ -97,6 +103,7 @@ public class DirectQuantilesMemoryRequestTest {
final int initBytes = ((2 * k) + 4) << 3; //just room for BB
WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
println("Initial mem size: " + wmem.getCapacity());
final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
for (int i = 1; i <= u; i++) {
@@ -108,6 +115,8 @@ public class DirectQuantilesMemoryRequestTest {
final int newSpace = usk1.getCombinedBufferItemCapacity();
println("newCombBurItemCap: " + newSpace);
assertEquals(newCB.length, 3 * k);
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
@@ -119,6 +128,7 @@ public class DirectQuantilesMemoryRequestTest {
final Memory origSketchMem = Memory.wrap(usk1.toByteArray());
WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
origSketchMem.copyTo(0, wmem, 0, initBytes);
UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem);
assertTrue(wmem.isSameResource(usk2.getMemory()));
@@ -135,6 +145,8 @@ public class DirectQuantilesMemoryRequestTest {
final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3);
assertEquals(mem2.getCapacity(), expectedSize);
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
index cf8e142b..fd1b7a72 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
@@ -140,18 +140,22 @@ public class DoublesSketchTest {
@Test
public void directSketchShouldMoveOntoHeapEventually() {
WritableMemory wmem = WritableMemory.allocateDirect(1000, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
Assert.assertTrue(sketch.isSameResource(wmem));
for (int i = 0; i < 1000; i++) {
sketch.update(i);
}
println(sketch.toString());
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
public void directSketchShouldMoveOntoHeapEventually2() {
int i = 0;
WritableMemory wmem = WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
Assert.assertTrue(sketch.isSameResource(wmem));
for (; i < 1000; i++) {
@@ -163,6 +167,8 @@ public class DoublesSketchTest {
}
}
assertFalse(wmem.isAlive());
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
index 9b129daa..2aa195a0 100644
--- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
@@ -781,13 +781,16 @@ public class DirectQuickSelectSketchTest {
int u = 2 * k;
int bytes = Sketches.getMaxUpdateSketchBytes(k);
WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request more memory
+ WritableMemory wmemCopy = wmem;
UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
assertTrue(sketch.isSameResource(wmem));
for (int i = 0; i < u; i++) { sketch.update(i); }
Memory mem = sketch.getMemory();
assertTrue(mem.isAlive());
assertFalse(mem.isDirect()); //now on heap.
+ assertTrue(wmemCopy.isDirect()); //original copy
assertFalse(wmem.isAlive()); //wmem closed by MemoryRequestServer
+ assertFalse(wmemCopy.isAlive()); //original copy closed
}
@Test
@@ -796,6 +799,7 @@ public class DirectQuickSelectSketchTest {
int u = 2 * k;
int bytes = Sketches.getMaxUpdateSketchBytes(k);
WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request more memory
+ WritableMemory wmemCopy = wmem;
UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
for (int i = 0; i < u; i++) { sketch.update(i); }
double est1 = sketch.getEstimate();
@@ -808,6 +812,8 @@ public class DirectQuickSelectSketchTest {
assertTrue(mem2.isAlive());
assertFalse(mem2.isDirect()); //now on heap
assertFalse(wmem.isAlive()); //wmem closed by MemoryRequestServer
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
try {
roSketch.rebuild();
fail();
diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java
index 2fbd7bdc..4bade62f 100644
--- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java
+++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java
@@ -193,6 +193,7 @@ public class UnionImplTest {
final int bytes = Sketches.getMaxUpdateSketchBytes(k);
WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2); //not really used, except as a reference.
WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2); //too small, forces new allocation on heap
+ WritableMemory wmem2Copy = wmem2;
final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
assertTrue(sketch.isSameResource(wmem)); //also testing the isSameResource function
@@ -206,6 +207,8 @@ public class UnionImplTest {
assertFalse(union2.isSameResource(wmem2)); //obviously not
wmem.close(); //empty, but we must close it anyway.
assertFalse(wmem2.isAlive());//previously closed via the DefaultMemoryRequestServer.
+ assertTrue(wmem2Copy.isDirect());
+ assertFalse(wmem2Copy.isAlive());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org