You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2024/10/06 18:35:58 UTC
(datasketches-java) branch bring_6.1.X_to_match_master created (now 3cf36437)
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a change to branch bring_6.1.X_to_match_master
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
at 3cf36437 This fixes some tests
This branch includes the following new commits:
new 3cf36437 This fixes some tests
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
(datasketches-java) 01/01: This fixes some tests
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch bring_6.1.X_to_match_master
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 3cf3643790e4219ddbb0aa16362f8b21f7670553
Author: Lee Rhodes <le...@gmail.com>
AuthorDate: Sun Oct 6 11:35:41 2024 -0700
This fixes some tests
that were not properly checking that the MemoryRequestServer (and the
sketch) were closing intermediate increases in size when off-heap.
---
.../org/apache/datasketches/hll/DirectAuxHashMapTest.java | 13 +++++++++----
.../quantiles/DirectQuantilesMemoryRequestTest.java | 12 ++++++++++++
.../apache/datasketches/quantiles/DoublesSketchTest.java | 6 ++++++
.../datasketches/theta/DirectQuickSelectSketchTest.java | 6 ++++++
.../java/org/apache/datasketches/theta/UnionImplTest.java | 3 +++
5 files changed, 36 insertions(+), 4 deletions(-)
diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
index 71774783..c1ef688c 100644
--- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
+++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java
@@ -43,13 +43,16 @@ import org.apache.datasketches.memory.WritableMemory;
public class DirectAuxHashMapTest {
@Test
- public void checkGrow() {
+ public void checkGrow() { //It is very rare, but this forces an HLL_4 to exceed its computed memory size.
int lgConfigK = 4;
TgtHllType tgtHllType = TgtHllType.HLL_4;
int n = 8; //put lgConfigK == 4 into HLL mode
int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
HllSketch hllSketch;
WritableMemory wmem = WritableMemory.allocateDirect(bytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem; //copy of wmem
+ assertTrue(wmemCopy.isDirect()); //original copy of wmem is off-heap
+ assertTrue(wmemCopy.isAlive()); //original copy of wmem is Alive
hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
for (int i = 0; i < n; i++) {
hllSketch.update(i);
@@ -57,7 +60,7 @@ public class DirectAuxHashMapTest {
hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values
hllSketch.couponUpdate(HllUtil.pair(8, 15));
hllSketch.couponUpdate(HllUtil.pair(9, 15));
- //println(hllSketch.toString(true, true, true, true));
+ //println(hllSketch.toString(true, true, true, true)); //
DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl;
assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2);
assertTrue(hllSketch.isMemory());
@@ -75,19 +78,21 @@ public class DirectAuxHashMapTest {
byteArray = hllSketch.toUpdatableByteArray();
WritableMemory wmem2 = WritableMemory.writableWrap(byteArray);
hllSketch2 = HllSketch.writableWrap(wmem2);
- //println(hllSketch2.toString(true, true, true, true));
+ //println(hllSketch2.toString(true, true, true, true)); //
DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl;
assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2);
assertEquals(dha2.getAuxHashMap().getAuxCount(), 3);
//Check grow to on-heap
hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow
- //println(hllSketch.toString(true, true, true, true));
+ //println(hllSketch.toString(true, true, true, true)); //
dha = (DirectHllArray) hllSketch.hllSketchImpl;
assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3);
assertEquals(dha.getAuxHashMap().getAuxCount(), 4);
assertTrue(hllSketch.isMemory());
assertFalse(hllSketch.isOffHeap());
+ assertTrue(wmemCopy.isDirect()); //original copy of wmem was off-heap and still is
+ assertFalse(wmemCopy.isAlive()); //original copy of wmem has been closed
}
@Test
diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
index 2ec2fbe2..cfa14b60 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java
@@ -48,6 +48,7 @@ public class DirectQuantilesMemoryRequestTest {
//########## Owning Implementation
// This part would actually be part of the Memory owning implementation so it is faked here
WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
println("Initial mem size: " + wmem.getCapacity());
//########## Receiving Application
@@ -70,6 +71,8 @@ public class DirectQuantilesMemoryRequestTest {
// so the the wmem reference is invalid. Use the sketch to get the last memory reference.
WritableMemory lastMem = usk1.getMemory();
println("Final mem size: " + usk1.getMemory().getCapacity());
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
@@ -79,6 +82,7 @@ public class DirectQuantilesMemoryRequestTest {
final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything
WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
println("Initial mem size: " + wmem.getCapacity());
final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
for (int i = 1; i <= u; i++) {
@@ -88,6 +92,8 @@ public class DirectQuantilesMemoryRequestTest {
println("curCombBufItemCap: " + currentSpace);
assertEquals(currentSpace, 2 * k);
println("last Mem Cap: " + usk1.getMemory().getCapacity());
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
@@ -97,6 +103,7 @@ public class DirectQuantilesMemoryRequestTest {
final int initBytes = ((2 * k) + 4) << 3; //just room for BB
WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
println("Initial mem size: " + wmem.getCapacity());
final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
for (int i = 1; i <= u; i++) {
@@ -108,6 +115,8 @@ public class DirectQuantilesMemoryRequestTest {
final int newSpace = usk1.getCombinedBufferItemCapacity();
println("newCombBurItemCap: " + newSpace);
assertEquals(newCB.length, 3 * k);
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
@@ -119,6 +128,7 @@ public class DirectQuantilesMemoryRequestTest {
final Memory origSketchMem = Memory.wrap(usk1.toByteArray());
WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
origSketchMem.copyTo(0, wmem, 0, initBytes);
UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem);
assertTrue(wmem.isSameResource(usk2.getMemory()));
@@ -135,6 +145,8 @@ public class DirectQuantilesMemoryRequestTest {
final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3);
assertEquals(mem2.getCapacity(), expectedSize);
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
index cf8e142b..fd1b7a72 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java
@@ -140,18 +140,22 @@ public class DoublesSketchTest {
@Test
public void directSketchShouldMoveOntoHeapEventually() {
WritableMemory wmem = WritableMemory.allocateDirect(1000, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
Assert.assertTrue(sketch.isSameResource(wmem));
for (int i = 0; i < 1000; i++) {
sketch.update(i);
}
println(sketch.toString());
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
public void directSketchShouldMoveOntoHeapEventually2() {
int i = 0;
WritableMemory wmem = WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer());
+ WritableMemory wmemCopy = wmem;
UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
Assert.assertTrue(sketch.isSameResource(wmem));
for (; i < 1000; i++) {
@@ -163,6 +167,8 @@ public class DoublesSketchTest {
}
}
assertFalse(wmem.isAlive());
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
}
@Test
diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
index 9b129daa..2aa195a0 100644
--- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
+++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java
@@ -781,13 +781,16 @@ public class DirectQuickSelectSketchTest {
int u = 2 * k;
int bytes = Sketches.getMaxUpdateSketchBytes(k);
WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request more memory
+ WritableMemory wmemCopy = wmem;
UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
assertTrue(sketch.isSameResource(wmem));
for (int i = 0; i < u; i++) { sketch.update(i); }
Memory mem = sketch.getMemory();
assertTrue(mem.isAlive());
assertFalse(mem.isDirect()); //now on heap.
+ assertTrue(wmemCopy.isDirect()); //original copy
assertFalse(wmem.isAlive()); //wmem closed by MemoryRequestServer
+ assertFalse(wmemCopy.isAlive()); //original copy closed
}
@Test
@@ -796,6 +799,7 @@ public class DirectQuickSelectSketchTest {
int u = 2 * k;
int bytes = Sketches.getMaxUpdateSketchBytes(k);
WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request more memory
+ WritableMemory wmemCopy = wmem;
UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
for (int i = 0; i < u; i++) { sketch.update(i); }
double est1 = sketch.getEstimate();
@@ -808,6 +812,8 @@ public class DirectQuickSelectSketchTest {
assertTrue(mem2.isAlive());
assertFalse(mem2.isDirect()); //now on heap
assertFalse(wmem.isAlive()); //wmem closed by MemoryRequestServer
+ assertTrue(wmemCopy.isDirect());
+ assertFalse(wmemCopy.isAlive());
try {
roSketch.rebuild();
fail();
diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java
index 2fbd7bdc..4bade62f 100644
--- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java
+++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java
@@ -193,6 +193,7 @@ public class UnionImplTest {
final int bytes = Sketches.getMaxUpdateSketchBytes(k);
WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2); //not really used, except as a reference.
WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2); //too small, forces new allocation on heap
+ WritableMemory wmem2Copy = wmem2;
final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
assertTrue(sketch.isSameResource(wmem)); //also testing the isSameResource function
@@ -206,6 +207,8 @@ public class UnionImplTest {
assertFalse(union2.isSameResource(wmem2)); //obviously not
wmem.close(); //empty, but we must close it anyway.
assertFalse(wmem2.isAlive());//previously closed via the DefaultMemoryRequestServer.
+ assertTrue(wmem2Copy.isDirect());
+ assertFalse(wmem2Copy.isAlive());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org