You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2021/08/10 18:13:11 UTC
[datasketches-memory] branch addMimicDruidTest created (now 58bb816)
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a change to branch addMimicDruidTest
in repository https://gitbox.apache.org/repos/asf/datasketches-memory.git.
at 58bb816 This test mimics the Druid issue #11544, but at a much smaller scale.
This branch includes the following new commits:
new 58bb816 This test mimics the Druid issue #11544, but at a much smaller scale.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-memory] 01/01: This test mimics the Druid issue
#11544, but at a much smaller scale.
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch addMimicDruidTest
in repository https://gitbox.apache.org/repos/asf/datasketches-memory.git
commit 58bb81668786868805a528fd508e98dce4c48ba1
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Tue Aug 10 11:12:22 2021 -0700
This test mimics the Druid issue #11544, but at a much smaller scale.
---
.../memory/test/DruidIssue11544Test.java | 89 ++++++++++++++++++++++
1 file changed, 89 insertions(+)
diff --git a/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/DruidIssue11544Test.java b/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/DruidIssue11544Test.java
new file mode 100644
index 0000000..b5e7e63
--- /dev/null
+++ b/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/DruidIssue11544Test.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.apache.datasketches.memory.MemoryRequestServer;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+/**
+ * The original design provided the MemoryRequestServer callback only for Memory segments allocated via
+ * <i>WritableMemory.allocateDirect(...)</i> calls. Memory segments allocated via
+ * <i>WritableMemory.wrap(ByteBuffer)</i> did not have this capability. This was a major oversight since
+ * all off-heap memory in Druid is allocated using ByteBuffers! It is unusual that no one has
+ * uncovered this until August 2021. Nonetheless, the fix involves instrumenting all the paths involved
+ * in providing this callback mechanism for wrapped ByteBuffers.
+ *
+ * This issues was first identified in Druid Issue #11544 and then posted as DataSketches-java Issue #358.
+ * But the actual source of the problem was in Memory.
+ *
+ * This test mimics the Druid issue but at a much smaller scale.
+ *
+ * @author Lee Rhodes
+ *
+ */
+public class DruidIssue11544Test {
+
+ @Test
+ public void withByteBuffer() {
+ int initialLongs = 1000;
+ int initialMemSize = initialLongs * 8;
+ ByteBuffer bb = ByteBuffer.allocateDirect(initialMemSize);
+ bb.order(ByteOrder.nativeOrder());
+
+ //Fill the byte buffer
+ for (int i = 0; i < initialLongs; i++) { bb.putLong(i * 8, i); }
+
+ //Wrap, assuming default MemoryRequestServer
+ WritableMemory mem = WritableMemory.writableWrap(bb);
+ assertTrue(mem.isDirect()); //confirm mem is off-heap
+
+ //Request Bigger Memory
+ MemoryRequestServer svr = mem.getMemoryRequestServer();
+ assertNotNull(svr); //before the fix, this was null.
+
+ WritableMemory newMem = svr.request(initialMemSize * 2);
+
+ //Confirm that newMem is on the heap (the default) and 2X size
+ assertFalse(newMem.isDirect());
+ assertEquals(newMem.getCapacity(), 2 * initialMemSize);
+
+ //Move data to new memory
+ mem.copyTo(0, newMem, 0, initialMemSize);
+
+ //Prepare to request deallocation
+ WritableMemory oldMem = mem;
+ mem = newMem;
+
+ //In the DefaultMemoryRequestServer, this is a no-op, so nothing is actually deallocated.
+ svr.requestClose(oldMem, newMem);
+ assertTrue(oldMem.isValid());
+ assertTrue(mem.isValid());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org