You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by dc...@apache.org on 2021/08/19 22:18:24 UTC
svn commit: r49567 [3/14] - in /dev/datasketches/memory/2.0.0-RC1: ./
apache-datasketches-memory-2.0.0-src/
apache-datasketches-memory-2.0.0-src/datasketches-memory-java11/
apache-datasketches-memory-2.0.0-src/datasketches-memory-java11/src/
apache-dat...
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/CommonMemoryTest.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/CommonMemoryTest.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/CommonMemoryTest.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,417 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import static org.apache.datasketches.memory.internal.Util.isAllBitsClear;
+import static org.apache.datasketches.memory.internal.Util.isAllBitsSet;
+import static org.apache.datasketches.memory.internal.Util.isAnyBitsClear;
+import static org.apache.datasketches.memory.internal.Util.isAnyBitsSet;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import org.apache.datasketches.memory.WritableHandle;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+@SuppressWarnings("javadoc")
+public class CommonMemoryTest {
+
+ @Test
+ public void checkSetGet() throws Exception {
+ int memCapacity = 16; //must be at least 8
+ try (WritableHandle wrh = WritableMemory.allocateDirect(memCapacity)) {
+ WritableMemory mem = wrh.getWritable();
+ assertEquals(mem.getCapacity(), memCapacity);
+ setGetTests(mem);
+ }
+ }
+
+ public static void setGetTests(WritableMemory mem) {
+ mem.putBoolean(0, true);
+ assertEquals(mem.getBoolean(0), true);
+ mem.putBoolean(0, false);
+ assertEquals(mem.getBoolean(0), false);
+
+ mem.putByte(0, (byte) -1);
+ assertEquals(mem.getByte(0), (byte) -1);
+ mem.putByte(0, (byte) 0);
+ assertEquals(mem.getByte(0), (byte) 0);
+
+ mem.putChar(0, 'A');
+ assertEquals(mem.getChar(0), 'A');
+ mem.putChar(0, 'Z');
+ assertEquals(mem.getChar(0), 'Z');
+
+ mem.putShort(0, Short.MAX_VALUE);
+ assertEquals(mem.getShort(0), Short.MAX_VALUE);
+ mem.putShort(0, Short.MIN_VALUE);
+ assertEquals(mem.getShort(0), Short.MIN_VALUE);
+
+ mem.putInt(0, Integer.MAX_VALUE);
+ assertEquals(mem.getInt(0), Integer.MAX_VALUE);
+ mem.putInt(0, Integer.MIN_VALUE);
+ assertEquals(mem.getInt(0), Integer.MIN_VALUE);
+
+ mem.putFloat(0, Float.MAX_VALUE);
+ assertEquals(mem.getFloat(0), Float.MAX_VALUE);
+ mem.putFloat(0, Float.MIN_VALUE);
+ assertEquals(mem.getFloat(0), Float.MIN_VALUE);
+
+ mem.putLong(0, Long.MAX_VALUE);
+ assertEquals(mem.getLong(0), Long.MAX_VALUE);
+ mem.putLong(0, Long.MIN_VALUE);
+ assertEquals(mem.getLong(0), Long.MIN_VALUE);
+
+ mem.putDouble(0, Double.MAX_VALUE);
+ assertEquals(mem.getDouble(0), Double.MAX_VALUE);
+ mem.putDouble(0, Double.MIN_VALUE);
+ assertEquals(mem.getDouble(0), Double.MIN_VALUE);
+ }
+
+ @Test
+ public void checkSetGetArrays() throws Exception {
+ int memCapacity = 32;
+ try (WritableHandle wrh = WritableMemory.allocateDirect(memCapacity)) {
+ WritableMemory mem = wrh.getWritable();
+ assertEquals(memCapacity, mem.getCapacity());
+ setGetArraysTests(mem);
+ }
+ }
+
+ public static void setGetArraysTests(WritableMemory mem) {
+ int accessCapacity = (int)mem.getCapacity();
+
+ int words = 4;
+ boolean[] srcArray1 = {true, false, true, false};
+ boolean[] dstArray1 = new boolean[words];
+ mem.fill(0, accessCapacity, (byte)127);
+ mem.putBooleanArray(0, srcArray1, 0, words);
+ mem.getBooleanArray(0, dstArray1, 0, words);
+ for (int i = 0; i < words; i++) {
+ assertEquals(dstArray1[i], srcArray1[i]);
+ }
+
+ byte[] srcArray2 = { 1, -2, 3, -4 };
+ byte[] dstArray2 = new byte[4];
+ mem.putByteArray(0, srcArray2, 0, words);
+ mem.getByteArray(0, dstArray2, 0, words);
+ for (int i = 0; i < words; i++) {
+ assertEquals(dstArray2[i], srcArray2[i]);
+ }
+
+ char[] srcArray3 = { 'A', 'B', 'C', 'D' };
+ char[] dstArray3 = new char[words];
+ mem.putCharArray(0, srcArray3, 0, words);
+ mem.getCharArray(0, dstArray3, 0, words);
+ for (int i = 0; i < words; i++) {
+ assertEquals(dstArray3[i], srcArray3[i]);
+ }
+
+ double[] srcArray4 = { 1.0, -2.0, 3.0, -4.0 };
+ double[] dstArray4 = new double[words];
+ mem.putDoubleArray(0, srcArray4, 0, words);
+ mem.getDoubleArray(0, dstArray4, 0, words);
+ for (int i = 0; i < words; i++) {
+ assertEquals(dstArray4[i], srcArray4[i], 0.0);
+ }
+
+ float[] srcArray5 = { (float)1.0, (float)-2.0, (float)3.0, (float)-4.0 };
+ float[] dstArray5 = new float[words];
+ mem.putFloatArray(0, srcArray5, 0, words);
+ mem.getFloatArray(0, dstArray5, 0, words);
+ for (int i = 0; i < words; i++) {
+ assertEquals(dstArray5[i], srcArray5[i], 0.0);
+ }
+
+ int[] srcArray6 = { 1, -2, 3, -4 };
+ int[] dstArray6 = new int[words];
+ mem.putIntArray(0, srcArray6, 0, words);
+ mem.getIntArray(0, dstArray6, 0, words);
+ for (int i = 0; i < words; i++) {
+ assertEquals(dstArray6[i], srcArray6[i]);
+ }
+
+ long[] srcArray7 = { 1, -2, 3, -4 };
+ long[] dstArray7 = new long[words];
+ mem.putLongArray(0, srcArray7, 0, words);
+ mem.getLongArray(0, dstArray7, 0, words);
+ for (int i = 0; i < words; i++) {
+ assertEquals(dstArray7[i], srcArray7[i]);
+ }
+
+ short[] srcArray8 = { 1, -2, 3, -4 };
+ short[] dstArray8 = new short[words];
+ mem.putShortArray(0, srcArray8, 0, words);
+ mem.getShortArray(0, dstArray8, 0, words);
+ for (int i = 0; i < words; i++) {
+ assertEquals(dstArray8[i], srcArray8[i]);
+ }
+ }
+
+ @Test
+ public void checkSetGetPartialArraysWithOffset() throws Exception {
+ int memCapacity = 32;
+ try (WritableHandle wrh = WritableMemory.allocateDirect(memCapacity)) {
+ WritableMemory mem = wrh.getWritable();
+ assertEquals(memCapacity, mem.getCapacity());
+ setGetPartialArraysWithOffsetTests(mem);
+ }
+ }
+
+ public static void setGetPartialArraysWithOffsetTests(WritableMemory mem) {
+ int items= 4;
+ boolean[] srcArray1 = {true, false, true, false};
+ boolean[] dstArray1 = new boolean[items];
+ mem.putBooleanArray(0, srcArray1, 2, items/2);
+ mem.getBooleanArray(0, dstArray1, 2, items/2);
+ for (int i = 2; i < items; i++) {
+ assertEquals(dstArray1[i], srcArray1[i]);
+ }
+
+ byte[] srcArray2 = { 1, -2, 3, -4 };
+ byte[] dstArray2 = new byte[items];
+ mem.putByteArray(0, srcArray2, 2, items/2);
+ mem.getByteArray(0, dstArray2, 2, items/2);
+ for (int i = 2; i < items; i++) {
+ assertEquals(dstArray2[i], srcArray2[i]);
+ }
+
+ char[] srcArray3 = { 'A', 'B', 'C', 'D' };
+ char[] dstArray3 = new char[items];
+ mem.putCharArray(0, srcArray3, 2, items/2);
+ mem.getCharArray(0, dstArray3, 2, items/2);
+ for (int i = 2; i < items; i++) {
+ assertEquals(dstArray3[i], srcArray3[i]);
+ }
+
+ double[] srcArray4 = { 1.0, -2.0, 3.0, -4.0 };
+ double[] dstArray4 = new double[items];
+ mem.putDoubleArray(0, srcArray4, 2, items/2);
+ mem.getDoubleArray(0, dstArray4, 2, items/2);
+ for (int i = 2; i < items; i++) {
+ assertEquals(dstArray4[i], srcArray4[i], 0.0);
+ }
+
+ float[] srcArray5 = { (float)1.0, (float)-2.0, (float)3.0, (float)-4.0 };
+ float[] dstArray5 = new float[items];
+ mem.putFloatArray(0, srcArray5, 2, items/2);
+ mem.getFloatArray(0, dstArray5, 2, items/2);
+ for (int i = 2; i < items; i++) {
+ assertEquals(dstArray5[i], srcArray5[i], 0.0);
+ }
+
+ int[] srcArray6 = { 1, -2, 3, -4 };
+ int[] dstArray6 = new int[items];
+ mem.putIntArray(0, srcArray6, 2, items/2);
+ mem.getIntArray(0, dstArray6, 2, items/2);
+ for (int i = 2; i < items; i++) {
+ assertEquals(dstArray6[i], srcArray6[i]);
+ }
+
+ long[] srcArray7 = { 1, -2, 3, -4 };
+ long[] dstArray7 = new long[items];
+ mem.putLongArray(0, srcArray7, 2, items/2);
+ mem.getLongArray(0, dstArray7, 2, items/2);
+ for (int i = 2; i < items; i++) {
+ assertEquals(dstArray7[i], srcArray7[i]);
+ }
+
+ short[] srcArray8 = { 1, -2, 3, -4 };
+ short[] dstArray8 = new short[items];
+ mem.putShortArray(0, srcArray8, 2, items/2);
+ mem.getShortArray(0, dstArray8, 2, items/2);
+ for (int i = 2; i < items; i++) {
+ assertEquals(dstArray8[i], srcArray8[i]);
+ }
+ }
+
+ @Test
+ public void checkSetClearIsBits() throws Exception {
+ int memCapacity = 8;
+ try (WritableHandle wrh = WritableMemory.allocateDirect(memCapacity)) {
+ WritableMemory mem = wrh.getWritable();
+ assertEquals(memCapacity, mem.getCapacity());
+ mem.clear();
+ setClearIsBitsTests(mem);
+ }
+ }
+
+ public static void setClearIsBitsTests(WritableMemory mem) {
+ //single bits
+ for (int i = 0; i < 8; i++) {
+ long bitMask = (1 << i);
+ long v = mem.getByte(0) & 0XFFL;
+ assertTrue(isAnyBitsClear(v, bitMask));
+ mem.setBits(0, (byte) bitMask);
+ v = mem.getByte(0) & 0XFFL;
+ assertTrue(isAnyBitsSet(v, bitMask));
+ mem.clearBits(0, (byte) bitMask);
+ v = mem.getByte(0) & 0XFFL;
+ assertTrue(isAnyBitsClear(v, bitMask));
+ }
+
+ //multiple bits
+ for (int i = 0; i < 7; i++) {
+ long bitMask1 = (1 << i);
+ long bitMask2 = (3 << i);
+ long v = mem.getByte(0) & 0XFFL;
+ assertTrue(isAnyBitsClear(v, bitMask1));
+ assertTrue(isAnyBitsClear(v, bitMask2));
+ mem.setBits(0, (byte) bitMask1); //set one bit
+ v = mem.getByte(0) & 0XFFL;
+ assertTrue(isAnyBitsSet(v, bitMask2));
+ assertTrue(isAnyBitsClear(v, bitMask2));
+ assertFalse(isAllBitsSet(v, bitMask2));
+ assertFalse(isAllBitsClear(v, bitMask2));
+ }
+ }
+
+ @Test
+ public void checkAtomicMethods() throws Exception {
+ int memCapacity = 8;
+ try (WritableHandle wrh = WritableMemory.allocateDirect(memCapacity)) {
+ WritableMemory mem = wrh.getWritable();
+ assertEquals(mem.getCapacity(), memCapacity);
+ atomicMethodTests(mem);
+ }
+ }
+
+ public static void atomicMethodTests(WritableMemory mem) {
+ mem.putLong(0, 500);
+ mem.getAndAddLong(0, 1);
+ assertEquals(mem.getLong(0), 501);
+
+ mem.putInt(0, 500);
+ boolean b = mem.compareAndSwapLong(0, 500, 501);
+ assertTrue(b);
+ assertEquals(mem.getLong(0), 501);
+
+ mem.putLong(0, 500);
+ long oldLong = mem.getAndSetLong(0, 501);
+ long newLong = mem.getLong(0);
+ assertEquals(oldLong, 500);
+ assertEquals(newLong, 501);
+ }
+
+ @Test
+ public void checkSetClearMemoryRegions() throws Exception {
+ int memCapacity = 64; //must be 64
+ try (WritableHandle wrh1 = WritableMemory.allocateDirect(memCapacity)) {
+ WritableMemory mem = wrh1.getWritable();
+
+ setClearMemoryRegionsTests(mem); //requires println enabled to visually check
+ for (int i = 0; i < memCapacity; i++) {
+ assertEquals(mem.getByte(i), 0);
+ }
+ }
+ }
+
+ //enable println stmts to visually check
+ public static void setClearMemoryRegionsTests(WritableMemory mem) {
+ int accessCapacity = (int)mem.getCapacity();
+
+ //define regions
+ int reg1Start = 0;
+ int reg1Len = 28;
+ int reg2Start = 28;
+ int reg2Len = 32;
+
+ //set region 1
+ byte b1 = 5;
+ mem.fill(reg1Start, reg1Len, b1);
+ for (int i = reg1Start; i < (reg1Len+reg1Start); i++) {
+ assertEquals(mem.getByte(i), b1);
+ }
+ //println(mem.toHexString("Region1 to 5", reg1Start, reg1Len));
+
+ //set region 2
+ byte b2 = 7;
+ mem.fill(reg2Start, reg2Len, b2);
+ //println(mem.toHexString("Fill", 0, (int)mem.getCapacity()));
+ for (int i = reg2Start; i < (reg2Len+reg2Start); i++) {
+ assertEquals(mem.getByte(i), b2);
+ }
+ //println(mem.toHexString("Region2 to 7", reg2Start, reg2Len));
+
+ //clear region 1
+ byte zeroByte = 0;
+ mem.clear(reg1Start, reg1Len);
+ for (int i = reg1Start; i < (reg1Len+reg1Start); i++) {
+ assertEquals(mem.getByte(i), zeroByte);
+ }
+ //println(mem.toHexString("Region1 cleared", reg1Start, reg1Len));
+
+ //clear region 2
+ mem.clear(reg2Start, reg2Len);
+ for (int i = reg2Start; i < (reg2Len+reg2Start); i++) {
+ assertEquals(mem.getByte(i), zeroByte);
+ }
+ //println(mem.toHexString("Region2 cleared", reg2Start, reg2Len));
+
+ //set all to ones
+ byte b4 = 127;
+ mem.fill(b4);
+ for (int i=0; i<accessCapacity; i++) {
+ assertEquals(mem.getByte(i), b4);
+ }
+ //println(mem.toHexString("Region1 + Region2 all ones", 0, accessCapacity));
+
+ //clear all
+ mem.clear();
+ for (int i = 0; i < accessCapacity; i++) {
+ assertEquals(mem.getByte(i), zeroByte);
+ }
+ //println(mem.toHexString("Region1 + Region2 cleared", 0, accessCapacity));
+ }
+
+ @Test
+ public void checkToHexStringAllMem() throws Exception {
+ int memCapacity = 48; //must be 48
+ try (WritableHandle wrh1 = WritableMemory.allocateDirect(memCapacity)) {
+ WritableMemory mem = wrh1.getWritable();
+ toHexStringAllMemTests(mem); //requires println enabled to visually check
+ }
+ }
+
+ //enable println to visually check
+ public static void toHexStringAllMemTests(WritableMemory mem) {
+ int memCapacity = (int)mem.getCapacity();
+
+ for (int i = 0; i < memCapacity; i++) {
+ mem.putByte(i, (byte)i);
+ }
+
+ //println(mem.toHexString("Check toHexString(0, 48) to integers", 0, memCapacity));
+ //println(mem.toHexString("Check toHexString(8, 40)", 8, 40));
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: "+this.getClass().getName());
+ }
+
+ /**
+ * @param s value to print
+ */
+ static void println(String s) {
+ //System.out.println(s); //disable here
+ }
+}
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/CopyMemoryOverlapTest.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/CopyMemoryOverlapTest.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/CopyMemoryOverlapTest.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import static org.testng.Assert.assertEquals;
+
+import org.apache.datasketches.memory.WritableHandle;
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+/**
+ * @author Lee Rhodes
+ */
+@SuppressWarnings("javadoc")
+public class CopyMemoryOverlapTest {
+
+ @Test
+ public void checkOverlapUsingMemory() throws Exception {
+ long copyLongs = 1 << 20;
+ double overlap = 0.5;
+ long start_mS = System.currentTimeMillis();
+
+ copyUsingDirectMemory(copyLongs, overlap, true);
+ long end1_mS = System.currentTimeMillis();
+
+ copyUsingDirectMemory(copyLongs, overlap, false);
+ long end2_mS = System.currentTimeMillis();
+
+ println("CopyUp Time Sec: " + ((end1_mS - start_mS)/1000.0));
+ println("CopyDn Time Sec: " + ((end2_mS - end1_mS)/1000.0));
+ }
+
+ @Test
+ public void checkOverlapUsingRegions() throws Exception {
+ long copyLongs = 1 << 20;
+ double overlap = 0.5;
+ long start_mS = System.currentTimeMillis();
+
+ copyUsingDirectRegions(copyLongs, overlap, true);
+ long end1_mS = System.currentTimeMillis();
+
+ copyUsingDirectRegions(copyLongs, overlap, false);
+ long end2_mS = System.currentTimeMillis();
+
+ println("CopyUp Time Sec: " + ((end1_mS - start_mS)/1000.0));
+ println("CopyDn Time Sec: " + ((end2_mS - end1_mS)/1000.0));
+ }
+
+ private static final void copyUsingDirectMemory(long copyLongs, double overlap, boolean copyUp) throws Exception {
+ println("Copy Using Direct Memory");
+ long overlapLongs = (long) (overlap * copyLongs);
+ long backingLongs = (2 * copyLongs) - overlapLongs;
+
+ long fromOffsetLongs;
+ long toOffsetLongs;
+ //long deltaLongs;
+
+ if (copyUp) {
+ fromOffsetLongs = 0;
+ toOffsetLongs = copyLongs - overlapLongs;
+ //deltaLongs = toOffsetLongs - fromOffsetLongs;
+ } else {
+ fromOffsetLongs = copyLongs - overlapLongs;
+ toOffsetLongs = 0;
+ //deltaLongs = toOffsetLongs - fromOffsetLongs;
+ }
+
+ long backingBytes = backingLongs << 3;
+ long copyBytes = copyLongs << 3;
+ long fromOffsetBytes = fromOffsetLongs << 3;
+ long toOffsetBytes = toOffsetLongs << 3;
+ //long deltaBytes = deltaLongs << 3;
+ println("Copy longs : " + copyLongs + "\t bytes: " + copyBytes);
+ println("Overlap : " + (overlap * 100.0) + "%");
+ println("CopyUp : " + copyUp);
+ println("Backing longs: " + backingLongs + "\t bytes: " + backingBytes);
+
+ try (WritableHandle backHandle = WritableMemory.allocateDirect(backingBytes)) {
+ WritableMemory backingMem = backHandle.getWritable();
+ fill(backingMem); //fill mem with 0 thru copyLongs -1
+ //listMem(backingMem, "Original");
+ backingMem.copyTo(fromOffsetBytes, backingMem, toOffsetBytes, copyBytes);
+ //listMem(backingMem, "After");
+ checkMemLongs(backingMem, fromOffsetLongs, toOffsetLongs, copyLongs);
+ }
+ println("");
+ }
+
+ private static final void copyUsingDirectRegions(long copyLongs, double overlap, boolean copyUp) throws Exception {
+ println("Copy Using Direct Memory");
+ long overlapLongs = (long) (overlap * copyLongs);
+ long backingLongs = (2 * copyLongs) - overlapLongs;
+
+ long fromOffsetLongs;
+ long toOffsetLongs;
+ //long deltaLongs;
+
+ if (copyUp) {
+ fromOffsetLongs = 0;
+ toOffsetLongs = copyLongs - overlapLongs;
+ //deltaLongs = toOffsetLongs - fromOffsetLongs;
+ } else {
+ fromOffsetLongs = copyLongs - overlapLongs;
+ toOffsetLongs = 0;
+ //deltaLongs = toOffsetLongs - fromOffsetLongs;
+ }
+
+ long backingBytes = backingLongs << 3;
+ long copyBytes = copyLongs << 3;
+ long fromOffsetBytes = fromOffsetLongs << 3;
+ long toOffsetBytes = toOffsetLongs << 3;
+ //long deltaBytes = deltaLongs << 3;
+ println("Copy longs : " + copyLongs + "\t bytes: " + copyBytes);
+ println("Overlap : " + (overlap * 100.0) + "%");
+ println("CopyUp : " + copyUp);
+ println("Backing longs: " + backingLongs + "\t bytes: " + backingBytes);
+
+ try (WritableHandle backHandle = WritableMemory.allocateDirect(backingBytes)) {
+ WritableMemory backingMem = backHandle.getWritable();
+ fill(backingMem); //fill mem with 0 thru copyLongs -1
+ //listMem(backingMem, "Original");
+ WritableMemory reg1 = backingMem.writableRegion(fromOffsetBytes, copyBytes);
+ WritableMemory reg2 = backingMem.writableRegion(toOffsetBytes, copyBytes);
+
+ reg1.copyTo(0, reg2, 0, copyBytes);
+ //listMem(backingMem, "After");
+ checkMemLongs(reg2, fromOffsetLongs, 0, copyLongs);
+ }
+ println("");
+ }
+
+ private static final void fill(WritableMemory wmem) {
+ long longs = wmem.getCapacity() >>> 3;
+ for (long i = 0; i < longs; i++) { wmem.putLong(i << 3, i); } //fill with 0 .. (longs - 1)
+ //checkMemLongs(wmem, 0L, 0L, longs);
+ }
+
+ private static final void checkMemLongs(Memory mem, long fromOffsetLongs, long toOffsetLongs, long copyLongs) {
+ for (long i = 0; i < copyLongs; i++) {
+ long memVal = mem.getLong((toOffsetLongs + i) << 3);
+ assertEquals(memVal, fromOffsetLongs + i);
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private static final void listMem(Memory mem, String comment) {
+ println(comment);
+ println("Idx\tValue");
+ long longs = mem.getCapacity() >>> 3;
+ for (long i = 0; i < longs; i++) {
+ println(i + "\t" + mem.getLong(i << 3));
+ }
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: "+this.getClass().getName());
+ }
+
+ /**
+ * @param s value to print
+ */
+ static void println(String s) {
+ //System.out.println(s); //disable here
+ }
+}
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/CopyMemoryTest.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/CopyMemoryTest.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/CopyMemoryTest.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import static org.apache.datasketches.memory.internal.Util.UNSAFE_COPY_THRESHOLD_BYTES;
+import static org.testng.Assert.assertEquals;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.apache.datasketches.memory.WritableHandle;
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+@SuppressWarnings("javadoc")
+public class CopyMemoryTest {
+
+ @Test
+ public void heapWSource() {
+ int k1 = 1 << 20; //longs
+ int k2 = 2 * k1;
+ WritableMemory srcMem = genMem(k1, false); //!empty
+ //println(srcMem.toHexString("src: ", 0, k1 << 3));
+ WritableMemory dstMem = genMem(k2, true);
+ srcMem.copyTo(0, dstMem, k1 << 3, k1 << 3);
+ //println(dstMem.toHexString("dst: ", 0, k2 << 3));
+ check(dstMem, k1, k1, 1);
+ }
+
+ @Test
+ public void heapROSource() {
+ int k1 = 1 << 20; //longs
+ int k2 = 2 * k1;
+ Memory srcMem = genMem(k1, false); //!empty
+ WritableMemory dstMem = genMem(k2, true);
+ srcMem.copyTo(0, dstMem, k1 << 3, k1 << 3);
+ check(dstMem, k1, k1, 1);
+ }
+
+ @Test
+ public void directWSource() throws Exception {
+ int k1 = 1 << 20; //longs
+ int k2 = 2 * k1;
+ try (WritableHandle wrh = genWRH(k1, false)) {
+ WritableMemory srcMem = wrh.getWritable();
+ WritableMemory dstMem = genMem(k2, true);
+ srcMem.copyTo(0, dstMem, k1 << 3, k1 << 3);
+ check(dstMem, k1, k1, 1);
+ }
+ }
+
+ @Test
+ public void directROSource() throws Exception {
+ int k1 = 1 << 20; //longs
+ int k2 = 2 * k1;
+ try (WritableHandle wrh = genWRH(k1, false)) {
+ Memory srcMem = wrh.get();
+ WritableMemory dstMem = genMem(k2, true);
+ srcMem.copyTo(0, dstMem, k1 << 3, k1 << 3);
+ check(dstMem, k1, k1, 1);
+ }
+ }
+
+ @Test
+ public void heapWSrcRegion() {
+ int k1 = 1 << 20; //longs
+ //gen baseMem of k1 longs w data
+ WritableMemory baseMem = genMem(k1, false); //!empty
+ //gen src region of k1/2 longs, off= k1/2
+ WritableMemory srcReg = baseMem.writableRegion((k1/2) << 3, (k1/2) << 3);
+ WritableMemory dstMem = genMem(2 * k1, true); //empty
+ srcReg.copyTo(0, dstMem, k1 << 3, (k1/2) << 3);
+ //println(dstMem.toHexString("dstMem: ", k1 << 3, (k1/2) << 3));
+ check(dstMem, k1, k1/2, (k1/2) + 1);
+ }
+
+ @Test
+ public void heapROSrcRegion() {
+ int k1 = 1 << 20; //longs
+ //gen baseMem of k1 longs w data
+ WritableMemory baseMem = genMem(k1, false); //!empty
+ //gen src region of k1/2 longs, off= k1/2
+ Memory srcReg = baseMem.region((k1/2) << 3, (k1/2) << 3);
+ WritableMemory dstMem = genMem(2 * k1, true); //empty
+ srcReg.copyTo(0, dstMem, k1 << 3, (k1/2) << 3);
+ check(dstMem, k1, k1/2, (k1/2) + 1);
+ }
+
+ @Test
+ public void directROSrcRegion() throws Exception {
+ int k1 = 1 << 20; //longs
+ //gen baseMem of k1 longs w data, direct
+ try (WritableHandle wrh = genWRH(k1, false)) {
+ Memory baseMem = wrh.get();
+ //gen src region of k1/2 longs, off= k1/2
+ Memory srcReg = baseMem.region((k1/2) << 3, (k1/2) << 3);
+ WritableMemory dstMem = genMem(2 * k1, true); //empty
+ srcReg.copyTo(0, dstMem, k1 << 3, (k1/2) << 3);
+ check(dstMem, k1, k1/2, (k1/2) + 1);
+ }
+ }
+
+ @Test
+ public void testOverlappingCopyLeftToRight() {
+ byte[] bytes = new byte[((UNSAFE_COPY_THRESHOLD_BYTES * 5) / 2) + 1];
+ ThreadLocalRandom.current().nextBytes(bytes);
+ byte[] referenceBytes = bytes.clone();
+ Memory referenceMem = Memory.wrap(referenceBytes);
+ WritableMemory mem = WritableMemory.writableWrap(bytes);
+ long copyLen = UNSAFE_COPY_THRESHOLD_BYTES * 2;
+ mem.copyTo(0, mem, UNSAFE_COPY_THRESHOLD_BYTES / 2, copyLen);
+ Assert.assertEquals(0, mem.compareTo(UNSAFE_COPY_THRESHOLD_BYTES / 2, copyLen, referenceMem, 0,
+ copyLen));
+ }
+
+ @Test
+ public void testOverlappingCopyRightToLeft() {
+ byte[] bytes = new byte[((UNSAFE_COPY_THRESHOLD_BYTES * 5) / 2) + 1];
+ ThreadLocalRandom.current().nextBytes(bytes);
+ byte[] referenceBytes = bytes.clone();
+ Memory referenceMem = Memory.wrap(referenceBytes);
+ WritableMemory mem = WritableMemory.writableWrap(bytes);
+ long copyLen = UNSAFE_COPY_THRESHOLD_BYTES * 2;
+ mem.copyTo(UNSAFE_COPY_THRESHOLD_BYTES / 2, mem, 0, copyLen);
+ Assert.assertEquals(0, mem.compareTo(0, copyLen, referenceMem, UNSAFE_COPY_THRESHOLD_BYTES / 2,
+ copyLen));
+ }
+
+ private static void check(Memory mem, int offsetLongs, int lengthLongs, int startValue) {
+ int offBytes = offsetLongs << 3;
+ for (long i = 0; i < lengthLongs; i++) {
+ assertEquals(mem.getLong(offBytes + (i << 3)), i + startValue);
+ }
+ }
+
+ private static WritableHandle genWRH(int longs, boolean empty) {
+ WritableHandle wrh = WritableMemory.allocateDirect(longs << 3);
+ WritableMemory mem = wrh.getWritable();
+ if (empty) {
+ mem.clear();
+ } else {
+ for (int i = 0; i < longs; i++) { mem.putLong(i << 3, i + 1); }
+ }
+ return wrh;
+ }
+
+
+ private static WritableMemory genMem(int longs, boolean empty) {
+ WritableMemory mem = WritableMemory.allocate(longs << 3);
+ if (!empty) {
+ for (int i = 0; i < longs; i++) { mem.putLong(i << 3, i + 1); }
+ }
+ return mem;
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: "+this.getClass().getName());
+ }
+
+ /**
+ * @param s value to print
+ */
+ static void println(String s) {
+ //System.out.println(s); //disable here
+ }
+}
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/DruidIssue11544Test.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/DruidIssue11544Test.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/DruidIssue11544Test.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.MemoryRequestServer;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+/**
+ * The original design provided the MemoryRequestServer callback only for Memory segments allocated via
+ * <i>WritableMemory.allocateDirect(...)</i> calls. Memory segments allocated via
+ * <i>WritableMemory.wrap(ByteBuffer)</i> did not have this capability. This was a major oversight since
+ * all off-heap memory in Druid is allocated using ByteBuffers! It is unusual that no one has
+ * uncovered this until August 2021. Nonetheless, the fix involves instrumenting all the paths involved
+ * in providing this callback mechanism for wrapped ByteBuffers.
+ *
+ * This issues was first identified in Druid Issue #11544 and then posted as DataSketches-java Issue #358.
+ * But the actual source of the problem was in Memory.
+ *
+ * This test mimics the Druid issue but at a much smaller scale.
+ *
+ * @author Lee Rhodes
+ *
+ */
+@SuppressWarnings("javadoc")
+public class DruidIssue11544Test {
+
+ @Test
+ public void withByteBuffer() {
+ int initialLongs = 1000;
+ int size1 = initialLongs * 8;
+
+ //Start with a ByteBuffer
+ ByteBuffer bb = ByteBuffer.allocateDirect(size1);
+ bb.order(ByteOrder.nativeOrder());
+
+ //Wrap bb into WritableMemory
+ WritableMemory mem1 = WritableMemory.writableWrap(bb);
+ assertTrue(mem1.isDirect()); //confirm mem1 is off-heap
+
+ //Acquire the DefaultMemoryRequestServer
+ //NOTE: it is a policy decision to allow the DefaultMemoryServer to be set as a default.
+ // It might be set to null. So we need to check what the current policy is.
+ MemoryRequestServer svr = mem1.getMemoryRequestServer();
+ if (svr == null) {
+ svr = new DefaultMemoryRequestServer();
+ }
+ assertNotNull(svr);
+
+ //Request Bigger Memory
+ int size2 = size1 * 2;
+ WritableMemory mem2 = svr.request(mem1, size2);
+
+ //Confirm that mem2 is on the heap (the default) and 2X size1
+ assertFalse(mem2.isDirect());
+ assertEquals(mem2.getCapacity(), size2);
+
+ //Move data to new memory
+ mem1.copyTo(0, mem2, 0, size1);
+
+ //Prepare to request deallocation
+ //In the DefaultMemoryRequestServer, this is a no-op, so nothing is actually deallocated.
+ svr.requestClose(mem1, mem2);
+ assertTrue(mem1.isValid());
+ assertTrue(mem2.isValid());
+
+ //Now we are on the heap and need to grow again:
+ int size3 = size2 * 2;
+ WritableMemory mem3 = svr.request(mem2, size3);
+
+ //Confirm that mem3 is still on the heap and 2X of size2
+ assertFalse(mem3.isDirect());
+ assertEquals(mem3.getCapacity(), size3);
+
+ //Move data to new memory
+ mem2.copyTo(0, mem3, 0, size2);
+
+ //Prepare to request deallocation
+
+ svr.requestClose(mem2, mem3); //No-op
+ assertTrue(mem2.isValid());
+ assertTrue(mem3.isValid());
+ }
+
+}
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/ExampleMemoryRequestServerTest.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/ExampleMemoryRequestServerTest.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/ExampleMemoryRequestServerTest.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import static org.testng.Assert.assertFalse;
+
+import java.nio.ByteOrder;
+import java.util.IdentityHashMap;
+
+import org.apache.datasketches.memory.MemoryRequestServer;
+import org.apache.datasketches.memory.WritableHandle;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+/**
+ * Examples of how to use the MemoryRequestServer with a memory hungry client.
+ * @author Lee Rhodes
+ */
+@SuppressWarnings("javadoc")
+public class ExampleMemoryRequestServerTest {
+
+ /**
+ * This version is without a TWR block.all of the memory allocations are done through the MemoryRequestServer
+ * and each is closed by the MemoryClient when it is done with each.
+ */
+ @SuppressWarnings("resource")
+ @Test
+ public void checkExampleMemoryRequestServer1() throws Exception {
+ int bytes = 8;
+ ExampleMemoryRequestServer svr = new ExampleMemoryRequestServer();
+ try (WritableHandle wh = WritableMemory.allocateDirect(8)) {
+ WritableMemory memStart = wh.getWritable();
+ WritableMemory wMem = svr.request(memStart, bytes);
+ MemoryClient client = new MemoryClient(wMem);
+ client.process();
+ svr.cleanup();
+ }
+ }
+
+ /**
+ * In this version the first memory allocation is done up front in a TWR block.
+ * And then the MemoryClient allocates new memories as needed, which are then closed
+ * by the MemoryClient when it is done with the new memory allocations.
+ * The initial allocation stays open until the end where it is closed at the end of the
+ * TWR scope.
+ * @throws Exception
+ */
+ @Test
+ public void checkExampleMemoryRequestServer2() throws Exception {
+ int bytes = 8;
+ ExampleMemoryRequestServer svr = new ExampleMemoryRequestServer();
+ try (WritableHandle handle = WritableMemory.allocateDirect(bytes, ByteOrder.nativeOrder(), svr)) {
+ WritableMemory memStart = handle.getWritable();
+ MemoryClient client = new MemoryClient(memStart);
+ client.process();
+ svr.cleanup(); //just to be sure all are closed.
+ }
+ }
+
+ @SuppressWarnings("resource")
+ @Test(expectedExceptions = IllegalArgumentException.class)
+ public void checkZeroCapacity() throws Exception {
+ ExampleMemoryRequestServer svr = new ExampleMemoryRequestServer();
+ try (WritableHandle wh = WritableMemory.allocateDirect(0, ByteOrder.nativeOrder(), svr)) {
+
+ }
+ }
+
+ /**
+ * This little client is never happy with how much memory it has been allocated and keeps
+ * requesting for more. When it does ask for more, it must copy its old data into the new
+ * memory, release the prior memory, and then continue working from there.
+ *
+ * <p>In reality, these memory requests should be quite rare.</p>
+ */
+ static class MemoryClient {
+ WritableMemory smallMem;
+ MemoryRequestServer svr;
+
+ MemoryClient(WritableMemory memStart) {
+ smallMem = memStart;
+ svr = memStart.getMemoryRequestServer();
+ }
+
+ void process() {
+ long cap1 = smallMem.getCapacity();
+ smallMem.fill((byte) 1); //fill it, but not big enough
+ println(smallMem.toHexString("Small", 0, (int)cap1));
+
+ WritableMemory bigMem = svr.request(smallMem, 2 * cap1); //get bigger mem
+ long cap2 = bigMem.getCapacity();
+ smallMem.copyTo(0, bigMem, 0, cap1); //copy data from small to big
+ svr.requestClose(smallMem, bigMem); //done with smallMem, release it
+
+ bigMem.fill(cap1, cap1, (byte) 2); //fill the rest of bigMem, still not big enough
+ println(bigMem.toHexString("Big", 0, (int)cap2));
+
+ WritableMemory giantMem = svr.request(bigMem, 2 * cap2); //get giant mem
+ long cap3 = giantMem.getCapacity();
+ bigMem.copyTo(0, giantMem, 0, cap2); //copy data from small to big
+ svr.requestClose(bigMem, giantMem); //done with bigMem, release it
+
+ giantMem.fill(cap2, cap2, (byte) 3); //fill the rest of giantMem
+ println(giantMem.toHexString("Giant", 0, (int)cap3));
+ svr.requestClose(giantMem, null); //done with giantMem, release it
+ }
+ }
+
+ /**
+ * This example MemoryRequestServer is simplistic but demonstrates one of many ways to
+ * possibly manage the continuous requests for larger memory and to track the associations between
+ * handles and their associated memory.
+ */
+ public static class ExampleMemoryRequestServer implements MemoryRequestServer {
+ IdentityHashMap<WritableMemory, WritableHandle> map = new IdentityHashMap<>();
+
+ @SuppressWarnings("resource")
+ @Override
+ public WritableMemory request(WritableMemory currentWMem, long capacityBytes) {
+ ByteOrder order = currentWMem.getTypeByteOrder();
+ WritableHandle handle = WritableMemory.allocateDirect(capacityBytes, order, this);
+ WritableMemory wmem = handle.getWritable();
+ map.put(wmem, handle); //We track the newly allocated memory and its handle.
+ return wmem;
+ }
+
+ @SuppressWarnings("resource")
+ @Override
+ //here we actually release it, in reality it might be a lot more complex.
+ public void requestClose(WritableMemory memToRelease, WritableMemory newMemory) {
+ WritableHandle handle = map.get(memToRelease);
+ if (handle != null && handle.getWritable() == memToRelease) {
+ try {
+ handle.close();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public void cleanup() {
+ map.forEach((k,v) -> {
+ assertFalse(k.isValid()); //all entries in the map should be invalid
+ try {
+ v.close();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: "+this.getClass().getName());
+ }
+
+ /**
+ * @param s value to print
+ */
+ static void println(String s) {
+ //System.out.println(s); //disable here
+ }
+}
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/IgnoredArrayOverflowTest.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/IgnoredArrayOverflowTest.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/IgnoredArrayOverflowTest.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import org.apache.datasketches.memory.WritableHandle;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+@SuppressWarnings("javadoc")
+public class IgnoredArrayOverflowTest {
+
+ private WritableHandle h;
+ private WritableMemory memory;
+ private static final long MAX_SIZE = (1L << 10); // use 1L << 31 to test int overrange
+
+ @BeforeClass
+ public void allocate() {
+ h = WritableMemory.allocateDirect(MAX_SIZE);
+ memory = h.getWritable();
+ }
+
+ @AfterClass
+ public void close() throws Exception {
+ h.close();
+ }
+
+ @Test
+ public void testCharArray() {
+ int size = (int) (memory.getCapacity() / 2);
+ char[] array = new char[size];
+ memory.getCharArray(0, array, 0, size);
+ memory.asBuffer().getCharArray(array, 0, size);
+ memory.putCharArray(0, array, 0, size);
+ memory.asWritableBuffer().putCharArray(array, 0, size);
+ }
+
+ @Test
+ public void testShortArray() {
+ int size = (int) (memory.getCapacity() / 2);
+ short[] array = new short[size];
+ memory.getShortArray(0, array, 0, size);
+ memory.asBuffer().getShortArray(array, 0, size);
+ memory.putShortArray(0, array, 0, size);
+ memory.asWritableBuffer().putShortArray(array, 0, size);
+ }
+
+ @Test
+ public void testIntArray() {
+ int size = (int) (memory.getCapacity() / 4);
+ int[] array = new int[size];
+ memory.getIntArray(0, array, 0, size);
+ memory.asBuffer().getIntArray(array, 0, size);
+ memory.putIntArray(0, array, 0, size);
+ memory.asWritableBuffer().putIntArray(array, 0, size);
+ }
+
+ @Test
+ public void testFloatArray() {
+ int size = (int) (memory.getCapacity() / 4);
+ float[] array = new float[size];
+ memory.getFloatArray(0, array, 0, size);
+ memory.asBuffer().getFloatArray(array, 0, size);
+ memory.putFloatArray(0, array, 0, size);
+ memory.asWritableBuffer().putFloatArray(array, 0, size);
+ }
+
+ @Test
+ public void testLongArray() {
+ int size = (int) (memory.getCapacity() / 8);
+ long[] array = new long[size];
+ memory.getLongArray(0, array, 0, size);
+ memory.asBuffer().getLongArray(array, 0, size);
+ memory.putLongArray(0, array, 0, size);
+ memory.asWritableBuffer().putLongArray(array, 0, size);
+ }
+
+ @Test
+ public void testDoubleArray() {
+ int size = (int) (memory.getCapacity() / 8);
+ double[] array = new double[size];
+ memory.getDoubleArray(0, array, 0, size);
+ memory.asBuffer().getDoubleArray(array, 0, size);
+ memory.putDoubleArray(0, array, 0, size);
+ memory.asWritableBuffer().putDoubleArray(array, 0, size);
+ }
+}
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/IsValidUtf8TestUtil.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/IsValidUtf8TestUtil.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/IsValidUtf8TestUtil.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+/**
+ * Stripped down version of
+ * https://github.com/protocolbuffers/protobuf/blob/master/java/core/src/test/java/com/google/protobuf/IsValidUtf8TestUtil.java
+ *
+ * <p>Copyright 2008 Google Inc. All rights reserved.
+ * https://developers.google.com/protocol-buffers/
+ * See LICENSE.
+ */
+public class IsValidUtf8TestUtil {
+
+ // 128 - [chars 0x0000 to 0x007f]
+ static final long ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS = (0x007f - 0x0000) + 1;
+
+ // 128
+ static final long EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT = ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS;
+
+ // 1920 [chars 0x0080 to 0x07FF]
+ static final long TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS = (0x07FF - 0x0080) + 1;
+
+ // 18,304
+ static final long EXPECTED_TWO_BYTE_ROUNDTRIPPABLE_COUNT =
+ // Both bytes are one byte characters
+ (long) Math.pow(EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT, 2)
+ // The possible number of two byte characters
+ + TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS;
+
+ // 2048
+ static final long THREE_BYTE_SURROGATES = 2 * 1024;
+
+ // 61,440 [chars 0x0800 to 0xFFFF, minus surrogates]
+ static final long THREE_BYTE_ROUNDTRIPPABLE_CHARACTERS =
+ ((0xFFFF - 0x0800) + 1) - THREE_BYTE_SURROGATES;
+
+ // 2,650,112
+ static final long EXPECTED_THREE_BYTE_ROUNDTRIPPABLE_COUNT =
+ // All one byte characters
+ (long) Math.pow(EXPECTED_ONE_BYTE_ROUNDTRIPPABLE_COUNT, 3)
+ // One two byte character and a one byte character
+ + (2 * TWO_BYTE_ROUNDTRIPPABLE_CHARACTERS * ONE_BYTE_ROUNDTRIPPABLE_CHARACTERS)
+ // Three byte characters
+ + THREE_BYTE_ROUNDTRIPPABLE_CHARACTERS;
+
+}
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/LeafImplTest.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/LeafImplTest.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/LeafImplTest.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import static org.apache.datasketches.memory.internal.Util.NON_NATIVE_BYTE_ORDER;
+import static org.apache.datasketches.memory.internal.Util.otherByteOrder;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.apache.datasketches.memory.MemoryRequestServer;
+import org.apache.datasketches.memory.WritableBuffer;
+import org.apache.datasketches.memory.WritableHandle;
+import org.apache.datasketches.memory.WritableMapHandle;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+/**
+ * @author Lee Rhodes
+ */
+@SuppressWarnings("javadoc")
+public class LeafImplTest {
+ private static final ByteOrder NBO = ByteOrder.nativeOrder();
+ private static final ByteOrder NNBO = NON_NATIVE_BYTE_ORDER;
+ private static final MemoryRequestServer dummyMemReqSvr = new DummyMemoryRequestServer();
+
+ static class DummyMemoryRequestServer implements MemoryRequestServer {
+ @Override
+ public WritableMemory request(WritableMemory currentWMem, long capacityBytes) { return null; }
+ @Override
+ public void requestClose(WritableMemory memToClose, WritableMemory newMemory) { }
+ }
+
+ @Test
+ public void checkDirectLeafs() throws Exception {
+ long off = 0;
+ long cap = 128;
+ // Off Heap, Native order, No ByteBuffer, has MemReqSvr
+ try (WritableHandle wdh = WritableMemory.allocateDirect(cap, NBO, dummyMemReqSvr)) {
+ WritableMemory memNO = wdh.getWritable();
+ memNO.putShort(0, (short) 1);
+ assertNull(ReflectUtil.getUnsafeObject(memNO));
+ assertTrue(memNO.isDirect());
+ checkCombinations(memNO, off, cap, memNO.isDirect(), NBO, false, true);
+ }
+ // Off Heap, Non Native order, No ByteBuffer, has MemReqSvr
+ try (WritableHandle wdh = WritableMemory.allocateDirect(cap, NNBO, dummyMemReqSvr)) {
+ WritableMemory memNNO = wdh.getWritable();
+ memNNO.putShort(0, (short) 1);
+ assertNull(ReflectUtil.getUnsafeObject(memNNO));
+ assertTrue(memNNO.isDirect());
+ checkCombinations(memNNO, off, cap, memNNO.isDirect(), NNBO, false, true);
+ }
+ }
+
+ @Test
+ public void checkByteBufferLeafs() {
+ long off = 0;
+ long cap = 128;
+ //BB on heap, native order, has ByteBuffer, has MemReqSvr
+ ByteBuffer bb = ByteBuffer.allocate((int)cap);
+ bb.order(NBO);
+ bb.putShort(0, (short) 1);
+ WritableMemory mem = WritableMemory.writableWrap(bb, NBO, dummyMemReqSvr);
+ assertEquals(bb.isDirect(), mem.isDirect());
+ assertNotNull(ReflectUtil.getUnsafeObject(mem));
+ checkCombinations(mem, off, cap, mem.isDirect(), mem.getTypeByteOrder(), true, true);
+
+ //BB off heap, native order, has ByteBuffer, has MemReqSvr
+ ByteBuffer dbb = ByteBuffer.allocateDirect((int)cap);
+ dbb.order(NBO);
+ dbb.putShort(0, (short) 1);
+ mem = WritableMemory.writableWrap(dbb, NBO, dummyMemReqSvr);
+ assertEquals(dbb.isDirect(), mem.isDirect());
+ assertNull(ReflectUtil.getUnsafeObject(mem));
+ checkCombinations(mem, off, cap, mem.isDirect(), mem.getTypeByteOrder(), true, true);
+
+ //BB on heap, non native order, has ByteBuffer, has MemReqSvr
+ bb = ByteBuffer.allocate((int)cap);
+ bb.order(NNBO);
+ bb.putShort(0, (short) 1);
+ mem = WritableMemory.writableWrap(bb, NNBO, dummyMemReqSvr);
+ assertEquals(bb.isDirect(), mem.isDirect());
+ assertNotNull(ReflectUtil.getUnsafeObject(mem));
+ checkCombinations(mem, off, cap, mem.isDirect(), mem.getTypeByteOrder(), true, true);
+
+ //BB off heap, non native order, has ByteBuffer, has MemReqSvr
+ dbb = ByteBuffer.allocateDirect((int)cap);
+ dbb.order(NNBO);
+ dbb.putShort(0, (short) 1);
+ mem = WritableMemory.writableWrap(dbb, NNBO, dummyMemReqSvr);
+ assertEquals(dbb.isDirect(), mem.isDirect());
+ assertNull(ReflectUtil.getUnsafeObject(mem));
+ checkCombinations(mem, off, cap, mem.isDirect(), mem.getTypeByteOrder(), true, true);
+ }
+
+ @Test
+ public void checkMapLeafs() throws Exception {
+ long off = 0;
+ long cap = 128;
+ File file = new File("TestFile2.bin");
+ if (file.exists()) {
+ try {
+ java.nio.file.Files.delete(file.toPath());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ assertTrue(file.createNewFile());
+ assertTrue(file.setWritable(true, false)); //writable=true, ownerOnly=false
+ assertTrue(file.isFile());
+ file.deleteOnExit(); //comment out if you want to examine the file.
+ // Off Heap, Native order, No ByteBuffer, No MemReqSvr
+ try (WritableMapHandle wmh = WritableMemory.writableMap(file, off, cap, NBO)) {
+ WritableMemory memNO = wmh.getWritable();
+ memNO.putShort(0, (short) 1);
+ assertNull(ReflectUtil.getUnsafeObject(memNO));
+ assertTrue(memNO.isDirect());
+ checkCombinations(memNO, off, cap, memNO.isDirect(), NBO, false, false);
+ }
+ // Off heap, Non Native order, No ByteBuffer, no MemReqSvr
+ try (WritableMapHandle wmh = WritableMemory.writableMap(file, off, cap, NNBO)) {
+ WritableMemory memNNO = wmh.getWritable();
+ memNNO.putShort(0, (short) 1);
+ assertNull(ReflectUtil.getUnsafeObject(memNNO));
+ assertTrue(memNNO.isDirect());
+ checkCombinations(memNNO, off, cap, memNNO.isDirect(), NNBO, false, false);
+ }
+ }
+
+ @Test
+ public void checkHeapLeafs() {
+ long off = 0;
+ long cap = 128;
+ // On Heap, Native order, No ByteBuffer, No MemReqSvr
+ WritableMemory memNO = WritableMemory.allocate((int)cap); //assumes NBO
+ memNO.putShort(0, (short) 1);
+ assertNotNull(ReflectUtil.getUnsafeObject(memNO));
+ assertFalse(memNO.isDirect());
+ checkCombinations(memNO, off, cap, memNO.isDirect(), NBO, false, false);
+ // On Heap, Non-native order, No ByteBuffer, No MemReqSvr
+ WritableMemory memNNO = WritableMemory.allocate((int)cap, NNBO);
+ memNNO.putShort(0, (short) 1);
+ assertNotNull(ReflectUtil.getUnsafeObject(memNNO));
+ assertFalse(memNNO.isDirect());
+ checkCombinations(memNNO, off, cap, memNNO.isDirect(), NNBO, false, false);
+ }
+
+ private static void checkCombinations(WritableMemory mem, long off, long cap,
+ boolean direct, ByteOrder bo, boolean hasByteBuffer, boolean hasMemReqSvr) {
+ ByteOrder oo = otherByteOrder(bo);
+
+ assertEquals(mem.writableRegion(off, cap, bo).getShort(0), 1);
+ assertEquals(mem.writableRegion(off, cap, oo).getShort(0), 256);
+
+ assertEquals(mem.asWritableBuffer(bo).getShort(0), 1);
+ assertEquals(mem.asWritableBuffer(oo).getShort(0), 256);
+
+ ByteBuffer bb = mem.getByteBuffer();
+ assertTrue( hasByteBuffer ? bb != null : bb == null);
+
+ assertTrue(mem.getTypeByteOrder() == bo);
+
+ if (hasMemReqSvr) { assertTrue(mem.getMemoryRequestServer() instanceof DummyMemoryRequestServer); }
+
+ Object obj = ReflectUtil.getUnsafeObject(mem);
+ if (direct) {
+ assertTrue(mem.isDirect());
+ assertNull(obj);
+ } else {
+ assertFalse(mem.isDirect());
+ assertNotNull(obj);
+ }
+
+ assertTrue(mem.isValid() == true);
+
+ WritableBuffer buf = mem.asWritableBuffer();
+
+ assertEquals(buf.writableRegion(off, cap, bo).getShort(0), 1);
+ assertEquals(buf.writableRegion(off, cap, oo).getShort(0), 256);
+ assertEquals(buf.writableDuplicate(bo).getShort(0), 1);
+ assertEquals(buf.writableDuplicate(oo).getShort(0), 256);
+
+ bb = buf.getByteBuffer();
+ assertTrue(hasByteBuffer ? bb != null : bb == null);
+
+ assertTrue(buf.getTypeByteOrder() == bo);
+
+ if (hasMemReqSvr) { assertTrue(buf.getMemoryRequestServer() instanceof DummyMemoryRequestServer); }
+
+ obj = ReflectUtil.getUnsafeObject(buf);
+ if (direct) {
+ assertTrue(buf.isDirect());
+ assertNull(obj);
+ } else {
+ assertFalse(buf.isDirect());
+ assertNotNull(obj);
+ }
+
+ assertTrue(buf.isValid() == true);
+
+ WritableMemory nnMem = mem.writableRegion(off, cap, oo);
+
+ assertEquals(nnMem.writableRegion(off, cap, bo).getShort(0), 1);
+ assertEquals(nnMem.writableRegion(off, cap, oo).getShort(0), 256);
+ assertEquals(nnMem.asWritableBuffer(bo).getShort(0), 1);
+ assertEquals(nnMem.asWritableBuffer(oo).getShort(0), 256);
+
+ bb = nnMem.getByteBuffer();
+ assertTrue( hasByteBuffer ? bb != null : bb == null);
+
+ assertTrue(nnMem.getTypeByteOrder() == oo);
+
+ if (hasMemReqSvr) { assertTrue(nnMem.getMemoryRequestServer() instanceof DummyMemoryRequestServer); }
+
+ obj = ReflectUtil.getUnsafeObject(nnMem);
+ if (direct) {
+ assertTrue(nnMem.isDirect());
+ assertNull(obj);
+ } else {
+ assertFalse(nnMem.isDirect());
+ assertNotNull(obj);
+ }
+
+ assertTrue(nnMem.isValid() == true);
+
+ WritableBuffer nnBuf = mem.asWritableBuffer(oo);
+
+ assertEquals(nnBuf.writableRegion(off, cap, bo).getShort(0), 1);
+ assertEquals(nnBuf.writableRegion(off, cap, oo).getShort(0), 256);
+ assertEquals(nnBuf.writableDuplicate(bo).getShort(0), 1);
+ assertEquals(nnBuf.writableDuplicate(oo).getShort(0), 256);
+
+ bb = nnBuf.getByteBuffer();
+ assertTrue( hasByteBuffer ? bb != null : bb == null);
+
+ assertTrue(nnBuf.getTypeByteOrder() == oo);
+
+ if (hasMemReqSvr) { assertTrue(nnBuf.getMemoryRequestServer() instanceof DummyMemoryRequestServer); }
+
+ obj = ReflectUtil.getUnsafeObject(nnBuf);
+ if (direct) {
+ assertTrue(nnBuf.isDirect());
+ assertNull(obj);
+ } else {
+ assertFalse(nnBuf.isDirect());
+ assertNotNull(obj);
+ }
+
+ assertTrue(nnBuf.isValid() == true);
+ }
+
+}
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryBoundaryCheckTest.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryBoundaryCheckTest.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryBoundaryCheckTest.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import org.apache.datasketches.memory.WritableBuffer;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+@SuppressWarnings("javadoc")
+public class MemoryBoundaryCheckTest {
+
+ private final WritableBuffer writableBuffer = WritableMemory.allocate(8).asWritableBuffer();
+
+ @Test
+ public void testGetBoolean() {
+ writableBuffer.getBoolean(7);
+ try {
+ writableBuffer.getBoolean(8);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testPutBoolean() {
+ writableBuffer.putBoolean(7, true);
+ try {
+ writableBuffer.putBoolean(8, true);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testGetByte() {
+ writableBuffer.getByte(7);
+ try {
+ writableBuffer.getByte(8);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testPutByte() {
+ writableBuffer.putByte(7, (byte) 1);
+ try {
+ writableBuffer.putByte(8, (byte) 1);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testGetChar() {
+ writableBuffer.getChar(6);
+ try {
+ writableBuffer.getChar(7);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testPutChar() {
+ writableBuffer.putChar(6, 'a');
+ try {
+ writableBuffer.putChar(7, 'a');
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testGetShort() {
+ writableBuffer.getShort(6);
+ try {
+ writableBuffer.getShort(7);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testPutShort() {
+ writableBuffer.putShort(6, (short) 1);
+ try {
+ writableBuffer.putShort(7, (short) 1);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testGetInt() {
+ writableBuffer.getInt(4);
+ try {
+ writableBuffer.getInt(5);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testPutInt() {
+ writableBuffer.putInt(4, 1);
+ try {
+ writableBuffer.putInt(5, 1);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testGetFloat() {
+ writableBuffer.getFloat(4);
+ try {
+ writableBuffer.getFloat(5);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testPutFloat() {
+ writableBuffer.putFloat(4, 1f);
+ try {
+ writableBuffer.putFloat(5, 1f);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testGetLong() {
+ writableBuffer.getLong(0);
+ try {
+ writableBuffer.getLong(1);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testPutLong() {
+ writableBuffer.putLong(0, 1L);
+ try {
+ writableBuffer.putLong(1, 1L);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testGetDouble() {
+ writableBuffer.getDouble(0);
+ try {
+ writableBuffer.getDouble(1);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+
+ @Test
+ public void testPutDouble() {
+ writableBuffer.putDouble(0, 1d);
+ try {
+ writableBuffer.putDouble(1, 1d);
+ throw new RuntimeException("Expected AssertionError");
+ } catch (final AssertionError expected) {
+ // ignore
+ }
+ }
+}
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryCleanerTest.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryCleanerTest.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryCleanerTest.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.datasketches.memory.internal.MemoryCleaner;
+import org.testng.annotations.Test;
+
+@SuppressWarnings("javadoc")
+public class MemoryCleanerTest {
+
+ @Test
+ public void cleanerDeallocates() {
+ SimpleDeallocator deallocator = new SimpleDeallocator();
+ MemoryCleaner cleaner = new MemoryCleaner(this, deallocator);
+ cleaner.clean();
+ assertTrue(SimpleDeallocator.getHasRun());
+ }
+
+ @SuppressWarnings("unused")
+ @Test
+ public void noDeallocation() {
+ SimpleDeallocator deallocator = new SimpleDeallocator();
+ new MemoryCleaner(this, deallocator);
+ assertFalse(SimpleDeallocator.getHasRun());
+ }
+
+ static final class SimpleDeallocator implements Runnable {
+ static final AtomicBoolean hasRun = new AtomicBoolean();
+
+ SimpleDeallocator() {
+ hasRun.set(false);
+ }
+
+ @Override
+ public void run() {
+ hasRun.compareAndSet(false, true);
+ }
+
+ public static Boolean getHasRun() {
+ return hasRun.get();
+ }
+ }
+}
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryCloseExceptionTest.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryCloseExceptionTest.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryCloseExceptionTest.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import org.apache.datasketches.memory.MemoryCloseException;
+import org.testng.annotations.Test;
+
+@SuppressWarnings("javadoc")
+public class MemoryCloseExceptionTest {
+
+ @Test
+ public void checkNoArgs() {
+ try {
+ throw new MemoryCloseException();
+ } catch (final MemoryCloseException e) {}
+
+ try {
+ throw new MemoryCloseException("Test Exception");
+ } catch (final MemoryCloseException e) {}
+ }
+}
+
+
Added: dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryReadWriteSafetyTest.java
==============================================================================
--- dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryReadWriteSafetyTest.java (added)
+++ dev/datasketches/memory/2.0.0-RC1/apache-datasketches-memory-2.0.0-src/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/MemoryReadWriteSafetyTest.java Thu Aug 19 22:18:24 2021
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.test;
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.apache.datasketches.memory.MapHandle;
+import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.memory.ReadOnlyException;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+@SuppressWarnings("javadoc")
+public class MemoryReadWriteSafetyTest {
+
+ // Test various operations with read-only Memory
+
+ final WritableMemory mem = (WritableMemory) Memory.wrap(new byte[8]);
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testPutByte() {
+ mem.putByte(0, (byte) 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testPutBoolean() {
+ mem.putBoolean(0, true);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testPutShort() {
+ mem.putShort(0, (short) 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testPutChar() {
+ mem.putChar(0, (char) 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testPutInt() {
+ mem.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testPutLong() {
+ mem.putLong(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testPutFloat() {
+ mem.putFloat(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testPutDouble() {
+ mem.putDouble(0, 1);
+ }
+
+ @Test(expectedExceptions = ReadOnlyException.class)
+ public void testPutByteArray() {
+ mem.putByteArray(0, new byte[] {1}, 0, 1);
+ }
+
+ @Test(expectedExceptions = ReadOnlyException.class)
+ public void testPutBooleanArray() {
+ mem.putBooleanArray(0, new boolean[] {true}, 0, 1);
+ }
+
+ @Test(expectedExceptions = ReadOnlyException.class)
+ public void testPutShortArray() {
+ mem.putShortArray(0, new short[] {1}, 0, 1);
+ }
+
+ @Test(expectedExceptions = ReadOnlyException.class)
+ public void testPutCharArray() {
+ mem.putCharArray(0, new char[] {1}, 0, 1);
+ }
+
+ @Test(expectedExceptions = ReadOnlyException.class)
+ public void testPutIntArray() {
+ mem.putIntArray(0, new int[] {1}, 0, 1);
+ }
+
+ @Test(expectedExceptions = ReadOnlyException.class)
+ public void testPutLongArray() {
+ mem.putLongArray(0, new long[] {1}, 0, 1);
+ }
+
+ @Test(expectedExceptions = ReadOnlyException.class)
+ public void testPutFloatArray() {
+ mem.putFloatArray(0, new float[] {1}, 0, 1);
+ }
+
+ @Test(expectedExceptions = ReadOnlyException.class)
+ public void testDoubleByteArray() {
+ mem.putDoubleArray(0, new double[] {1}, 0, 1);
+ }
+
+ // Now, test that various ways to obtain a read-only memory produce a read-only memory indeed
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testWritableMemoryRegion() {
+ WritableMemory mem1 = (WritableMemory) WritableMemory.allocate(8).region(0, 8);
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testByteArrayWrap() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(new byte[8]);
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testByteArrayWrapWithBO() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(new byte[8], ByteOrder.nativeOrder());
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testByteArrayWrapWithOffsetsAndBO() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(new byte[8], 0, 4, ByteOrder.nativeOrder());
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testBooleanArrayWrap() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(new boolean[8]);
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testShortArrayWrap() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(new short[8]);
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testCharArrayWrap() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(new char[8]);
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testIntArrayWrap() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(new int[8]);
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testLongArrayWrap() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(new long[8]);
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testFloatArrayWrap() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(new float[8]);
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testDoubleArrayWrap() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(new double[8]);
+ mem1.putInt(0, 1);
+ }
+
+ @Test(expectedExceptions = AssertionError.class)
+ public void testByteBufferWrap() {
+ WritableMemory mem1 = (WritableMemory) Memory.wrap(ByteBuffer.allocate(8));
+ mem1.putInt(0, 1);
+ }
+
+ //@SuppressWarnings("resource")
+ @Test(expectedExceptions = AssertionError.class)
+ public void testMapFile() throws Exception {
+ File tempFile = File.createTempFile("test", null);
+ tempFile.deleteOnExit();
+ try (RandomAccessFile raf = new RandomAccessFile(tempFile, "rw")) {
+ raf.setLength(8);
+ //System.out.println(UtilTest.getFileAttributes(tempFile));
+ try (MapHandle h = Memory.map(tempFile)) {
+ ((WritableMemory) h.get()).putInt(0, 1);
+ }
+ }
+ }
+
+ @SuppressWarnings("resource")
+ @Test(expectedExceptions = AssertionError.class)
+ public void testMapFileWithOffsetsAndBO() throws Exception {
+ File tempFile = File.createTempFile("test", "test");
+ tempFile.deleteOnExit();
+ new RandomAccessFile(tempFile, "rw").setLength(8);
+ try (MapHandle h = Memory.map(tempFile, 0, 4, ByteOrder.nativeOrder())) {
+ ((WritableMemory) h.get()).putInt(0, 1);
+ }
+ }
+
+ @SuppressWarnings("resource")
+ @Test(expectedExceptions = IllegalArgumentException.class)
+ public void testMapFileBeyondTheFileSize() throws Exception {
+ File tempFile = File.createTempFile("test", "test");
+ tempFile.deleteOnExit();
+ new RandomAccessFile(tempFile, "rw").setLength(8);
+ try (MapHandle unused = Memory.map(tempFile, 0, 16, ByteOrder.nativeOrder())) {
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org