You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by dc...@apache.org on 2021/12/20 20:07:13 UTC
[datasketches-memory] 01/03: Dummy implementation
This is an automated email from the ASF dual-hosted git repository.
dcromberge pushed a commit to branch Jdk17_Panama_David
in repository https://gitbox.apache.org/repos/asf/datasketches-memory.git
commit 1711e8992dc7c2ee913be0b92656c4edd6563b1b
Author: David Cromberge <dc...@apache.org>
AuthorDate: Mon Dec 20 19:47:06 2021 +0000
Dummy implementation
---
.../memory/internal/BaseStateImpl.java | 497 +++++++++++++++++++++
.../apache/datasketches/memory/internal/Dummy.java | 27 --
.../datasketches/memory/internal/UnsafeUtil.java | 228 ++++++++++
.../apache/datasketches/memory/internal/Util.java | 356 +++++++++++++++
4 files changed, 1081 insertions(+), 27 deletions(-)
diff --git a/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/BaseStateImpl.java b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/BaseStateImpl.java
new file mode 100644
index 0000000..bb03e55
--- /dev/null
+++ b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/BaseStateImpl.java
@@ -0,0 +1,497 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.internal;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.datasketches.memory.BaseState;
+import org.apache.datasketches.memory.MemoryRequestServer;
+import org.apache.datasketches.memory.ReadOnlyException;
+import org.apache.datasketches.memory.internal.UnsafeUtil;
+
+import static org.apache.datasketches.memory.internal.UnsafeUtil.checkBounds;
+import static org.apache.datasketches.memory.internal.UnsafeUtil.assertBounds;
+import static org.apache.datasketches.memory.internal.UnsafeUtil.unsafe;
+
+/**
+ * Keeps key configuration state for MemoryImpl and BufferImpl plus some common static variables
+ * and check methods.
+ *
+ * @author Lee Rhodes
+ */
+@SuppressWarnings("restriction")
+public abstract class BaseStateImpl implements BaseState {
+
+ //Monitoring
+ static final AtomicLong currentDirectMemoryAllocations_ = new AtomicLong();
+ static final AtomicLong currentDirectMemoryAllocated_ = new AtomicLong();
+ static final AtomicLong currentDirectMemoryMapAllocations_ = new AtomicLong();
+ static final AtomicLong currentDirectMemoryMapAllocated_ = new AtomicLong();
+ static final String LS = System.getProperty("line.separator");
+
+ //class type IDs. Do not change the bit orders
+ //The first 3 bits are set dynamically
+ // 0000 0XXX
+ static final int READONLY = 1;
+ static final int REGION = 2;
+ static final int DUPLICATE = 4;
+
+ //The following 4 bits are set by the 16 leaf nodes
+ // 000X X000
+ static final int HEAP = 0;
+ static final int DIRECT = 1 << 3;
+ static final int MAP = 2 << 3;
+ static final int BYTEBUF = 3 << 3;
+
+ // 00X0 0000
+ static final int NATIVE = 0;
+ static final int NONNATIVE = 1 << 5;
+
+ // 0X00 0000
+ static final int MEMORY = 0;
+ static final int BUFFER = 1 << 6;
+
+ private final long capacityBytes_;
+
+ /**
+ * This becomes the base offset used by all Unsafe calls. It is cumulative in that in includes
+ * all offsets from regions, user-defined offsets when creating MemoryImpl, and the array object
+ * header offset when creating MemoryImpl from primitive arrays.
+ */
+ private final long cumBaseOffset_;
+
+ /**
+ *
+ * @param unsafeObj The primitive backing array. It may be null. Used by Unsafe calls.
+ * @param nativeBaseOffset The off-heap memory address including DirectByteBuffer split offsets.
+ * @param regionOffset This offset defines address zero of this object (usually a region)
+ * relative to address zero of the backing resource. It is used to compute cumBaseOffset.
+ * This will be loaded from heap ByteBuffers, which have a similar field used for slices.
+ * It is used by region() and writableRegion().
+ * This offset does not include the size of an object array header, if there is one.
+ * @param capacityBytes the capacity of this object. Used by all methods when checking bounds.
+ */
+ BaseStateImpl(final Object unsafeObj, final long nativeBaseOffset, final long regionOffset,
+ final long capacityBytes) {
+ capacityBytes_ = capacityBytes;
+ cumBaseOffset_ = regionOffset + (unsafeObj == null
+ ? nativeBaseOffset
+ : UnsafeUtil.getArrayBaseOffset(unsafeObj.getClass()));
+ }
+
+ //Byte Order Related
+
+ @Override
+ public final ByteOrder getTypeByteOrder() {
+ return isNonNativeType() ? Util.NON_NATIVE_BYTE_ORDER : ByteOrder.nativeOrder();
+ }
+
+ /**
+ * Returns true if the given byteOrder is the same as the native byte order.
+ * @param byteOrder the given byte order
+ * @return true if the given byteOrder is the same as the native byte order.
+ */
+ public static boolean isNativeByteOrder(final ByteOrder byteOrder) {
+ if (byteOrder == null) {
+ throw new IllegalArgumentException("ByteOrder parameter cannot be null.");
+ }
+ return ByteOrder.nativeOrder() == byteOrder;
+ }
+
+ @Override
+ public final boolean isByteOrderCompatible(final ByteOrder byteOrder) {
+ final ByteOrder typeBO = getTypeByteOrder();
+ return typeBO == ByteOrder.nativeOrder() && typeBO == byteOrder;
+ }
+
+ @Override
+ public final boolean equals(final Object that) {
+ if (this == that) { return true; }
+ return that instanceof BaseStateImpl
+ ? CompareAndCopy.equals(this, (BaseStateImpl) that)
+ : false;
+ }
+
+ @Override
+ public final boolean equalTo(final long thisOffsetBytes, final Object that,
+ final long thatOffsetBytes, final long lengthBytes) {
+ return that instanceof BaseStateImpl
+ ? CompareAndCopy.equals(this, thisOffsetBytes, (BaseStateImpl) that, thatOffsetBytes, lengthBytes)
+ : false;
+ }
+
+ //Overridden by ByteBuffer Leafs
+ @Override
+ public ByteBuffer getByteBuffer() {
+ return null;
+ }
+
+ @Override
+ public final long getCapacity() {
+ assertValid();
+ return capacityBytes_;
+ }
+
+ @Override
+ public final long getCumulativeOffset() {
+ assertValid();
+ return cumBaseOffset_;
+ }
+
+ @Override
+ public final long getCumulativeOffset(final long offsetBytes) {
+ assertValid();
+ return cumBaseOffset_ + offsetBytes;
+ }
+
+ //Documented in WritableMemory and WritableBuffer interfaces.
+ //Implemented in the Leaf nodes; Required here by toHex(...).
+ abstract MemoryRequestServer getMemoryRequestServer();
+
+ //Overridden by ByteBuffer, Direct and Map leafs
+ long getNativeBaseOffset() {
+ return 0;
+ }
+
+ @Override
+ public final long getRegionOffset() {
+ final Object unsafeObj = getUnsafeObject();
+ return unsafeObj == null
+ ? cumBaseOffset_ - getNativeBaseOffset()
+ : cumBaseOffset_ - UnsafeUtil.getArrayBaseOffset(unsafeObj.getClass());
+ }
+
+ @Override
+ public final long getRegionOffset(final long offsetBytes) {
+ return getRegionOffset() + offsetBytes;
+ }
+
+ //Overridden by all leafs
+ abstract int getTypeId();
+
+ //Overridden by Heap and ByteBuffer Leafs. Made public as getArray() in WritableMemoryImpl and
+ // WritableBufferImpl
+ Object getUnsafeObject() {
+ return null;
+ }
+
+ @Override
+ public final boolean hasArray() {
+ assertValid();
+ return getUnsafeObject() != null;
+ }
+
+ @Override
+ public final int hashCode() {
+ return (int) xxHash64(0, capacityBytes_, 0); //xxHash64() calls checkValid()
+ }
+
+ @Override
+ public final long xxHash64(final long offsetBytes, final long lengthBytes, final long seed) {
+ checkValid();
+ return XxHash64.hash(getUnsafeObject(), cumBaseOffset_ + offsetBytes, lengthBytes, seed);
+ }
+
+ @Override
+ public final long xxHash64(final long in, final long seed) {
+ return XxHash64.hash(in, seed);
+ }
+
+ @Override
+ public final boolean hasByteBuffer() {
+ assertValid();
+ return getByteBuffer() != null;
+ }
+
+ @Override
+ public final boolean isDirect() {
+ return getUnsafeObject() == null;
+ }
+
+ @Override
+ public final boolean isReadOnly() {
+ assertValid();
+ return isReadOnlyType();
+ }
+
+ @Override
+ public final boolean isSameResource(final Object that) {
+ checkValid();
+ if (that == null) { return false; }
+ final BaseStateImpl that1 = (BaseStateImpl) that;
+ that1.checkValid();
+ if (this == that1) { return true; }
+
+ return cumBaseOffset_ == that1.cumBaseOffset_
+ && capacityBytes_ == that1.capacityBytes_
+ && getUnsafeObject() == that1.getUnsafeObject()
+ && getByteBuffer() == that1.getByteBuffer();
+ }
+
+ //Overridden by Direct and Map leafs
+ @Override
+ public boolean isValid() {
+ return true;
+ }
+
+ //ASSERTS AND CHECKS
+ final void assertValid() {
+ assert isValid() : "MemoryImpl not valid.";
+ }
+
+ void checkValid() {
+ if (!isValid()) {
+ throw new IllegalStateException("MemoryImpl not valid.");
+ }
+ }
+
+ final void assertValidAndBoundsForRead(final long offsetBytes, final long lengthBytes) {
+ assertValid();
+ // capacityBytes_ is intentionally read directly instead of calling getCapacity()
+ // because the later can make JVM to not inline the assert code path (and entirely remove it)
+ // even though it does nothing in production code path.
+ assertBounds(offsetBytes, lengthBytes, capacityBytes_);
+ }
+
+ final void assertValidAndBoundsForWrite(final long offsetBytes, final long lengthBytes) {
+ assertValid();
+ // capacityBytes_ is intentionally read directly instead of calling getCapacity()
+ // because the later can make JVM to not inline the assert code path (and entirely remove it)
+ // even though it does nothing in production code path.
+ assertBounds(offsetBytes, lengthBytes, capacityBytes_);
+ assert !isReadOnly() : "MemoryImpl is read-only.";
+ }
+
+ @Override
+ public final void checkValidAndBounds(final long offsetBytes, final long lengthBytes) {
+ checkValid();
+ //read capacityBytes_ directly to eliminate extra checkValid() call
+ checkBounds(offsetBytes, lengthBytes, capacityBytes_);
+ }
+
+ final void checkValidAndBoundsForWrite(final long offsetBytes, final long lengthBytes) {
+ checkValid();
+ //read capacityBytes_ directly to eliminate extra checkValid() call
+ checkBounds(offsetBytes, lengthBytes, capacityBytes_);
+ if (isReadOnly()) {
+ throw new ReadOnlyException("MemoryImpl is read-only.");
+ }
+ }
+
+ //TYPE ID Management
+ final boolean isReadOnlyType() {
+ return (getTypeId() & READONLY) > 0;
+ }
+
+ final static byte setReadOnlyType(final byte type, final boolean readOnly) {
+ return (byte)((type & ~1) | (readOnly ? READONLY : 0));
+ }
+
+ final boolean isRegionType() {
+ return (getTypeId() & REGION) > 0;
+ }
+
+ final boolean isDuplicateType() {
+ return (getTypeId() & DUPLICATE) > 0;
+ }
+
+ //The following are set by the leaf nodes
+ final boolean isBufferType() {
+ return (getTypeId() & BUFFER) > 0;
+ }
+
+ final boolean isNonNativeType() {
+ return (getTypeId() & NONNATIVE) > 0;
+ }
+
+ final boolean isHeapType() {
+ return (getTypeId() >>> 3 & 3) == 0;
+ }
+
+ final boolean isDirectType() {
+ return (getTypeId() >>> 3 & 3) == 1;
+ }
+
+ final boolean isMapType() {
+ return (getTypeId() >>> 3 & 3) == 2;
+ }
+
+ final boolean isBBType() {
+ return (getTypeId() >>> 3 & 3) == 3;
+ }
+
+
+ //TO STRING
+ /**
+ * Decodes the resource type. This is primarily for debugging.
+ * @param typeId the given typeId
+ * @return a human readable string.
+ */
+ public static final String typeDecode(final int typeId) {
+ final StringBuilder sb = new StringBuilder();
+ final int group1 = typeId & 0x7;
+ switch (group1) {
+ case 1 : sb.append("ReadOnly, "); break;
+ case 2 : sb.append("Region, "); break;
+ case 3 : sb.append("ReadOnly Region, "); break;
+ case 4 : sb.append("Duplicate, "); break;
+ case 5 : sb.append("ReadOnly Duplicate, "); break;
+ case 6 : sb.append("Region Duplicate, "); break;
+ case 7 : sb.append("ReadOnly Region Duplicate, "); break;
+ default: break;
+ }
+ final int group2 = (typeId >>> 3) & 0x3;
+ switch (group2) {
+ case 0 : sb.append("Heap, "); break;
+ case 1 : sb.append("Direct, "); break;
+ case 2 : sb.append("Map, "); break;
+ case 3 : sb.append("ByteBuffer, "); break;
+ default: break;
+ }
+ final int group3 = (typeId >>> 5) & 0x1;
+ switch (group3) {
+ case 0 : sb.append("Native, "); break;
+ case 1 : sb.append("NonNative, "); break;
+ default: break;
+ }
+ final int group4 = (typeId >>> 6) & 0x1;
+ switch (group4) {
+ case 0 : sb.append("Memory"); break;
+ case 1 : sb.append("Buffer"); break;
+ default: break;
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public final String toHexString(final String header, final long offsetBytes,
+ final int lengthBytes) {
+ checkValid();
+ final String klass = this.getClass().getSimpleName();
+ final String s1 = String.format("(..., %d, %d)", offsetBytes, lengthBytes);
+ final long hcode = hashCode() & 0XFFFFFFFFL;
+ final String call = ".toHexString" + s1 + ", hashCode: " + hcode;
+ final StringBuilder sb = new StringBuilder();
+ sb.append("### ").append(klass).append(" SUMMARY ###").append(LS);
+ sb.append("Header Comment : ").append(header).append(LS);
+ sb.append("Call Parameters : ").append(call);
+ return toHex(this, sb.toString(), offsetBytes, lengthBytes);
+ }
+
+ /**
+ * Returns a formatted hex string of an area of this object.
+ * Used primarily for testing.
+ * @param state the BaseStateImpl
+ * @param preamble a descriptive header
+ * @param offsetBytes offset bytes relative to the MemoryImpl start
+ * @param lengthBytes number of bytes to convert to a hex string
+ * @return a formatted hex string in a human readable array
+ */
+ static final String toHex(final BaseStateImpl state, final String preamble, final long offsetBytes,
+ final int lengthBytes) {
+ final long capacity = state.getCapacity();
+ checkBounds(offsetBytes, lengthBytes, capacity);
+ final StringBuilder sb = new StringBuilder();
+ final Object uObj = state.getUnsafeObject();
+ final String uObjStr;
+ final long uObjHeader;
+ if (uObj == null) {
+ uObjStr = "null";
+ uObjHeader = 0;
+ } else {
+ uObjStr = uObj.getClass().getSimpleName() + ", " + (uObj.hashCode() & 0XFFFFFFFFL);
+ uObjHeader = UnsafeUtil.getArrayBaseOffset(uObj.getClass());
+ }
+ final ByteBuffer bb = state.getByteBuffer();
+ final String bbStr = bb == null ? "null"
+ : bb.getClass().getSimpleName() + ", " + (bb.hashCode() & 0XFFFFFFFFL);
+ final MemoryRequestServer memReqSvr = state.getMemoryRequestServer();
+ final String memReqStr = memReqSvr != null
+ ? memReqSvr.getClass().getSimpleName() + ", " + (memReqSvr.hashCode() & 0XFFFFFFFFL)
+ : "null";
+ final long cumBaseOffset = state.getCumulativeOffset();
+ sb.append(preamble).append(LS);
+ sb.append("UnsafeObj, hashCode : ").append(uObjStr).append(LS);
+ sb.append("UnsafeObjHeader : ").append(uObjHeader).append(LS);
+ sb.append("ByteBuf, hashCode : ").append(bbStr).append(LS);
+ sb.append("RegionOffset : ").append(state.getRegionOffset()).append(LS);
+ sb.append("Capacity : ").append(capacity).append(LS);
+ sb.append("CumBaseOffset : ").append(cumBaseOffset).append(LS);
+ sb.append("MemReq, hashCode : ").append(memReqStr).append(LS);
+ sb.append("Valid : ").append(state.isValid()).append(LS);
+ sb.append("Read Only : ").append(state.isReadOnly()).append(LS);
+ sb.append("Type Byte Order : ").append(state.getTypeByteOrder().toString()).append(LS);
+ sb.append("Native Byte Order : ").append(ByteOrder.nativeOrder().toString()).append(LS);
+ sb.append("JDK Runtime Version : ").append(UnsafeUtil.JDK).append(LS);
+ //Data detail
+ sb.append("Data, littleEndian : 0 1 2 3 4 5 6 7");
+
+ for (long i = 0; i < lengthBytes; i++) {
+ final int b = unsafe.getByte(uObj, cumBaseOffset + offsetBytes + i) & 0XFF;
+ if (i % 8 == 0) { //row header
+ sb.append(String.format("%n%20s: ", offsetBytes + i));
+ }
+ sb.append(String.format("%02x ", b));
+ }
+ sb.append(LS);
+
+ return sb.toString();
+ }
+
+ //MONITORING
+
+ /**
+ * Gets the current number of active direct memory allocations.
+ * @return the current number of active direct memory allocations.
+ */
+ public static final long getCurrentDirectMemoryAllocations() {
+ return BaseStateImpl.currentDirectMemoryAllocations_.get();
+ }
+
+ /**
+ * Gets the current size of active direct memory allocated.
+ * @return the current size of active direct memory allocated.
+ */
+ public static final long getCurrentDirectMemoryAllocated() {
+ return BaseStateImpl.currentDirectMemoryAllocated_.get();
+ }
+
+ /**
+ * Gets the current number of active direct memory map allocations.
+ * @return the current number of active direct memory map allocations.
+ */
+ public static final long getCurrentDirectMemoryMapAllocations() {
+ return BaseStateImpl.currentDirectMemoryMapAllocations_.get();
+ }
+
+ /**
+ * Gets the current size of active direct memory map allocated.
+ * @return the current size of active direct memory map allocated.
+ */
+ public static final long getCurrentDirectMemoryMapAllocated() {
+ return BaseStateImpl.currentDirectMemoryMapAllocated_.get();
+ }
+
+ //REACHABILITY FENCE
+ static void reachabilityFence(@SuppressWarnings("unused") final Object obj) { }
+
+}
\ No newline at end of file
diff --git a/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Dummy.java b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Dummy.java
deleted file mode 100644
index aefdcba..0000000
--- a/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Dummy.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.datasketches.memory.internal;
-
-/**
- * Temporary class & placeholder. It may not be needed.
- */
-public class Dummy {
-
-}
diff --git a/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/UnsafeUtil.java b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/UnsafeUtil.java
new file mode 100644
index 0000000..9165b0d
--- /dev/null
+++ b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/UnsafeUtil.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.internal;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+import sun.misc.Unsafe;
+
+/**
+ * Provides access to the sun.misc.Unsafe class and its key static fields.
+ *
+ * @author Lee Rhodes
+ */
+@SuppressWarnings({"restriction", "javadoc"})
+public final class UnsafeUtil {
+ public static final Unsafe unsafe;
+ public static final String JDK; //must be at least "1.8"
+ public static final int JDK_MAJOR; //8, 9, 10, 11, 12, etc
+
+ //not an indicator of whether compressed references are used.
+ public static final int ADDRESS_SIZE;
+ public static final int PAGE_SIZE;
+
+ //For 64-bit JVMs: these offsets vary depending on coop: 16 for JVM <= 32GB; 24 for JVM > 32GB.
+ // Making this constant long-typed, rather than int, to exclude possibility of accidental overflow
+ // in expressions like arrayLength * ARRAY_BYTE_BASE_OFFSET, where arrayLength is int-typed.
+ // The same consideration for constants below: ARRAY_*_INDEX_SCALE, ARRAY_*_INDEX_SHIFT.
+ public static final long ARRAY_BOOLEAN_BASE_OFFSET;
+ public static final long ARRAY_BYTE_BASE_OFFSET;
+ public static final long ARRAY_SHORT_BASE_OFFSET;
+ public static final long ARRAY_CHAR_BASE_OFFSET;
+ public static final long ARRAY_INT_BASE_OFFSET;
+ public static final long ARRAY_LONG_BASE_OFFSET;
+ public static final long ARRAY_FLOAT_BASE_OFFSET;
+ public static final long ARRAY_DOUBLE_BASE_OFFSET;
+ public static final long ARRAY_OBJECT_BASE_OFFSET;
+
+ //@formatter:off
+
+ // Setting those values directly instead of using unsafe.arrayIndexScale(), because it may be
+ // beneficial for runtime execution, those values are backed into generated machine code as
+ // constants. E. g. see https://shipilev.net/jvm-anatomy-park/14-constant-variables/
+ public static final int ARRAY_BOOLEAN_INDEX_SCALE = 1;
+ public static final int ARRAY_BYTE_INDEX_SCALE = 1;
+ public static final long ARRAY_SHORT_INDEX_SCALE = 2;
+ public static final long ARRAY_CHAR_INDEX_SCALE = 2;
+ public static final long ARRAY_INT_INDEX_SCALE = 4;
+ public static final long ARRAY_LONG_INDEX_SCALE = 8;
+ public static final long ARRAY_FLOAT_INDEX_SCALE = 4;
+ public static final long ARRAY_DOUBLE_INDEX_SCALE = 8;
+ public static final long ARRAY_OBJECT_INDEX_SCALE; // varies, 4 or 8 depending on coop
+
+ //Used to convert "type" to bytes: bytes = longs << LONG_SHIFT
+ public static final int BOOLEAN_SHIFT = 0;
+ public static final int BYTE_SHIFT = 0;
+ public static final long SHORT_SHIFT = 1;
+ public static final long CHAR_SHIFT = 1;
+ public static final long INT_SHIFT = 2;
+ public static final long LONG_SHIFT = 3;
+ public static final long FLOAT_SHIFT = 2;
+ public static final long DOUBLE_SHIFT = 3;
+ public static final long OBJECT_SHIFT; // varies, 2 or 3 depending on coop
+
+ public static final String LS = System.getProperty("line.separator");
+
+ //@formatter:on
+
+ static {
+ try {
+ final Constructor<Unsafe> unsafeConstructor = Unsafe.class.getDeclaredConstructor();
+ unsafeConstructor.setAccessible(true);
+ unsafe = unsafeConstructor.newInstance();
+
+ // Alternative, but may not work across different JVMs.
+ // Field field = Unsafe.class.getDeclaredField("theUnsafe");
+ // field.setAccessible(true);
+ // unsafe = (Unsafe) field.get(null);
+
+ } catch (final InstantiationException | IllegalAccessException | IllegalArgumentException
+ | InvocationTargetException | NoSuchMethodException e) {
+ e.printStackTrace();
+ throw new RuntimeException("Unable to acquire Unsafe. " + e);
+ }
+
+ //4 on 32-bit systems. 4 on 64-bit systems < 32GB, otherwise 8.
+ //This alone is not an indicator of compressed ref (coop)
+ ADDRESS_SIZE = unsafe.addressSize();
+ PAGE_SIZE = unsafe.pageSize();
+
+ ARRAY_BOOLEAN_BASE_OFFSET = unsafe.arrayBaseOffset(boolean[].class);
+ ARRAY_BYTE_BASE_OFFSET = unsafe.arrayBaseOffset(byte[].class);
+ ARRAY_SHORT_BASE_OFFSET = unsafe.arrayBaseOffset(short[].class);
+ ARRAY_CHAR_BASE_OFFSET = unsafe.arrayBaseOffset(char[].class);
+ ARRAY_INT_BASE_OFFSET = unsafe.arrayBaseOffset(int[].class);
+ ARRAY_LONG_BASE_OFFSET = unsafe.arrayBaseOffset(long[].class);
+ ARRAY_FLOAT_BASE_OFFSET = unsafe.arrayBaseOffset(float[].class);
+ ARRAY_DOUBLE_BASE_OFFSET = unsafe.arrayBaseOffset(double[].class);
+ ARRAY_OBJECT_BASE_OFFSET = unsafe.arrayBaseOffset(Object[].class);
+
+ ARRAY_OBJECT_INDEX_SCALE = unsafe.arrayIndexScale(Object[].class);
+ OBJECT_SHIFT = ARRAY_OBJECT_INDEX_SCALE == 4 ? 2 : 3;
+
+ final String jdkVer = System.getProperty("java.version");
+ final int[] p = parseJavaVersion(jdkVer);
+ JDK = p[0] + "." + p[1];
+ JDK_MAJOR = (p[0] == 1) ? p[1] : p[0];
+ }
+
+ private UnsafeUtil() {}
+
+ /**
+ * Returns first two number groups of the java version string.
+ * @param jdkVer the java version string from System.getProperty("java.version").
+ * @return first two number groups of the java version string.
+ */
+ public static int[] parseJavaVersion(final String jdkVer) {
+ final int p0, p1;
+ try {
+ String[] parts = jdkVer.trim().split("[^0-9\\.]");//grab only number groups and "."
+ parts = parts[0].split("\\."); //split out the number groups
+ p0 = Integer.parseInt(parts[0]); //the first number group
+ p1 = (parts.length > 1) ? Integer.parseInt(parts[1]) : 0; //2nd number group, or 0
+ } catch (final NumberFormatException | ArrayIndexOutOfBoundsException e) {
+ throw new IllegalArgumentException("Improper Java -version string: " + jdkVer + "\n" + e);
+ }
+ //checkJavaVersion(jdkVer, p0, p1); //TODO Optional to omit this.
+ return new int[] {p0, p1};
+ }
+
+ public static void checkJavaVersion(final String jdkVer, final int p0, final int p1) {
+ if ( (p0 < 1) || ((p0 == 1) && (p1 < 8)) || (p0 > 13) ) {
+ throw new IllegalArgumentException(
+ "Unsupported JDK Major Version, must be one of 1.8, 8, 9, 10, 11, 12, 13: " + jdkVer);
+ }
+ }
+
+ public static long getFieldOffset(final Class<?> c, final String fieldName) {
+ try {
+ return unsafe.objectFieldOffset(c.getDeclaredField(fieldName));
+ } catch (final NoSuchFieldException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Like {@link Unsafe#arrayBaseOffset(Class)}, but caches return values for common array types.
+ * Useful because calling {@link Unsafe#arrayBaseOffset(Class)} directly incurs more overhead.
+ * @param c The given Class<?>.
+ * @return the base-offset
+ */
+ public static long getArrayBaseOffset(final Class<?> c) {
+ // Ordering here is roughly in order of what we expect to be most popular.
+ if (c == byte[].class) {
+ return ARRAY_BYTE_BASE_OFFSET;
+ } else if (c == int[].class) {
+ return ARRAY_INT_BASE_OFFSET;
+ } else if (c == long[].class) {
+ return ARRAY_LONG_BASE_OFFSET;
+ } else if (c == float[].class) {
+ return ARRAY_FLOAT_BASE_OFFSET;
+ } else if (c == double[].class) {
+ return ARRAY_DOUBLE_BASE_OFFSET;
+ } else if (c == boolean[].class) {
+ return ARRAY_BOOLEAN_BASE_OFFSET;
+ } else if (c == short[].class) {
+ return ARRAY_SHORT_BASE_OFFSET;
+ } else if (c == char[].class) {
+ return ARRAY_CHAR_BASE_OFFSET;
+ } else if (c == Object[].class) {
+ return ARRAY_OBJECT_BASE_OFFSET;
+ } else {
+ return unsafe.arrayBaseOffset(c);
+ }
+ }
+
+ public static long pageCount(final long bytes) {
+ return (int)((bytes + PAGE_SIZE) - 1L) / PAGE_SIZE;
+ }
+
+ /**
+ * Assert the requested offset and length against the allocated size.
+ * The invariants equation is: {@code 0 <= reqOff <= reqLen <= reqOff + reqLen <= allocSize}.
+ * If this equation is violated and assertions are enabled, an {@link AssertionError} will
+ * be thrown.
+ * @param reqOff the requested offset
+ * @param reqLen the requested length
+ * @param allocSize the allocated size.
+ */
+ public static void assertBounds(final long reqOff, final long reqLen, final long allocSize) {
+ assert ((reqOff | reqLen | (reqOff + reqLen) | (allocSize - (reqOff + reqLen))) >= 0) :
+ "reqOffset: " + reqOff + ", reqLength: " + reqLen
+ + ", (reqOff + reqLen): " + (reqOff + reqLen) + ", allocSize: " + allocSize;
+ }
+
+ /**
+ * Check the requested offset and length against the allocated size.
+ * The invariants equation is: {@code 0 <= reqOff <= reqLen <= reqOff + reqLen <= allocSize}.
+ * If this equation is violated an {@link IllegalArgumentException} will be thrown.
+ * @param reqOff the requested offset
+ * @param reqLen the requested length
+ * @param allocSize the allocated size.
+ */
+ public static void checkBounds(final long reqOff, final long reqLen, final long allocSize) {
+ if ((reqOff | reqLen | (reqOff + reqLen) | (allocSize - (reqOff + reqLen))) < 0) {
+ throw new IllegalArgumentException(
+ "reqOffset: " + reqOff + ", reqLength: " + reqLen
+ + ", (reqOff + reqLen): " + (reqOff + reqLen) + ", allocSize: " + allocSize);
+ }
+ }
+}
\ No newline at end of file
diff --git a/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Util.java b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Util.java
new file mode 100644
index 0000000..d7e0343
--- /dev/null
+++ b/datasketches-memory-java17/src/main/java/org/apache/datasketches/memory/internal/Util.java
@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.internal;
+
+import static jdk.incubator.foreign.MemoryAccess.getLongAtIndex;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Random;
+
+import jdk.incubator.foreign.MemorySegment;
+
+/**
+ * @author Lee Rhodes
+ */
+@SuppressWarnings("javadoc")
+public final class Util {
+ public static final String LS = System.getProperty("line.separator");
+
+ //Byte Order related
+ public static final ByteOrder NON_NATIVE_BYTE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN
+ ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN;
+
+ public static ByteOrder otherByteOrder(final ByteOrder order) {
+ return (order == ByteOrder.nativeOrder()) ? NON_NATIVE_BYTE_ORDER : ByteOrder.nativeOrder();
+ }
+
+
+ /**
+ * Don't use sun.misc.Unsafe#copyMemory to copy blocks of memory larger than this
+ * threshold, because internally it doesn't have safepoint polls, that may cause long
+ * "Time To Safe Point" pauses in the application. This has been fixed in JDK 9 (see
+ * https://bugs.openjdk.java.net/browse/JDK-8149596 and
+ * https://bugs.openjdk.java.net/browse/JDK-8141491), but not in JDK 8, so the Memory library
+ * should keep having this boilerplate as long as it supports Java 8.
+ *
+ * <p>A reference to this can be found in java.nio.Bits.</p>
+ */
+ public static final int UNSAFE_COPY_THRESHOLD_BYTES = 1024 * 1024;
+
+ private Util() { }
+
+ //Byte Order Related
+
+ /**
+ * Returns true if the given byteOrder is the same as the native byte order.
+ * @param byteOrder the given byte order
+ * @return true if the given byteOrder is the same as the native byte order.
+ */
+ public static boolean isNativeByteOrder(final ByteOrder byteOrder) {
+ if (byteOrder == null) {
+ throw new IllegalArgumentException("ByteOrder parameter cannot be null.");
+ }
+ return ByteOrder.nativeOrder() == byteOrder;
+ }
+
+
+ /**
+ * Searches a range of the specified array of longs for the specified value using the binary
+ * search algorithm. The range must be sorted method) prior to making this call.
+ * If it is not sorted, the results are undefined. If the range contains
+ * multiple elements with the specified value, there is no guarantee which one will be found.
+ * @param seg the MemorySegment to be searched
+ * @param fromLongIndex the index of the first long element (inclusive) to be searched
+ * @param toLongIndex the index of the last long element (exclusive) to be searched
+ * @param key the value to be searched for
+ * @return index of the search key, if it is contained in the array within the specified range;
+ * otherwise, (-(insertion point) - 1). The insertion point is defined as the point at which
+ * the key would be inserted into the array: the index of the first element in the range greater
+ * than the key, or toIndex if all elements in the range are less than the specified key.
+ * Note that this guarantees that the return value will be ≥ 0 if and only if the key is found.
+ */
+ public static long binarySearchLongs(final MemorySegment seg, final long fromLongIndex,
+ final long toLongIndex, final long key) {
+ UnsafeUtil.checkBounds(fromLongIndex << 3, (toLongIndex - fromLongIndex) << 3, seg.byteSize());
+ long low = fromLongIndex;
+ long high = toLongIndex - 1L;
+
+ while (low <= high) {
+ final long mid = (low + high) >>> 1;
+ final long midVal = getLongAtIndex(seg, mid);
+
+ if (midVal < key) { low = mid + 1; }
+ else if (midVal > key) { high = mid - 1; }
+ else { return mid; } // key found
+ }
+ return -(low + 1); // key not found.
+ }
+
+ /**
+ * Prepend the given string with zeros. If the given string is equal or greater than the given
+ * field length, it will be returned without modification.
+ * @param s the given string
+ * @param fieldLength desired total field length including the given string
+ * @return the given string prepended with zeros.
+ */
+ public static final String zeroPad(final String s, final int fieldLength) {
+ return characterPad(s, fieldLength, '0', false);
+ }
+
+ /**
+ * Prepend or postpend the given string with the given character to fill the given field length.
+ * If the given string is equal or greater than the given field length, it will be returned
+ * without modification.
+ * @param s the given string
+ * @param fieldLength the desired field length
+ * @param padChar the desired pad character
+ * @param postpend if true append the pacCharacters to the end of the string.
+ * @return prepended or postpended given string with the given character to fill the given field
+ * length.
+ */
+ public static final String characterPad(final String s, final int fieldLength,
+ final char padChar, final boolean postpend) {
+ final char[] chArr = s.toCharArray();
+ final int sLen = chArr.length;
+ if (sLen < fieldLength) {
+ final char[] out = new char[fieldLength];
+ final int blanks = fieldLength - sLen;
+
+ if (postpend) {
+ for (int i = 0; i < sLen; i++) {
+ out[i] = chArr[i];
+ }
+ for (int i = sLen; i < fieldLength; i++) {
+ out[i] = padChar;
+ }
+ } else { //prepend
+ for (int i = 0; i < blanks; i++) {
+ out[i] = padChar;
+ }
+ for (int i = blanks; i < fieldLength; i++) {
+ out[i] = chArr[i - blanks];
+ }
+ }
+
+ return String.valueOf(out);
+ }
+ return s;
+ }
+
+ /**
+ * Return true if all the masked bits of value are zero
+ * @param value the value to be tested
+ * @param bitMask defines the bits of interest
+ * @return true if all the masked bits of value are zero
+ */
+ public static final boolean isAllBitsClear(final long value, final long bitMask) {
+ return (~value & bitMask) == bitMask;
+ }
+
+ /**
+ * Return true if all the masked bits of value are one
+ * @param value the value to be tested
+ * @param bitMask defines the bits of interest
+ * @return true if all the masked bits of value are one
+ */
+ public static final boolean isAllBitsSet(final long value, final long bitMask) {
+ return (value & bitMask) == bitMask;
+ }
+
+ /**
+ * Return true if any the masked bits of value are zero
+ * @param value the value to be tested
+ * @param bitMask defines the bits of interest
+ * @return true if any the masked bits of value are zero
+ */
+ public static final boolean isAnyBitsClear(final long value, final long bitMask) {
+ return (~value & bitMask) != 0;
+ }
+
+ /**
+ * Return true if any the masked bits of value are one
+ * @param value the value to be tested
+ * @param bitMask defines the bits of interest
+ * @return true if any the masked bits of value are one
+ */
+ public static final boolean isAnyBitsSet(final long value, final long bitMask) {
+ return (value & bitMask) != 0;
+ }
+
+ /**
+ * Creates random valid Character Code Points (as integers). By definition, valid CodePoints
+ * are integers in the range 0 to Character.MAX_CODE_POINT, and exclude the surrogate values.
+ * This is used in unit testing and characterization testing of the UTF8 class. Because the
+ * characterization tools are in a separate package, this must remain public.
+ *
+ * @author Lee Rhodes
+ */
+ public static class RandomCodePoints {
+ private Random rand; //
+ private static final int ALL_CP = Character.MAX_CODE_POINT + 1;
+ private static final int MIN_SUR = Character.MIN_SURROGATE;
+ private static final int MAX_SUR = Character.MAX_SURROGATE;
+
+ /**
+ * @param deterministic if true, configure java.util.Random with a fixed seed.
+ */
+ public RandomCodePoints(final boolean deterministic) {
+ rand = deterministic ? new Random(0) : new Random();
+ }
+
+ /**
+ * Fills the given array with random valid Code Points from 0, inclusive, to
+ * <i>Character.MAX_CODE_POINT</i>, inclusive.
+ * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to
+ * <i>Character.MAX_SURROGATE</i>, inclusive, is always <i>excluded</i>.
+ * @param cpArr the array to fill
+ */
+ public final void fillCodePointArray(final int[] cpArr) {
+ fillCodePointArray(cpArr, 0, ALL_CP);
+ }
+
+ /**
+ * Fills the given array with random valid Code Points from <i>startCP</i>, inclusive, to
+ * <i>endCP</i>, exclusive.
+ * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to
+ * <i>Character.MAX_SURROGATE</i>, inclusive, is always <i>excluded</i>.
+ * @param cpArr the array to fill
+ * @param startCP the starting Code Point, included.
+ * @param endCP the ending Code Point, excluded. This value cannot exceed 0x110000.
+ */
+ public final void fillCodePointArray(final int[] cpArr, final int startCP, final int endCP) {
+ final int arrLen = cpArr.length;
+ final int numCP = Math.min(endCP, 0X110000) - Math.min(0, startCP);
+ int idx = 0;
+ while (idx < arrLen) {
+ final int cp = startCP + rand.nextInt(numCP);
+ if ((cp >= MIN_SUR) && (cp <= MAX_SUR)) {
+ continue;
+ }
+ cpArr[idx++] = cp;
+ }
+ }
+
+ /**
+ * Return a single valid random Code Point from 0, inclusive, to
+ * <i>Character.MAX_CODE_POINT</i>, inclusive.
+ * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to
+ * <i>Character.MAX_SURROGATE</i>, inclusive, is always <i>excluded</i>.
+ * @return a single valid random CodePoint.
+ */
+ public final int getCodePoint() {
+ return getCodePoint(0, ALL_CP);
+ }
+
+ /**
+ * Return a single valid random Code Point from <i>startCP</i>, inclusive, to
+ * <i>endCP</i>, exclusive.
+ * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to
+ * <i>Character.MAX_SURROGATE</i>, inclusive, is always <i>excluded</i>.
+ * @param startCP the starting Code Point, included.
+ * @param endCP the ending Code Point, excluded. This value cannot exceed 0x110000.
+ * @return a single valid random CodePoint.
+ */
+ public final int getCodePoint(final int startCP, final int endCP) {
+ final int numCP = Math.min(endCP, 0X110000) - Math.min(0, startCP);
+ while (true) {
+ final int cp = startCP + rand.nextInt(numCP);
+ if ((cp < MIN_SUR) || (cp > MAX_SUR)) {
+ return cp;
+ }
+ }
+ }
+ } //End class RandomCodePoints
+
+ public static final void zeroCheck(final long value, final String arg) {
+ if (value <= 0) {
+ throw new IllegalArgumentException("The argument '" + arg + "' may not be negative or zero.");
+ }
+ }
+
+ public static final void negativeCheck(final long value, final String arg) {
+ if (value < 0) {
+ throw new IllegalArgumentException("The argument '" + arg + "' may not be negative.");
+ }
+ }
+
+ public static final void nullCheck(final Object obj, final String arg) {
+ if (obj == null) {
+ throw new IllegalArgumentException("The argument '" + arg + "' may not be null.");
+ }
+ }
+
+ //Resources
+
+ /**
+ * Gets the absolute path of the given resource file's shortName.
+ *
+ * <p>Note that the ClassLoader.getResource(shortName) returns a URL,
+ * which can have special characters, e.g., "%20" for spaces. This method
+ * obtains the URL, converts it to a URI, then does a uri.getPath(), which
+ * decodes any special characters in the URI path. This is required to make
+ * obtaining resources operating-system independent.</p>
+ *
+ * @param shortFileName the last name in the pathname's name sequence.
+ * @return the absolute path of the given resource file's shortName.
+ */
+ public static String getResourcePath(final String shortFileName) {
+ try {
+ final URL url = Util.class.getClassLoader().getResource(shortFileName);
+ final URI uri = url.toURI();
+ final String path = uri.getPath(); //decodes any special characters
+ return path;
+ } catch (final NullPointerException | URISyntaxException e) {
+ throw new IllegalArgumentException("Cannot find resource: " + shortFileName + LS + e);
+ }
+ }
+
+ /**
+ * Gets the file defined by the given resource file's shortFileName.
+ * @param shortFileName the last name in the pathname's name sequence.
+ * @return the file defined by the given resource file's shortFileName.
+ */
+ public static File getResourceFile(final String shortFileName) {
+ return new File(getResourcePath(shortFileName));
+ }
+
+ /**
+ * Returns a byte array of the contents of the file defined by the given resource file's
+ * shortFileName.
+ * @param shortFileName the last name in the pathname's name sequence.
+ * @return a byte array of the contents of the file defined by the given resource file's
+ * shortFileName.
+ */
+ public static byte[] getResourceBytes(final String shortFileName) {
+ try {
+ return Files.readAllBytes(Paths.get(getResourcePath(shortFileName)));
+ } catch (final IOException e) {
+ throw new IllegalArgumentException("Cannot read resource: " + shortFileName + LS + e);
+ }
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org