You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2022/06/01 05:47:54 UTC
[datasketches-memory17] branch main updated: Interim 5 Some cleanup of unused stuff.
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datasketches-memory17.git
The following commit(s) were added to refs/heads/main by this push:
new de5cdc0 Interim 5 Some cleanup of unused stuff.
de5cdc0 is described below
commit de5cdc0f66c69af2bd1d00dda6d5cc3dedd806a5
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Tue May 31 22:47:49 2022 -0700
Interim 5 Some cleanup of unused stuff.
---
.../org/apache/datasketches/memory/Memory.java | 3 -
.../apache/datasketches/memory/WritableMemory.java | 3 -
.../memory/internal/BaseWritableBufferImpl.java | 6 +-
.../memory/internal/BaseWritableMemoryImpl.java | 8 +-
.../apache/datasketches/memory/internal/Ints.java | 39 ----
.../apache/datasketches/memory/internal/Util.java | 220 +--------------------
.../datasketches/memory/internal/UtilTest.java | 89 ---------
7 files changed, 8 insertions(+), 360 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/memory/Memory.java b/src/main/java/org/apache/datasketches/memory/Memory.java
index 437515c..2c4586d 100644
--- a/src/main/java/org/apache/datasketches/memory/Memory.java
+++ b/src/main/java/org/apache/datasketches/memory/Memory.java
@@ -27,7 +27,6 @@ import java.nio.ByteOrder;
import java.util.Objects;
import org.apache.datasketches.memory.internal.BaseWritableMemoryImpl;
-import org.apache.datasketches.memory.internal.Util;
import jdk.incubator.foreign.MemorySegment;
import jdk.incubator.foreign.ResourceScope;
@@ -101,8 +100,6 @@ public interface Memory extends BaseState {
Objects.requireNonNull(file, "File must be non-null.");
Objects.requireNonNull(byteOrder, "ByteOrder must be non-null.");
Objects.requireNonNull(scope, "ResourceScope must be non-null.");
- Util.negativeCheck(fileOffsetBytes, "fileOffsetBytes");
- Util.negativeCheck(capacityBytes, "capacityBytes");
if (!file.canRead()) { throw new IllegalArgumentException("File must be readable."); }
return BaseWritableMemoryImpl.wrapMap(file, fileOffsetBytes, capacityBytes, scope, true, byteOrder);
}
diff --git a/src/main/java/org/apache/datasketches/memory/WritableMemory.java b/src/main/java/org/apache/datasketches/memory/WritableMemory.java
index 73b0ca0..3d4f007 100644
--- a/src/main/java/org/apache/datasketches/memory/WritableMemory.java
+++ b/src/main/java/org/apache/datasketches/memory/WritableMemory.java
@@ -25,7 +25,6 @@ import java.nio.ByteOrder;
import java.util.Objects;
import org.apache.datasketches.memory.internal.BaseWritableMemoryImpl;
-import org.apache.datasketches.memory.internal.Util;
import jdk.incubator.foreign.MemorySegment;
import jdk.incubator.foreign.ResourceScope;
@@ -102,8 +101,6 @@ public interface WritableMemory extends Memory {
Objects.requireNonNull(file, "File must be non-null.");
Objects.requireNonNull(byteOrder, "ByteOrder must be non-null.");
Objects.requireNonNull(scope, "ResourceScope must be non-null.");
- Util.negativeCheck(fileOffsetBytes, "fileOffsetBytes");
- Util.negativeCheck(capacityBytes, "capacityBytes");
if (!file.canWrite()) { throw new ReadOnlyException("file must be writable."); }
return BaseWritableMemoryImpl.wrapMap(file, fileOffsetBytes, capacityBytes, scope, false, byteOrder);
}
diff --git a/src/main/java/org/apache/datasketches/memory/internal/BaseWritableBufferImpl.java b/src/main/java/org/apache/datasketches/memory/internal/BaseWritableBufferImpl.java
index 7002053..8fab698 100644
--- a/src/main/java/org/apache/datasketches/memory/internal/BaseWritableBufferImpl.java
+++ b/src/main/java/org/apache/datasketches/memory/internal/BaseWritableBufferImpl.java
@@ -133,7 +133,7 @@ public abstract class BaseWritableBufferImpl extends BaseBufferImpl implements W
final boolean duplicateType = isDuplicateType();
final boolean mapType = seg.isMapped();
final boolean directType = seg.isNative();
- final boolean nativeBOType = Util.isNativeByteOrder(byteOrder);
+ final boolean nativeBOType = byteOrder == ByteOrder.nativeOrder();
final boolean byteBufferType = isByteBufferType();
final int type = BUFFER | REGION
| (readOnly ? READONLY : 0)
@@ -194,7 +194,7 @@ public abstract class BaseWritableBufferImpl extends BaseBufferImpl implements W
final boolean duplicateType = true;
final boolean mapType = seg.isMapped();
final boolean directType = seg.isNative();
- final boolean nativeBOType = Util.isNativeByteOrder(byteOrder);
+ final boolean nativeBOType = byteOrder == ByteOrder.nativeOrder();
final boolean byteBufferType = isByteBufferType();
final int type = BUFFER
| (readOnly ? READONLY : 0)
@@ -250,7 +250,7 @@ public abstract class BaseWritableBufferImpl extends BaseBufferImpl implements W
final boolean duplicateType = isDuplicateType();
final boolean mapType = seg.isMapped();
final boolean directType = seg.isNative();
- final boolean nativeBOType = Util.isNativeByteOrder(byteOrder);
+ final boolean nativeBOType = byteOrder == ByteOrder.nativeOrder();
final boolean byteBufferType = isByteBufferType();
final int type = MEMORY
| (readOnly ? READONLY : 0)
diff --git a/src/main/java/org/apache/datasketches/memory/internal/BaseWritableMemoryImpl.java b/src/main/java/org/apache/datasketches/memory/internal/BaseWritableMemoryImpl.java
index 2b9f052..e4527cc 100644
--- a/src/main/java/org/apache/datasketches/memory/internal/BaseWritableMemoryImpl.java
+++ b/src/main/java/org/apache/datasketches/memory/internal/BaseWritableMemoryImpl.java
@@ -136,7 +136,7 @@ public abstract class BaseWritableMemoryImpl extends BaseStateImpl implements Wr
scope); }
catch (final IllegalArgumentException | IllegalStateException | UnsupportedOperationException
| IOException | SecurityException e) { throw e; }
- final boolean nativeBOType = Util.isNativeByteOrder(byteOrder);
+ final boolean nativeBOType = byteOrder == ByteOrder.nativeOrder();
final int type = MEMORY | MAP | DIRECT
| (localReadOnly ? READONLY : 0)
| (nativeBOType ? NATIVE : NONNATIVE);
@@ -169,7 +169,7 @@ public abstract class BaseWritableMemoryImpl extends BaseStateImpl implements Wr
capacityBytes,
alignmentBytes,
scope);
- final boolean nativeBOType = Util.isNativeByteOrder(byteOrder);
+ final boolean nativeBOType = byteOrder == ByteOrder.nativeOrder();
final int type = MEMORY | DIRECT
| (nativeBOType ? NATIVE : NONNATIVE);
return nativeBOType
@@ -206,7 +206,7 @@ public abstract class BaseWritableMemoryImpl extends BaseStateImpl implements Wr
final boolean duplicateType = isDuplicateType();
final boolean mapType = seg.isMapped();
final boolean directType = seg.isNative();
- final boolean nativeBOType = Util.isNativeByteOrder(byteOrder);
+ final boolean nativeBOType = byteOrder == ByteOrder.nativeOrder();
final boolean byteBufferType = isByteBufferType();
final int type = MEMORY | REGION
| (readOnly ? READONLY : 0)
@@ -257,7 +257,7 @@ public abstract class BaseWritableMemoryImpl extends BaseStateImpl implements Wr
final boolean duplicateType = isDuplicateType();
final boolean mapType = seg.isMapped();
final boolean directType = seg.isNative();
- final boolean nativeBOType = Util.isNativeByteOrder(byteOrder);
+ final boolean nativeBOType = byteOrder == ByteOrder.nativeOrder();
final boolean byteBufferType = isByteBufferType();
final int type = BUFFER
| (readOnly ? READONLY : 0)
diff --git a/src/main/java/org/apache/datasketches/memory/internal/Ints.java b/src/main/java/org/apache/datasketches/memory/internal/Ints.java
deleted file mode 100644
index 72a447d..0000000
--- a/src/main/java/org/apache/datasketches/memory/internal/Ints.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.datasketches.memory.internal;
-
-/** Equivalent of Guava's Ints. */
-public final class Ints {
-
- private Ints() {}
-
- /**
- * Checks if a cast of a long to an int is within the range of an int
- * @param v the given long
- * @return returns the cast int, or throws an exception that the long was out-of-range of an int.
- */
- public static int checkedCast(final long v) {
- final int result = (int) v;
- if (result != v) {
- throw new IllegalArgumentException("Out of range: " + v);
- }
- return result;
- }
-}
diff --git a/src/main/java/org/apache/datasketches/memory/internal/Util.java b/src/main/java/org/apache/datasketches/memory/internal/Util.java
index 8d0e68d..b66b17f 100644
--- a/src/main/java/org/apache/datasketches/memory/internal/Util.java
+++ b/src/main/java/org/apache/datasketches/memory/internal/Util.java
@@ -24,13 +24,9 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
-import java.nio.ByteOrder;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Objects;
-import java.util.Random;
-
-import org.apache.datasketches.memory.Memory;
/**
* @author Lee Rhodes
@@ -39,119 +35,8 @@ import org.apache.datasketches.memory.Memory;
public final class Util {
public static final String LS = System.getProperty("line.separator");
- /**
- * Don't use sun.misc.Unsafe#copyMemory to copy blocks of memory larger than this
- * threshold, because internally it doesn't have safepoint polls, that may cause long
- * "Time To Safe Point" pauses in the application. This has been fixed in JDK 9 (see
- * https://bugs.openjdk.java.net/browse/JDK-8149596 and
- * https://bugs.openjdk.java.net/browse/JDK-8141491), but not in JDK 8, so the Memory library
- * should keep having this boilerplate as long as it supports Java 8.
- *
- * <p>A reference to this can be found in java.nio.Bits.</p>
- */
- public static final int UNSAFE_COPY_THRESHOLD_BYTES = 1024 * 1024;
-
private Util() { }
- //Byte Order Related
-
- /**
- * Returns true if the given byteOrder is the same as the native byte order.
- * @param byteOrder the given byte order
- * @return true if the given byteOrder is the same as the native byte order.
- */
- //TODO Duplicate in BaseStateImpl
- public static boolean isNativeByteOrder(final ByteOrder byteOrder) {
- if (byteOrder == null) {
- throw new IllegalArgumentException("ByteOrder parameter cannot be null.");
- }
- return ByteOrder.nativeOrder() == byteOrder;
- }
-
-
- /**
- * Searches a range of the specified array of longs for the specified value using the binary
- * search algorithm. The range must be sorted method) prior to making this call.
- * If it is not sorted, the results are undefined. If the range contains
- * multiple elements with the specified value, there is no guarantee which one will be found.
- * @param mem the Memory to be searched
- * @param fromLongIndex the index of the first element (inclusive) to be searched
- * @param toLongIndex the index of the last element (exclusive) to be searched
- * @param key the value to be searched for
- * @return index of the search key, if it is contained in the array within the specified range;
- * otherwise, (-(insertion point) - 1). The insertion point is defined as the point at which
- * the key would be inserted into the array: the index of the first element in the range greater
- * than the key, or toIndex if all elements in the range are less than the specified key.
- * Note that this guarantees that the return value will be ≥ 0 if and only if the key is found.
- */
- public static long binarySearchLongs(final Memory mem, final long fromLongIndex,
- final long toLongIndex, final long key) {
- //UnsafeUtil.checkBounds(fromLongIndex << 3, (toLongIndex - fromLongIndex) << 3, mem.getCapacity());
- long low = fromLongIndex;
- long high = toLongIndex - 1L;
-
- while (low <= high) {
- final long mid = (low + high) >>> 1;
- final long midVal = mem.getLong(mid << 3);
-
- if (midVal < key) { low = mid + 1; }
- else if (midVal > key) { high = mid - 1; }
- else { return mid; } // key found
- }
- return -(low + 1); // key not found.
- }
-
- /**
- * Prepend the given string with zeros. If the given string is equal or greater than the given
- * field length, it will be returned without modification.
- * @param s the given string
- * @param fieldLength desired total field length including the given string
- * @return the given string prepended with zeros.
- */
- public static final String zeroPad(final String s, final int fieldLength) {
- return characterPad(s, fieldLength, '0', false);
- }
-
- /**
- * Prepend or postpend the given string with the given character to fill the given field length.
- * If the given string is equal or greater than the given field length, it will be returned
- * without modification.
- * @param s the given string
- * @param fieldLength the desired field length
- * @param padChar the desired pad character
- * @param postpend if true append the pacCharacters to the end of the string.
- * @return prepended or postpended given string with the given character to fill the given field
- * length.
- */
- public static final String characterPad(final String s, final int fieldLength,
- final char padChar, final boolean postpend) {
- final char[] chArr = s.toCharArray();
- final int sLen = chArr.length;
- if (sLen < fieldLength) {
- final char[] out = new char[fieldLength];
- final int blanks = fieldLength - sLen;
-
- if (postpend) {
- for (int i = 0; i < sLen; i++) {
- out[i] = chArr[i];
- }
- for (int i = sLen; i < fieldLength; i++) {
- out[i] = padChar;
- }
- } else { //prepend
- for (int i = 0; i < blanks; i++) {
- out[i] = padChar;
- }
- for (int i = blanks; i < fieldLength; i++) {
- out[i] = chArr[i - blanks];
- }
- }
-
- return String.valueOf(out);
- }
- return s;
- }
-
/**
* Return true if all the masked bits of value are zero
* @param value the value to be tested
@@ -192,110 +77,7 @@ public final class Util {
return (value & bitMask) != 0;
}
- /**
- * Creates random valid Character Code Points (as integers). By definition, valid CodePoints
- * are integers in the range 0 to Character.MAX_CODE_POINT, and exclude the surrogate values.
- * This is used in unit testing and characterization testing of the UTF8 class. Because the
- * characterization tools are in a separate package, this must remain public.
- *
- * @author Lee Rhodes
- */
- public static class RandomCodePoints {
- private Random rand; //
- private static final int ALL_CP = Character.MAX_CODE_POINT + 1;
- private static final int MIN_SUR = Character.MIN_SURROGATE;
- private static final int MAX_SUR = Character.MAX_SURROGATE;
-
- /**
- * @param deterministic if true, configure java.util.Random with a fixed seed.
- */
- public RandomCodePoints(final boolean deterministic) {
- rand = deterministic ? new Random(0) : new Random();
- }
-
- /**
- * Fills the given array with random valid Code Points from 0, inclusive, to
- * <i>Character.MAX_CODE_POINT</i>, inclusive.
- * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to
- * <i>Character.MAX_SURROGATE</i>, inclusive, is always <i>excluded</i>.
- * @param cpArr the array to fill
- */
- public final void fillCodePointArray(final int[] cpArr) {
- fillCodePointArray(cpArr, 0, ALL_CP);
- }
-
- /**
- * Fills the given array with random valid Code Points from <i>startCP</i>, inclusive, to
- * <i>endCP</i>, exclusive.
- * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to
- * <i>Character.MAX_SURROGATE</i>, inclusive, is always <u>excluded</u>.
- * @param cpArr the array to fill
- * @param startCP the starting Code Point, included.
- * @param endCP the ending Code Point, excluded. This value cannot exceed 0x110000.
- */
- public final void fillCodePointArray(final int[] cpArr, final int startCP, final int endCP) {
- final int arrLen = cpArr.length;
- final int numCP = Math.min(endCP, 0X110000) - Math.min(0, startCP);
- int idx = 0;
- while (idx < arrLen) {
- final int cp = startCP + rand.nextInt(numCP);
- if ((cp >= MIN_SUR) && (cp <= MAX_SUR)) {
- continue;
- }
- cpArr[idx++] = cp;
- }
- }
-
- /**
- * Return a single valid random Code Point from 0, inclusive, to
- * <i>Character.MAX_CODE_POINT</i>, inclusive.
- * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to
- * <i>Character.MAX_SURROGATE</i>, inclusive, is always <u>excluded</u>.
- * @return a single valid random CodePoint.
- */
- public final int getCodePoint() {
- return getCodePoint(0, ALL_CP);
- }
-
- /**
- * Return a single valid random Code Point from <i>startCP</i>, inclusive, to
- * <i>endCP</i>, exclusive.
- * The surrogate range, which is from <i>Character.MIN_SURROGATE</i>, inclusive, to
- * <i>Character.MAX_SURROGATE</i>, inclusive, is always <u>excluded</u>.
- * @param startCP the starting Code Point, included.
- * @param endCP the ending Code Point, excluded. This value cannot exceed 0x110000.
- * @return a single valid random CodePoint.
- */
- public final int getCodePoint(final int startCP, final int endCP) {
- final int numCP = Math.min(endCP, 0X110000) - Math.min(0, startCP);
- while (true) {
- final int cp = startCP + rand.nextInt(numCP);
- if ((cp < MIN_SUR) || (cp > MAX_SUR)) {
- return cp;
- }
- }
- }
- } //End class RandomCodePoints
-
- public static final void zeroCheck(final long value, final String arg) {
- if (value <= 0) {
- throw new IllegalArgumentException("The argument '" + arg + "' must not be negative or zero.");
- }
- }
-
- public static final void negativeCheck(final long value, final String arg) {
- if (value < 0) {
- throw new IllegalArgumentException("The argument '" + arg + "' must not be negative.");
- }
- }
-
- public static final void nullCheck(final Object obj, final String arg) {
- if (obj == null) {
- throw new IllegalArgumentException("The argument '" + arg + "' must not be null.");
- }
- }
-
- //Resources NOTE: these 3 methods are duplicated in Java/ datasketches/Util
+ //Resources mention: these 3 methods are duplicated in Java/ datasketches/Util
/**
* Gets the absolute path of the given resource file's shortName.
diff --git a/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java b/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java
index 7752c76..2b6004a 100644
--- a/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java
+++ b/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java
@@ -23,16 +23,9 @@
package org.apache.datasketches.memory.internal;
-import static org.apache.datasketches.memory.internal.Util.characterPad;
import static org.apache.datasketches.memory.internal.Util.getResourceBytes;
import static org.apache.datasketches.memory.internal.Util.getResourceFile;
-import static org.apache.datasketches.memory.internal.Util.negativeCheck;
-import static org.apache.datasketches.memory.internal.Util.nullCheck;
-import static org.apache.datasketches.memory.internal.Util.zeroCheck;
-import static org.apache.datasketches.memory.internal.Util.zeroPad;
-import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
import java.io.File;
import java.io.IOException;
@@ -42,93 +35,11 @@ import java.nio.file.attribute.PosixFileAttributeView;
import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.PosixFilePermissions;
-import org.apache.datasketches.memory.WritableMemory;
import org.testng.annotations.Test;
public class UtilTest {
private static final String LS = System.getProperty("line.separator");
- //Binary Search
- @Test
- public void checkBinarySearch() {
- int k = 1024; //longs
- WritableMemory wMem = WritableMemory.allocate(k << 3); //1024 longs
- for (int i = 0; i < k; i++) { wMem.putLong(i << 3, i); }
- long idx = Util.binarySearchLongs(wMem, 0, k - 1, k / 2);
- long val = wMem.getLong(idx << 3);
- assertEquals(idx, k/2);
- assertEquals(val, k/2);
-
- idx = Util.binarySearchLongs(wMem, 0, k - 1, k);
- assertEquals(idx, -1024);
- }
-
-// @Test(expectedExceptions = IllegalArgumentException.class)
-// public void checkBoundsTest() {
-// UnsafeUtil.checkBounds(999, 2, 1000);
-// }
-
- @Test
- public void checkPadding() {
- String s = "123";
- String t = zeroPad(s, 4);
- assertTrue(t.startsWith("0"));
-
- t = characterPad(s, 4, '0', true);
- assertTrue(t.endsWith("0"));
-
- t = characterPad(s, 3, '0', false);
- assertEquals(s, t);
- }
-
- @Test
- public void checkNullZeroNegativeChecks() {
- Object obj = null;
- try {
- nullCheck(obj, "Test Object");
- fail();
- } catch (IllegalArgumentException e) {
- //OK
- }
- try {
- zeroCheck(0, "Test Long");
- fail();
- } catch (IllegalArgumentException e) {
- //OK
- }
- try {
- negativeCheck(-1L, "Test Long");
- fail();
- } catch (IllegalArgumentException e) {
- //OK
- }
- }
-
- @Test
- public void checkCodePointArr() {
- final Util.RandomCodePoints rvcp = new Util.RandomCodePoints(true);
- final int n = 1000;
- final int[] cpArr = new int[n];
- rvcp.fillCodePointArray(cpArr);
- for (int i = 0; i < n; i++) {
- int cp = cpArr[i];
- if ((cp >= Character.MIN_SURROGATE) && (cp <= Character.MAX_SURROGATE)) {
- fail();
- }
- }
- }
-
- @Test
- public void checkCodePoint() {
- final Util.RandomCodePoints rvcp = new Util.RandomCodePoints(true);
- final int n = 1000;
- for (int i = 0; i < n; i++) {
- int cp = rvcp.getCodePoint();
- if ((cp >= Character.MIN_SURROGATE) && (cp <= Character.MAX_SURROGATE)) {
- fail();
- }
- }
- }
static final String getFileAttributes(File file) {
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org