You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2022/06/20 00:01:16 UTC
[datasketches-memory] 01/02: Interim 1
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch move-java8test-to-java8
in repository https://gitbox.apache.org/repos/asf/datasketches-memory.git
commit 3771dd7ab37a415a49f0b5c536a7cfa21ffa2b70
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Fri Jun 17 16:55:08 2022 -0700
Interim 1
---
datasketches-memory-java8-tests/pom.xml | 27 +--
datasketches-memory-java8/pom.xml | 54 ++----
.../internal}/AllocateDirectMapMemoryTest.java | 4 +-
.../memory/internal/AllocateDirectMemoryTest.java | 131 +++++++++++++
.../datasketches/memory/internal/UtilTest.java | 206 +++++++++++++++++++++
.../src/test/resources/GettysburgAddress.txt | 7 +
6 files changed, 379 insertions(+), 50 deletions(-)
diff --git a/datasketches-memory-java8-tests/pom.xml b/datasketches-memory-java8-tests/pom.xml
index 11bff9c..11c853d 100644
--- a/datasketches-memory-java8-tests/pom.xml
+++ b/datasketches-memory-java8-tests/pom.xml
@@ -17,8 +17,9 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
- http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -28,6 +29,17 @@
<version>2.2.0-SNAPSHOT</version>
</parent>
+ <artifactId>datasketches-memory-java8-tests</artifactId>
+ <name>${project.artifactId}</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <java.version>1.8</java.version>
+ <jdk-toolchain.version>8</jdk-toolchain.version>
+ <maven.compiler.source>${java.version}</maven.compiler.source>
+ <maven.compiler.target>${java.version}</maven.compiler.target>
+ </properties>
+
<dependencies>
<!-- Test Scope -->
<dependency>
@@ -55,15 +67,4 @@
</dependency>
</dependencies>
- <artifactId>datasketches-memory-java8-tests</artifactId>
- <name>${project.artifactId}</name>
- <packaging>jar</packaging>
-
- <properties>
- <java.version>1.8</java.version>
- <jdk-toolchain.version>8</jdk-toolchain.version>
- <maven.compiler.source>${java.version}</maven.compiler.source>
- <maven.compiler.target>${java.version}</maven.compiler.target>
- </properties>
-
</project>
diff --git a/datasketches-memory-java8/pom.xml b/datasketches-memory-java8/pom.xml
index 42deb9a..c8d0a1d 100644
--- a/datasketches-memory-java8/pom.xml
+++ b/datasketches-memory-java8/pom.xml
@@ -41,40 +41,24 @@
<maven.javadoc.skip>false</maven.javadoc.skip>
</properties>
- <profiles>
-
- <profile>
- <id>strict</id>
- <!-- The error prone compiler is only currently supported for compilation in JDK 8 -->
- <activation>
- <jdk>[1.8, 1.9),8</jdk>
- </activation>
- <build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>${maven-compiler-plugin.version}</version>
- <dependencies>
- <dependency>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-compiler-javac-errorprone</artifactId>
- <version>${plexus-compiler-javac-errorprone.version}</version>
- </dependency>
- </dependencies>
- <configuration>
- <source>${maven.compiler.source}</source>
- <target>${maven.compiler.target}</target>
- <compilerId>javac-with-errorprone</compilerId>
- <forceJavacCompilerUse>true</forceJavacCompilerUse>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
- </profile>
-
- </profiles>
+ <dependencies>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <!-- Used for UTF8 testing -->
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <!-- Used for xxHash testing -->
+ <groupId>net.openhft</groupId>
+ <artifactId>zero-allocation-hashing</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
</project>
diff --git a/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/AllocateDirectMapMemoryTest.java b/datasketches-memory-java8/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMapMemoryTest.java
similarity index 97%
rename from datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/AllocateDirectMapMemoryTest.java
rename to datasketches-memory-java8/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMapMemoryTest.java
index a197e01..4911e7d 100644
--- a/datasketches-memory-java8-tests/src/test/java/org/apache/datasketches/memory/test/AllocateDirectMapMemoryTest.java
+++ b/datasketches-memory-java8/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMapMemoryTest.java
@@ -21,7 +21,7 @@
* Note: Lincoln's Gettysburg Address is in the public domain. See LICENSE.
*/
-package org.apache.datasketches.memory.test;
+package org.apache.datasketches.memory.internal;
import static org.apache.datasketches.memory.internal.Util.*;
import static org.testng.Assert.assertEquals;
@@ -50,7 +50,7 @@ public class AllocateDirectMapMemoryTest {
@Test
public void simpleMap() throws Exception {
File file = getResourceFile("GettysburgAddress.txt");
- assertTrue(ReflectUtil.isFileReadOnly(file));
+ assertTrue(AllocateDirectMap.isFileReadOnly(file));
try (MapHandle rh = Memory.map(file)) {
rh.close();
}
diff --git a/datasketches-memory-java8/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java b/datasketches-memory-java8/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java
new file mode 100644
index 0000000..f14e3ec
--- /dev/null
+++ b/datasketches-memory-java8/src/test/java/org/apache/datasketches/memory/internal/AllocateDirectMemoryTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.memory.internal;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.fail;
+
+import static org.apache.datasketches.memory.BaseState.*;
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.MemoryRequestServer;
+import org.apache.datasketches.memory.WritableHandle;
+import org.apache.datasketches.memory.WritableMemory;
+import org.apache.datasketches.memory.internal.Util;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.Test;
+
+public class AllocateDirectMemoryTest {
+
+ @Test
+ public void simpleAllocateDirect() throws Exception {
+ int longs = 32;
+ WritableMemory wMem;
+ try (WritableHandle wh = WritableMemory.allocateDirect(longs << 3)) {
+ wMem = wh.getWritable();
+ for (int i = 0; i<longs; i++) {
+ wMem.putLong(i << 3, i);
+ assertEquals(wMem.getLong(i << 3), i);
+ }
+ //inside the TWR block the memory should be valid
+ BaseState.checkValid(wMem);
+ //OK
+ }
+ //The TWR block has exited, so the memory should be invalid
+ try {
+ ReflectUtil.checkValid(wMem);
+ fail();
+ } catch (final RuntimeException e) {
+ //OK
+ }
+ }
+
+ @Test
+ public void checkDefaultMemoryRequestServer() throws Exception {
+ int longs1 = 32;
+ int bytes1 = longs1 << 3;
+ try (WritableHandle wh = WritableMemory.allocateDirect(bytes1)) {
+ WritableMemory origWmem = wh.getWritable();
+ for (int i = 0; i<longs1; i++) { //puts data in wMem1
+ origWmem.putLong(i << 3, i);
+ assertEquals(origWmem.getLong(i << 3), i);
+ }
+ println(origWmem.toHexString("Test", 0, 32 * 8));
+
+ int longs2 = 64;
+ int bytes2 = longs2 << 3;
+ MemoryRequestServer memReqSvr;
+ if (BaseState.defaultMemReqSvr == null) {
+ memReqSvr = new DefaultMemoryRequestServer();
+ } else {
+ memReqSvr = origWmem.getMemoryRequestServer();
+ }
+ WritableMemory newWmem = memReqSvr.request(origWmem, bytes2);
+ assertFalse(newWmem.isDirect()); //on heap by default
+ for (int i = 0; i < longs2; i++) {
+ newWmem.putLong(i << 3, i);
+ assertEquals(newWmem.getLong(i << 3), i);
+ }
+ memReqSvr.requestClose(origWmem, newWmem);
+ //The default MRS doesn't actually release because it could be easily misused.
+ // So we let the TWR release it.
+ }
+ }
+
+ @Test
+ public void checkNonNativeDirect() throws Exception {
+ try (WritableHandle h = WritableMemory.allocateDirect(128, Util.NON_NATIVE_BYTE_ORDER, null)) {
+ WritableMemory wmem = h.getWritable();
+ wmem.putChar(0, (char) 1);
+ assertEquals(wmem.getByte(1), (byte) 1);
+ }
+ }
+
+ @Test
+ public void checkExplicitClose() throws Exception {
+ final long cap = 128;
+ try (WritableHandle wdh = WritableMemory.allocateDirect(cap)) {
+ wdh.close(); //explicit close. Does the work of closing
+ } //end of scope call to Cleaner/Deallocator also will be redundant
+ }
+
+
+ @AfterClass
+ public void checkDirectCounter() {
+ WritableMemory.writableWrap(new byte[8]);
+ long count = BaseState.getCurrentDirectMemoryAllocations();
+ if (count != 0) {
+ println(""+count);
+ fail();
+ }
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: "+this.getClass().getName());
+ }
+
+ /**
+ * @param s value to print
+ */
+ static void println(String s) {
+ //System.out.println(s); //disable here
+ }
+}
diff --git a/datasketches-memory-java8/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java b/datasketches-memory-java8/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java
new file mode 100644
index 0000000..83a8352
--- /dev/null
+++ b/datasketches-memory-java8/src/test/java/org/apache/datasketches/memory/internal/UtilTest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Note: Lincoln's Gettysburg Address is in the public domain. See LICENSE.
+ */
+
+package org.apache.datasketches.memory.internal;
+
+import static org.apache.datasketches.memory.internal.Util.characterPad;
+import static org.apache.datasketches.memory.internal.Util.getResourceBytes;
+import static org.apache.datasketches.memory.internal.Util.getResourceFile;
+import static org.apache.datasketches.memory.internal.Util.negativeCheck;
+import static org.apache.datasketches.memory.internal.Util.nullCheck;
+import static org.apache.datasketches.memory.internal.Util.zeroCheck;
+import static org.apache.datasketches.memory.internal.Util.zeroPad;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFileAttributes;
+import java.nio.file.attribute.PosixFilePermissions;
+
+import org.apache.datasketches.memory.internal.UnsafeUtil;
+import org.apache.datasketches.memory.internal.Util;
+import org.apache.datasketches.memory.WritableMemory;
+import org.testng.annotations.Test;
+
+public class UtilTest {
+ private static final String LS = System.getProperty("line.separator");
+
+ //Binary Search
+ @Test
+ public void checkBinarySearch() {
+ int k = 1024; //longs
+ WritableMemory wMem = WritableMemory.allocate(k << 3); //1024 longs
+ for (int i = 0; i < k; i++) { wMem.putLong(i << 3, i); }
+ long idx = Util.binarySearchLongs(wMem, 0, k - 1, k / 2);
+ long val = wMem.getLong(idx << 3);
+ assertEquals(idx, k/2);
+ assertEquals(val, k/2);
+
+ idx = Util.binarySearchLongs(wMem, 0, k - 1, k);
+ assertEquals(idx, -1024);
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class)
+ public void checkBoundsTest() {
+ UnsafeUtil.checkBounds(999, 2, 1000);
+ }
+
+ @Test
+ public void checkPadding() {
+ String s = "123";
+ String t = zeroPad(s, 4);
+ assertTrue(t.startsWith("0"));
+
+ t = characterPad(s, 4, '0', true);
+ assertTrue(t.endsWith("0"));
+
+ t = characterPad(s, 3, '0', false);
+ assertEquals(s, t);
+ }
+
+ @Test
+ public void checkNullZeroNegativeChecks() {
+ Object obj = null;
+ try {
+ nullCheck(obj, "Test Object");
+ fail();
+ } catch (IllegalArgumentException e) {
+ //OK
+ }
+ try {
+ zeroCheck(0, "Test Long");
+ fail();
+ } catch (IllegalArgumentException e) {
+ //OK
+ }
+ try {
+ negativeCheck(-1L, "Test Long");
+ fail();
+ } catch (IllegalArgumentException e) {
+ //OK
+ }
+ }
+
+ @Test
+ public void checkCodePointArr() {
+ final Util.RandomCodePoints rvcp = new Util.RandomCodePoints(true);
+ final int n = 1000;
+ final int[] cpArr = new int[n];
+ rvcp.fillCodePointArray(cpArr);
+ for (int i = 0; i < n; i++) {
+ int cp = cpArr[i];
+ if ((cp >= Character.MIN_SURROGATE) && (cp <= Character.MAX_SURROGATE)) {
+ fail();
+ }
+ }
+ }
+
+ @Test
+ public void checkCodePoint() {
+ final Util.RandomCodePoints rvcp = new Util.RandomCodePoints(true);
+ final int n = 1000;
+ for (int i = 0; i < n; i++) {
+ int cp = rvcp.getCodePoint();
+ if ((cp >= Character.MIN_SURROGATE) && (cp <= Character.MAX_SURROGATE)) {
+ fail();
+ }
+ }
+ }
+
+ static final String getFileAttributes(File file) {
+ try {
+ PosixFileAttributes attrs = Files.getFileAttributeView(
+ file.toPath(), PosixFileAttributeView.class, new LinkOption[0]).readAttributes();
+ String s = String.format("%s: %s %s %s%n",
+ file.getPath(),
+ attrs.owner().getName(),
+ attrs.group().getName(),
+ PosixFilePermissions.toString(attrs.permissions()));
+ return s;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static final void setGettysburgAddressFileToReadOnly() {
+ File file = getResourceFile("GettysburgAddress.txt");
+ try {
+ Files.setPosixFilePermissions(file.toPath(), PosixFilePermissions.fromString("r--r--r--"));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ //Resources
+
+ @Test
+ public void resourceFileExits() {
+ final String shortFileName = "GettysburgAddress.txt";
+ final File file = getResourceFile(shortFileName);
+ assertTrue(file.exists());
+ }
+
+ @Test(expectedExceptions = NullPointerException.class)
+ public void resourceFileNotFound() {
+ final String shortFileName = "GettysburgAddress.txt";
+ getResourceFile(shortFileName + "123");
+ }
+
+ @Test
+ public void resourceBytesCorrect() {
+ final String shortFileName = "GettysburgAddress.txt";
+ final byte[] bytes = getResourceBytes(shortFileName);
+ assertTrue(bytes.length == 1541);
+ }
+
+ @Test(expectedExceptions = NullPointerException.class)
+ public void resourceBytesFileNotFound() {
+ final String shortFileName = "GettysburgAddress.txt";
+ getResourceBytes(shortFileName + "123");
+ }
+
+ @Test
+ public void printlnTest() {
+ println("PRINTING: "+this.getClass().getName());
+ }
+
+ static void println(final Object o) {
+ if (o == null) { print(LS); }
+ else { print(o.toString() + LS); }
+ }
+
+ /**
+ * @param o value to print
+ */
+ static void print(final Object o) {
+ if (o != null) {
+ //System.out.print(o.toString()); //disable here
+ }
+ }
+
+}
diff --git a/datasketches-memory-java8/src/test/resources/GettysburgAddress.txt b/datasketches-memory-java8/src/test/resources/GettysburgAddress.txt
new file mode 100644
index 0000000..3969d17
--- /dev/null
+++ b/datasketches-memory-java8/src/test/resources/GettysburgAddress.txt
@@ -0,0 +1,7 @@
+Abraham Lincoln's Gettysburg Address:
+
+ Four score and seven years ago our fathers brought forth on this continent, a new nation, conceived in Liberty, and dedicated to the proposition that all men are created equal.
+
+ Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting place for those who here gave their lives that that nation might live. It is altogether fitting and proper that we should do this.
+
+ But, in a larger sense, we can not dedicate —- we can not consecrate —- we can not hallow —- this ground. The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us [...]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org