You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2021/08/20 12:38:13 UTC
[ignite-3] branch ignite-15163 created (now b48b448)
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a change to branch ignite-15163
in repository https://gitbox.apache.org/repos/asf/ignite-3.git.
at b48b448 Support BitSet, Number and Decimal types.
This branch includes the following new commits:
new b48b448 Support BitSet, Number and Decimal types.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[ignite-3] 01/01: Support BitSet, Number and Decimal types.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch ignite-15163
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
commit b48b4485824d242994d62c208deee542378c58c0
Author: Andrew Mashenkov <an...@gmail.com>
AuthorDate: Fri Aug 20 15:36:05 2021 +0300
Support BitSet, Number and Decimal types.
---
.../apache/ignite/client/proto/ClientDataType.java | 3 +
.../ignite/client/proto/ClientMessagePacker.java | 50 ++++++++++--
.../ignite/client/proto/ClientMessageUnpacker.java | 60 ++++++++++++--
.../ignite/client/proto/ClientMsgPackType.java | 3 +
.../proto/ClientMessagePackerUnpackerTest.java | 95 +++++++++++++++++++++-
.../internal/testframework/IgniteTestUtils.java | 13 +++
modules/schema/pom.xml | 7 ++
.../apache/ignite/internal/schema/TestUtils.java | 15 +---
.../schema/marshaller/JavaSerializerTest.java | 2 +-
.../marshaller/reflection/FieldAccessorTest.java | 2 +-
10 files changed, 219 insertions(+), 31 deletions(-)
diff --git a/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientDataType.java b/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientDataType.java
index 45d1280..d744987 100644
--- a/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientDataType.java
+++ b/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientDataType.java
@@ -53,4 +53,7 @@ public class ClientDataType {
/** BitMask. */
public static final int BITMASK = 11;
+
+ /** Number. */
+ public static final int NUMBER = 12;
}
diff --git a/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMessagePacker.java b/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMessagePacker.java
index facefa2..bbe0758 100644
--- a/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMessagePacker.java
+++ b/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMessagePacker.java
@@ -17,6 +17,8 @@
package org.apache.ignite.client.proto;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufOutputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.math.BigDecimal;
@@ -24,8 +26,6 @@ import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.UUID;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufOutputStream;
import org.msgpack.core.MessagePack;
import org.msgpack.core.MessagePacker;
import org.msgpack.core.buffer.OutputStreamBufferOutput;
@@ -316,7 +316,7 @@ public class ClientMessagePacker extends MessagePacker {
bb.putLong(val.getMostSignificantBits());
bb.putLong(val.getLeastSignificantBits());
- writePayload(bytes);
+ addPayload(bytes);
return this;
}
@@ -326,25 +326,58 @@ public class ClientMessagePacker extends MessagePacker {
*
* @param val Decimal value.
* @return This instance.
- * @throws UnsupportedOperationException Not supported.
*/
public ClientMessagePacker packDecimal(BigDecimal val) {
assert !closed : "Packer is closed";
- throw new UnsupportedOperationException("TODO: IGNITE-15163");
+ // TODO: Pack directly to ByteBuf without allocating IGNITE-15234.
+ byte[] unscaledValue = val.unscaledValue().toByteArray();
+
+ packExtensionTypeHeader(ClientMsgPackType.DECIMAL, 4 + unscaledValue.length); // Scale length + data length
+
+ addPayload(ByteBuffer.wrap(new byte[4]).putInt(val.scale()).array());
+ addPayload(unscaledValue);
+
+ return this;
}
/**
+ * Writes a decimal.
+ *
+ * @param val Decimal value.
+ * @return This instance.
+ */
+ public ClientMessagePacker packNumber(BigInteger val) {
+ assert !closed : "Packer is closed";
+
+ // TODO: Pack directly to ByteBuf without allocating IGNITE-15234.
+ byte[] data = val.toByteArray();
+
+ packExtensionTypeHeader(ClientMsgPackType.NUMBER, data.length);
+
+ addPayload(data);
+
+ return this;
+ }
+
+
+ /**
* Writes a bit set.
*
* @param val Bit set value.
* @return This instance.
- * @throws UnsupportedOperationException Not supported.
*/
public ClientMessagePacker packBitSet(BitSet val) {
assert !closed : "Packer is closed";
- throw new UnsupportedOperationException("TODO: IGNITE-15163");
+ // TODO: Pack directly to ByteBuf without allocating IGNITE-15234.
+ byte[] data = val.toByteArray();
+
+ packExtensionTypeHeader(ClientMsgPackType.BITMASK, data.length);
+
+ addPayload(data);
+
+ return this;
}
/**
@@ -381,6 +414,9 @@ public class ClientMessagePacker extends MessagePacker {
if (val instanceof BigDecimal)
return packDecimal((BigDecimal) val);
+ if (val instanceof BigInteger)
+ return packNumber((BigInteger)val);
+
if (val instanceof BitSet)
return packBitSet((BitSet) val);
diff --git a/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMessageUnpacker.java b/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMessageUnpacker.java
index a058c9c..c0c6aec 100644
--- a/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMessageUnpacker.java
+++ b/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMessageUnpacker.java
@@ -17,6 +17,8 @@
package org.apache.ignite.client.proto;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufInputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.math.BigDecimal;
@@ -24,8 +26,6 @@ import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.UUID;
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufInputStream;
import org.apache.ignite.lang.IgniteException;
import org.msgpack.core.ExtensionTypeHeader;
import org.msgpack.core.MessageFormat;
@@ -45,6 +45,7 @@ import static org.apache.ignite.client.proto.ClientDataType.INT16;
import static org.apache.ignite.client.proto.ClientDataType.INT32;
import static org.apache.ignite.client.proto.ClientDataType.INT64;
import static org.apache.ignite.client.proto.ClientDataType.INT8;
+import static org.apache.ignite.client.proto.ClientDataType.NUMBER;
import static org.apache.ignite.client.proto.ClientDataType.STRING;
/**
@@ -325,7 +326,7 @@ public class ClientMessageUnpacker extends MessageUnpacker {
var len = hdr.getLength();
if (type != ClientMsgPackType.UUID)
- throw new MessageTypeException("Expected UUID extension (1), but got " + type);
+ throw new MessageTypeException("Expected UUID extension (3), but got " + type);
if (len != 16)
throw new MessageSizeException("Expected 16 bytes for UUID extension, but got " + len, len);
@@ -346,19 +347,60 @@ public class ClientMessageUnpacker extends MessageUnpacker {
public BigDecimal unpackDecimal() {
assert refCnt > 0 : "Unpacker is closed";
- throw new UnsupportedOperationException("TODO: IGNITE-15163");
+ var hdr = unpackExtensionTypeHeader();
+ var type = hdr.getType();
+ var len = hdr.getLength();
+
+ if (type != ClientMsgPackType.DECIMAL)
+ throw new MessageTypeException("Expected DECIMAL extension (2), but got " + type);
+
+ var bytes = readPayload(len);
+
+ ByteBuffer bb = ByteBuffer.wrap(bytes);
+
+ int scale = bb.getInt();
+
+ return new BigDecimal(new BigInteger(bytes, bb.position(), bb.remaining()), scale);
}
/**
* Reads a bit set.
*
* @return Bit set.
- * @throws UnsupportedOperationException Not supported yet.
*/
public BitSet unpackBitSet() {
assert refCnt > 0 : "Unpacker is closed";
- throw new UnsupportedOperationException("TODO: IGNITE-15163");
+ var hdr = unpackExtensionTypeHeader();
+ var type = hdr.getType();
+ var len = hdr.getLength();
+
+ if (type != ClientMsgPackType.BITMASK)
+ throw new MessageTypeException("Expected DECIMAL extension (7), but got " + type);
+
+ var bytes = readPayload(len);
+
+ return BitSet.valueOf(bytes);
+ }
+
+ /**
+ * Reads a big number.
+ *
+ * @return Bit set.
+ */
+ public BigInteger unpackNumber() {
+ assert refCnt > 0 : "Unpacker is closed";
+
+ var hdr = unpackExtensionTypeHeader();
+ var type = hdr.getType();
+ var len = hdr.getLength();
+
+ if (type != ClientMsgPackType.NUMBER)
+ throw new MessageTypeException("Expected NUMBER extension (1), but got " + type);
+
+ var bytes = readPayload(len);
+
+ return new BigInteger(bytes);
}
/**
@@ -398,16 +440,20 @@ public class ClientMessageUnpacker extends MessageUnpacker {
case STRING:
return unpackString();
- case BYTES:
+ case BYTES: {
var cnt = unpackBinaryHeader();
return readPayload(cnt);
+ }
case DECIMAL:
return unpackDecimal();
case BITMASK:
return unpackBitSet();
+
+ case NUMBER:
+ return unpackNumber();
}
throw new IgniteException("Unknown client data type: " + dataType);
diff --git a/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMsgPackType.java b/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMsgPackType.java
index dd92c46..7db9de5 100644
--- a/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMsgPackType.java
+++ b/modules/client-common/src/main/java/org/apache/ignite/client/proto/ClientMsgPackType.java
@@ -38,4 +38,7 @@ public class ClientMsgPackType {
/** DateTime. */
public static final byte DATETIME = 6;
+
+ /** Bit mask. */
+ public static final byte BITMASK = 7;
}
diff --git a/modules/client-common/src/test/java/org/apache/ignite/client/proto/ClientMessagePackerUnpackerTest.java b/modules/client-common/src/test/java/org/apache/ignite/client/proto/ClientMessagePackerUnpackerTest.java
index 3d04434..92f0e55 100644
--- a/modules/client-common/src/test/java/org/apache/ignite/client/proto/ClientMessagePackerUnpackerTest.java
+++ b/modules/client-common/src/test/java/org/apache/ignite/client/proto/ClientMessagePackerUnpackerTest.java
@@ -17,12 +17,18 @@
package org.apache.ignite.client.proto;
-import java.io.IOException;
-import java.util.UUID;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.Unpooled;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.BitSet;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.internal.testframework.IgniteTestUtils;
import org.junit.jupiter.api.Test;
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.randomBytes;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
@@ -73,6 +79,55 @@ public class ClientMessagePackerUnpackerTest {
testUUID(new UUID(0, 0));
}
+ @Test
+ public void testNumber() throws IOException {
+ testNumber(BigInteger.ZERO);
+ testNumber(BigInteger.valueOf(Long.MIN_VALUE));
+ testNumber(BigInteger.valueOf(Long.MAX_VALUE));
+
+ testNumber(new BigInteger(randomBytes(ThreadLocalRandom.current(), 100)));
+ testNumber(new BigInteger(randomBytes(ThreadLocalRandom.current(), 250)));
+ testNumber(new BigInteger(randomBytes(ThreadLocalRandom.current(), 1000)));
+ }
+
+ @Test
+ public void testDecimal() throws IOException {
+ testDecimal(BigDecimal.ZERO);
+ testDecimal(BigDecimal.valueOf(Long.MIN_VALUE));
+ testDecimal(BigDecimal.valueOf(Long.MAX_VALUE));
+
+ testDecimal(new BigDecimal(new BigInteger(randomBytes(ThreadLocalRandom.current(), 100)), 50));
+ testDecimal(new BigDecimal(new BigInteger(randomBytes(ThreadLocalRandom.current(), 250)), 200));
+ testDecimal(new BigDecimal(new BigInteger(randomBytes(ThreadLocalRandom.current(), 1000)), 500));
+ }
+
+
+ @Test
+ public void testBitSet() throws IOException {
+ testBitSet(BitSet.valueOf(new byte[0]));
+ testBitSet(BitSet.valueOf(randomBytes(ThreadLocalRandom.current(), 1)));
+ testBitSet(BitSet.valueOf(randomBytes(ThreadLocalRandom.current(), 100)));
+ testBitSet(BitSet.valueOf(randomBytes(ThreadLocalRandom.current(), 1000)));
+ }
+
+ private void testBitSet(BitSet val) {
+ try (var packer = new ClientMessagePacker(PooledByteBufAllocator.DEFAULT.directBuffer())) {
+ packer.packBitSet(val);
+
+ var buf = packer.getBuffer();
+ var len = buf.readInt();
+
+ byte[] data = new byte[buf.readableBytes()];
+ buf.readBytes(data);
+
+ try (var unpacker = new ClientMessageUnpacker(Unpooled.wrappedBuffer(data))) {
+ var res = unpacker.unpackBitSet();
+
+ assertEquals(val, res);
+ }
+ }
+ }
+
private void testUUID(UUID u) throws IOException {
try (var packer = new ClientMessagePacker(PooledByteBufAllocator.DEFAULT.directBuffer())) {
packer.packUuid(u);
@@ -91,4 +146,40 @@ public class ClientMessagePackerUnpackerTest {
}
}
}
+
+ private void testNumber(BigInteger val) {
+ try (var packer = new ClientMessagePacker(PooledByteBufAllocator.DEFAULT.directBuffer())) {
+ packer.packNumber(val);
+
+ var buf = packer.getBuffer();
+ var len = buf.readInt();
+
+ byte[] data = new byte[buf.readableBytes()];
+ buf.readBytes(data);
+
+ try (var unpacker = new ClientMessageUnpacker(Unpooled.wrappedBuffer(data))) {
+ var res = unpacker.unpackNumber();
+
+ assertEquals(val, res);
+ }
+ }
+ }
+
+ private void testDecimal(BigDecimal val) {
+ try (var packer = new ClientMessagePacker(PooledByteBufAllocator.DEFAULT.directBuffer())) {
+ packer.packDecimal(val);
+
+ var buf = packer.getBuffer();
+ var len = buf.readInt();
+
+ byte[] data = new byte[buf.readableBytes()];
+ buf.readBytes(data);
+
+ try (var unpacker = new ClientMessageUnpacker(Unpooled.wrappedBuffer(data))) {
+ var res = unpacker.unpackDecimal();
+
+ assertEquals(val, res);
+ }
+ }
+ }
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/testframework/IgniteTestUtils.java b/modules/core/src/test/java/org/apache/ignite/internal/testframework/IgniteTestUtils.java
index 838b7eb..dfceebd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/testframework/IgniteTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/testframework/IgniteTestUtils.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.testframework;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
+import java.util.Random;
import java.util.function.BooleanSupplier;
import org.apache.ignite.lang.IgniteInternalException;
import org.jetbrains.annotations.NotNull;
@@ -207,4 +208,16 @@ public final class IgniteTestUtils {
return false;
}
+
+ /**
+ * @param rnd Random generator.
+ * @param len Byte array length.
+ * @return Random byte array.
+ */
+ public static byte[] randomBytes(Random rnd, int len) {
+ byte[] data = new byte[len];
+ rnd.nextBytes(data);
+
+ return data;
+ }
}
diff --git a/modules/schema/pom.xml b/modules/schema/pom.xml
index cb3f3c7..6faa0d5 100644
--- a/modules/schema/pom.xml
+++ b/modules/schema/pom.xml
@@ -68,6 +68,13 @@
</dependency>
<dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-core</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
diff --git a/modules/schema/src/test/java/org/apache/ignite/internal/schema/TestUtils.java b/modules/schema/src/test/java/org/apache/ignite/internal/schema/TestUtils.java
index 878fb76..8758b5b 100644
--- a/modules/schema/src/test/java/org/apache/ignite/internal/schema/TestUtils.java
+++ b/modules/schema/src/test/java/org/apache/ignite/internal/schema/TestUtils.java
@@ -21,6 +21,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.BitSet;
import java.util.Random;
+import org.apache.ignite.internal.testframework.IgniteTestUtils;
/**
* Test utility class.
@@ -60,7 +61,7 @@ public final class TestUtils {
return randomString(rnd, rnd.nextInt(255));
case BYTES:
- return randomBytes(rnd, rnd.nextInt(255));
+ return IgniteTestUtils.randomBytes(rnd, rnd.nextInt(255));
case NUMBER:
return BigInteger.probablePrime(12, rnd);
@@ -97,18 +98,6 @@ public final class TestUtils {
/**
* @param rnd Random generator.
- * @param len Byte array length.
- * @return Random byte array.
- */
- public static byte[] randomBytes(Random rnd, int len) {
- byte[] data = new byte[len];
- rnd.nextBytes(data);
-
- return data;
- }
-
- /**
- * @param rnd Random generator.
* @param len String length.
* @return Random string.
*/
diff --git a/modules/schema/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java b/modules/schema/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java
index 84e32da..c25eef3 100644
--- a/modules/schema/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java
+++ b/modules/schema/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java
@@ -476,7 +476,7 @@ public class JavaSerializerTest {
obj.uuidCol = new UUID(rnd.nextLong(), rnd.nextLong());
obj.bitmaskCol = TestUtils.randomBitSet(rnd, 42);
obj.stringCol = TestUtils.randomString(rnd, rnd.nextInt(255));
- obj.bytesCol = TestUtils.randomBytes(rnd, rnd.nextInt(255));
+ obj.bytesCol = IgniteTestUtils.randomBytes(rnd, rnd.nextInt(255));
obj.numberCol = (BigInteger)TestUtils.generateRandomValue(rnd, NativeTypes.numberOf(12));
obj.decimalCol = (BigDecimal)TestUtils.generateRandomValue(rnd, NativeTypes.decimalOf(19, 3));
diff --git a/modules/schema/src/test/java/org/apache/ignite/internal/schema/marshaller/reflection/FieldAccessorTest.java b/modules/schema/src/test/java/org/apache/ignite/internal/schema/marshaller/reflection/FieldAccessorTest.java
index 0c75ceb..951824a 100644
--- a/modules/schema/src/test/java/org/apache/ignite/internal/schema/marshaller/reflection/FieldAccessorTest.java
+++ b/modules/schema/src/test/java/org/apache/ignite/internal/schema/marshaller/reflection/FieldAccessorTest.java
@@ -320,7 +320,7 @@ public class FieldAccessorTest {
obj.uuidCol = new UUID(rnd.nextLong(), rnd.nextLong());
obj.bitmaskCol = TestUtils.randomBitSet(rnd, rnd.nextInt(42));
obj.stringCol = TestUtils.randomString(rnd, rnd.nextInt(255));
- obj.bytesCol = TestUtils.randomBytes(rnd, rnd.nextInt(255));
+ obj.bytesCol = IgniteTestUtils.randomBytes(rnd, rnd.nextInt(255));
obj.numberCol = (BigInteger)TestUtils.generateRandomValue(rnd, NativeTypes.numberOf(12));
obj.decimalCol = (BigDecimal) TestUtils.generateRandomValue(rnd, NativeTypes.decimalOf(19, 3));