You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by iv...@apache.org on 2023/01/20 11:07:49 UTC
[ignite] branch master updated: IGNITE-18584 Enlarge compression buffer in order to provide sufficient space for compressors. (#10490)
This is an automated email from the ASF dual-hosted git repository.
ivandasch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 21ffc3e92ea IGNITE-18584 Enlarge compression buffer in order to provide sufficient space for compressors. (#10490)
21ffc3e92ea is described below
commit 21ffc3e92ea4ea13ece8a3b45094185665e8cd69
Author: Ivan Daschinskiy <iv...@apache.org>
AuthorDate: Fri Jan 20 14:07:39 2023 +0300
IGNITE-18584 Enlarge compression buffer in order to provide sufficient space for compressors. (#10490)
---
.../compress/CompressionProcessorImpl.java | 13 +-
.../compress/CompressionProcessorTest.java | 864 +++------------------
.../platforms/cpp/core/src/jni/os/linux/utils.cpp | 2 +-
.../platforms/cpp/core/src/jni/os/win/utils.cpp | 2 +-
4 files changed, 119 insertions(+), 762 deletions(-)
diff --git a/modules/compress/src/main/java/org/apache/ignite/internal/processors/compress/CompressionProcessorImpl.java b/modules/compress/src/main/java/org/apache/ignite/internal/processors/compress/CompressionProcessorImpl.java
index 623ff50ee3d..d2e22caf702 100644
--- a/modules/compress/src/main/java/org/apache/ignite/internal/processors/compress/CompressionProcessorImpl.java
+++ b/modules/compress/src/main/java/org/apache/ignite/internal/processors/compress/CompressionProcessorImpl.java
@@ -50,8 +50,8 @@ public class CompressionProcessorImpl extends CompressionProcessor {
/** Max page size. */
private final ThreadLocalByteBuffer compactBuf = new ThreadLocalByteBuffer(MAX_PAGE_SIZE);
- /** A bit more than max page size. */
- private final ThreadLocalByteBuffer compressBuf = new ThreadLocalByteBuffer(MAX_PAGE_SIZE + 1024);
+ /** A bit more than max page size, extra space is required by compressors. */
+ private final ThreadLocalByteBuffer compressBuf = new ThreadLocalByteBuffer(maxCompressedBufferSize(MAX_PAGE_SIZE));
/**
* @param ctx Kernal context.
@@ -420,6 +420,15 @@ public class CompressionProcessorImpl extends CompressionProcessor {
setCompressionInfo(page, DiskPageCompression.DISABLED, 0, 0);
}
+ /** */
+ private static int maxCompressedBufferSize(int baseSz) {
+ int lz4Sz = Lz4.fastCompressor.maxCompressedLength(baseSz);
+ int zstdSz = (int)Zstd.compressBound(baseSz);
+ int snappySz = Snappy.maxCompressedLength(baseSz);
+
+ return Math.max(Math.max(lz4Sz, zstdSz), snappySz);
+ }
+
/** */
static class Lz4 {
/** */
diff --git a/modules/compress/src/test/java/org/apache/ignite/internal/processors/compress/CompressionProcessorTest.java b/modules/compress/src/test/java/org/apache/ignite/internal/processors/compress/CompressionProcessorTest.java
index ff328e89bff..f499059a1fe 100644
--- a/modules/compress/src/test/java/org/apache/ignite/internal/processors/compress/CompressionProcessorTest.java
+++ b/modules/compress/src/test/java/org/apache/ignite/internal/processors/compress/CompressionProcessorTest.java
@@ -42,7 +42,10 @@ import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.testframework.junits.GridTestKernalContext;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import static org.apache.ignite.configuration.DataStorageConfiguration.MAX_PAGE_SIZE;
import static org.apache.ignite.configuration.DiskPageCompression.LZ4;
import static org.apache.ignite.configuration.DiskPageCompression.SKIP_GARBAGE;
import static org.apache.ignite.configuration.DiskPageCompression.SNAPPY;
@@ -50,741 +53,165 @@ import static org.apache.ignite.configuration.DiskPageCompression.ZSTD;
import static org.apache.ignite.internal.processors.compress.CompressionProcessor.LZ4_MAX_LEVEL;
import static org.apache.ignite.internal.processors.compress.CompressionProcessor.LZ4_MIN_LEVEL;
import static org.apache.ignite.internal.processors.compress.CompressionProcessor.UNCOMPRESSED_PAGE;
+import static org.apache.ignite.internal.processors.compress.CompressionProcessor.ZSTD_DEFAULT_LEVEL;
import static org.apache.ignite.internal.processors.compress.CompressionProcessor.ZSTD_MAX_LEVEL;
import static org.apache.ignite.internal.processors.compress.CompressionProcessorImpl.allocateDirectBuffer;
-import static org.apache.ignite.internal.processors.compress.CompressionProcessorTest.TestInnerIO.INNER_IO;
-import static org.apache.ignite.internal.processors.compress.CompressionProcessorTest.TestLeafIO.LEAF_IO;
import static org.apache.ignite.internal.util.GridUnsafe.bufferAddress;
/**
*/
+@RunWith(Parameterized.class)
public class CompressionProcessorTest extends GridCommonAbstractTest {
/** */
- private static final int ITEM_SIZE = 6; // To fill the whole page.
-
- /** */
- private int blockSize = 16;
-
- /** */
- private int pageSize = 4 * 1024;
-
- /** */
- private DiskPageCompression compression;
-
- /** */
- private int compressLevel;
-
- /** */
- private CompressionProcessor p;
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
- PageIO.registerTest(INNER_IO, LEAF_IO);
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() {
- p = new CompressionProcessorImpl(new GridTestKernalContext(log));
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageCompact16() throws IgniteCheckedException {
- blockSize = 16;
- compression = SKIP_GARBAGE;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageCompact128() throws IgniteCheckedException {
- blockSize = 128;
- compression = SKIP_GARBAGE;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageCompact1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = SKIP_GARBAGE;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageCompact2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = SKIP_GARBAGE;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageZstd16() throws IgniteCheckedException {
- blockSize = 16;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageZstd128() throws IgniteCheckedException {
- blockSize = 128;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageZstd1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageZstd2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageSnappy16() throws IgniteCheckedException {
- blockSize = 16;
- compression = SNAPPY;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageSnappy128() throws IgniteCheckedException {
- blockSize = 128;
- compression = SNAPPY;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageSnappy1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = SNAPPY;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageSnappy2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = SNAPPY;
-
- doTestDataPage();
- }
-
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageLz4Fast16() throws IgniteCheckedException {
- blockSize = 16;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageLz4Fast128() throws IgniteCheckedException {
- blockSize = 128;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageLz4Fast1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageLz4Fast2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageLz4Slow16() throws IgniteCheckedException {
- blockSize = 16;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageLz4Slow128() throws IgniteCheckedException {
- blockSize = 128;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageLz4Slow1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testDataPageLz4Slow2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
-
- doTestDataPage();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageCompact16() throws IgniteCheckedException {
- blockSize = 16;
- compression = SKIP_GARBAGE;
-
- doTestBTreePage(INNER_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageCompact16() throws IgniteCheckedException {
- blockSize = 16;
- compression = SKIP_GARBAGE;
-
- doTestBTreePage(LEAF_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageZstd16() throws IgniteCheckedException {
- blockSize = 16;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
-
- doTestBTreePage(INNER_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageZstd16() throws IgniteCheckedException {
- blockSize = 16;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
-
- doTestBTreePage(LEAF_IO);
- }
-
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageLz4Fast16() throws IgniteCheckedException {
- blockSize = 16;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
-
- doTestBTreePage(INNER_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageLz4Fast16() throws IgniteCheckedException {
- blockSize = 16;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
-
- doTestBTreePage(LEAF_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageLz4Slow16() throws IgniteCheckedException {
- blockSize = 16;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
-
- doTestBTreePage(INNER_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageLz4Slow16() throws IgniteCheckedException {
- blockSize = 16;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
-
- doTestBTreePage(LEAF_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageSnappy16() throws IgniteCheckedException {
- blockSize = 16;
- compression = SNAPPY;
-
- doTestBTreePage(INNER_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageSnappy16() throws IgniteCheckedException {
- blockSize = 16;
- compression = SNAPPY;
-
- doTestBTreePage(LEAF_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageCompact128() throws IgniteCheckedException {
- blockSize = 128;
- compression = SKIP_GARBAGE;
-
- doTestBTreePage(INNER_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageCompact128() throws IgniteCheckedException {
- blockSize = 128;
- compression = SKIP_GARBAGE;
-
- doTestBTreePage(LEAF_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageZstd128() throws IgniteCheckedException {
- blockSize = 128;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
-
- doTestBTreePage(INNER_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageZstd128() throws IgniteCheckedException {
- blockSize = 128;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
-
- doTestBTreePage(LEAF_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageLz4Fast128() throws IgniteCheckedException {
- blockSize = 128;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
-
- doTestBTreePage(INNER_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageLz4Fast128() throws IgniteCheckedException {
- blockSize = 128;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
-
- doTestBTreePage(LEAF_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageLz4Slow128() throws IgniteCheckedException {
- blockSize = 128;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
-
- doTestBTreePage(INNER_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageLz4Slow128() throws IgniteCheckedException {
- blockSize = 128;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
-
- doTestBTreePage(LEAF_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageSnappy128() throws IgniteCheckedException {
- blockSize = 128;
- compression = SNAPPY;
-
- doTestBTreePage(INNER_IO);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageSnappy128() throws IgniteCheckedException {
- blockSize = 128;
- compression = SNAPPY;
+ @Parameterized.Parameters(name = "page_size = {0}, block_size = {1}, compression = {2}, level = {3}")
+ public static Iterable<Object[]> testParameters() {
+ List<Object[]> params = new ArrayList<>();
+
+ for (int pageSz = 4 * 1024; pageSz <= MAX_PAGE_SIZE; pageSz *= 2) {
+ for (int blockSize : Arrays.asList(16, 128, 1024, 2048)) {
+ params.add(new Object[] {pageSz, blockSize, SKIP_GARBAGE, 0});
+ params.add(new Object[] {pageSz, blockSize, SNAPPY, 0});
+ params.add(new Object[] {pageSz, blockSize, LZ4, LZ4_MIN_LEVEL});
+ params.add(new Object[] {pageSz, blockSize, LZ4, LZ4_MAX_LEVEL});
+ params.add(new Object[] {pageSz, blockSize, ZSTD, ZSTD_DEFAULT_LEVEL});
+ params.add(new Object[] {pageSz, blockSize, ZSTD, ZSTD_MAX_LEVEL});
+ }
+ }
- doTestBTreePage(LEAF_IO);
+ return params;
}
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageCompact1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = SKIP_GARBAGE;
-
- doTestBTreePage(INNER_IO);
- }
+ /** */
+ @Parameterized.Parameter
+ public int pageSize;
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageCompact1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = SKIP_GARBAGE;
+ /** */
+ @Parameterized.Parameter(1)
+ public int blockSize;
- doTestBTreePage(LEAF_IO);
- }
+ /** */
+ @Parameterized.Parameter(2)
+ public DiskPageCompression compression;
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageZstd1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
+ /** */
+ @Parameterized.Parameter(3)
+ public int compressLevel;
- doTestBTreePage(INNER_IO);
- }
+ /** */
+ private CompressionProcessor p;
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageZstd1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
+ /** */
+ private TestInnerIO innerIo;
- doTestBTreePage(LEAF_IO);
- }
+ /** */
+ private TestLeafIO leafIo;
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageLz4Fast1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() {
+ /**
+ * 7 is divisor of 8192 - 58, 6 is divisor of 4096 - 58 and 16384 - 58. So an item size should be
+ * 7 for a 8k page and 6 for 4k and 16k pages respectively in order to fill the page fully.
+ * See {@see org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO#getMaxCount}
+ */
+ int itemSize = pageSize == 8192 ? 7 : 6;
- doTestBTreePage(INNER_IO);
- }
+ innerIo = new TestInnerIO(itemSize);
+ leafIo = new TestLeafIO(itemSize);
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageLz4Fast1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
+ PageIO.registerTest(innerIo, leafIo);
- doTestBTreePage(LEAF_IO);
+ p = new CompressionProcessorImpl(new GridTestKernalContext(log));
}
/**
* @throws IgniteCheckedException If failed.
*/
@Test
- public void testInnerPageLz4Slow1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
-
- doTestBTreePage(INNER_IO);
- }
+ public void testDataPage() throws IgniteCheckedException {
+ Random rnd = ThreadLocalRandom.current();
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageLz4Slow1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
+ final byte[][] rows = new byte[][]{
+ new byte[17], new byte[37], new byte[71]
+ };
- doTestBTreePage(LEAF_IO);
- }
+ for (int i = 0; i < rows.length; i++)
+ rnd.nextBytes(rows[i]);
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageSnappy1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = SNAPPY;
+ ByteBuffer page = allocateDirectBuffer(pageSize);
+ long pageAddr = bufferAddress(page);
- doTestBTreePage(INNER_IO);
- }
+ SimpleDataPageIO io = SimpleDataPageIO.VERSIONS.latest();
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageSnappy1k() throws IgniteCheckedException {
- blockSize = 1024;
- compression = SNAPPY;
+ long pageId = PageIdUtils.pageId(PageIdAllocator.MAX_PARTITION_ID, PageIdAllocator.FLAG_DATA, 171717);
- doTestBTreePage(LEAF_IO);
- }
+ io.initNewPage(pageAddr, pageId, pageSize, null);
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageCompact2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = SKIP_GARBAGE;
+ checkIo(io, page);
- doTestBTreePage(INNER_IO);
- }
+ Function<ByteBuffer, List<Bytes>> getContents = (buf) -> {
+ try {
+ long addr = bufferAddress(buf);
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageCompact2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = SKIP_GARBAGE;
+ return io.forAllItems(addr, (link) -> {
+ DataPagePayload payload = io.readPayload(addr, PageIdUtils.itemId(link), pageSize);
- doTestBTreePage(LEAF_IO);
- }
+ return new Bytes(payload.getBytes(addr));
+ });
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ };
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageZstd2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
+ // Empty data page.
+ checkCompressDecompress(page, getContents, false);
- doTestBTreePage(INNER_IO);
- }
+ GridIntList itemIds = new GridIntList();
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageZstd2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = ZSTD;
- compressLevel = ZSTD_MAX_LEVEL;
+ for (;;) {
+ byte[] row = rows[rnd.nextInt(rows.length)];
- doTestBTreePage(LEAF_IO);
- }
+ if (io.getFreeSpace(pageAddr) < row.length)
+ break;
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageLz4Fast2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
+ itemIds.add(io.addRow(pageAddr, row, pageSize));
+ }
- doTestBTreePage(INNER_IO);
- }
+ int freeSpace = io.getFreeSpace(pageAddr);
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageLz4Fast2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = LZ4;
- compressLevel = LZ4_MIN_LEVEL;
+ if (freeSpace != 0) {
+ byte[] lastRow = new byte[freeSpace];
+ rnd.nextBytes(lastRow);
- doTestBTreePage(LEAF_IO);
- }
+ io.addRowFragment(pageId, pageAddr, lastRow, 777L, pageSize);
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testInnerPageLz4Slow2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
+ assertEquals(0, io.getRealFreeSpace(pageAddr));
+ }
- doTestBTreePage(INNER_IO);
- }
+ // Full data page.
+ checkCompressDecompress(page, getContents, io.getRealFreeSpace(pageAddr) == 0);
- /**
- * @throws IgniteCheckedException If failed.
- */
- @Test
- public void testLeafPageLz4Slow2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = LZ4;
- compressLevel = LZ4_MAX_LEVEL;
+ for (int i = 0; i < itemIds.size(); i += 2)
+ io.removeRow(pageAddr, itemIds.get(i), pageSize);
- doTestBTreePage(LEAF_IO);
+ // Half-filled data page.
+ checkCompressDecompress(page, getContents, false);
}
/**
* @throws IgniteCheckedException If failed.
*/
@Test
- public void testInnerPageSnappy2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = SNAPPY;
-
- doTestBTreePage(INNER_IO);
+ public void testInnerPage() throws IgniteCheckedException {
+ doTestBTreePage(innerIo);
}
/**
* @throws IgniteCheckedException If failed.
*/
@Test
- public void testLeafPageSnappy2k() throws IgniteCheckedException {
- blockSize = 2 * 1024;
- compression = SNAPPY;
-
- doTestBTreePage(LEAF_IO);
+ public void testLeafPage() throws IgniteCheckedException {
+ doTestBTreePage(leafIo);
}
/**
@@ -855,80 +282,6 @@ public class CompressionProcessorTest extends GridCommonAbstractTest {
checkCompressDecompress(page, getContents, false);
}
- /**
- * @throws IgniteCheckedException If failed.
- */
- private void doTestDataPage() throws IgniteCheckedException {
- Random rnd = ThreadLocalRandom.current();
-
- final byte[][] rows = new byte[][]{
- new byte[17], new byte[37], new byte[71]
- };
-
- for (int i = 0; i < rows.length; i++)
- rnd.nextBytes(rows[i]);
-
- ByteBuffer page = allocateDirectBuffer(pageSize);
- long pageAddr = bufferAddress(page);
-
- SimpleDataPageIO io = SimpleDataPageIO.VERSIONS.latest();
-
- long pageId = PageIdUtils.pageId(PageIdAllocator.MAX_PARTITION_ID, PageIdAllocator.FLAG_DATA, 171717);
-
- io.initNewPage(pageAddr, pageId, pageSize, null);
-
- checkIo(io, page);
-
- Function<ByteBuffer, List<Bytes>> getContents = (buf) -> {
- try {
- long addr = bufferAddress(buf);
-
- return io.forAllItems(addr, (link) -> {
- DataPagePayload payload = io.readPayload(addr, PageIdUtils.itemId(link), pageSize);
-
- return new Bytes(payload.getBytes(addr));
- });
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- };
-
- // Empty data page.
- checkCompressDecompress(page, getContents, false);
-
- GridIntList itemIds = new GridIntList();
-
- for (;;) {
- byte[] row = rows[rnd.nextInt(rows.length)];
-
- if (io.getFreeSpace(pageAddr) < row.length)
- break;
-
- itemIds.add(io.addRow(pageAddr, row, pageSize));
- }
-
- int freeSpace = io.getFreeSpace(pageAddr);
-
- if (freeSpace != 0) {
- byte[] lastRow = new byte[freeSpace];
- rnd.nextBytes(lastRow);
-
- io.addRowFragment(pageId, pageAddr, lastRow, 777L, pageSize);
-
- assertEquals(0, io.getRealFreeSpace(pageAddr));
- }
-
- // Full data page.
- checkCompressDecompress(page, getContents, io.getRealFreeSpace(pageAddr) == 0);
-
- for (int i = 0; i < itemIds.size(); i += 2)
- io.removeRow(pageAddr, itemIds.get(i), pageSize);
-
- // Half-filled data page.
- checkCompressDecompress(page, getContents, false);
- }
-
/** */
private void checkIo(PageIO io, ByteBuffer page) throws IgniteCheckedException {
assertSame(io, PageIO.getPageIO(bufferAddress(page)));
@@ -1027,13 +380,10 @@ public class CompressionProcessorTest extends GridCommonAbstractTest {
/**
*/
static class TestLeafIO extends BPlusLeafIO<byte[]> {
- /** */
- static final TestLeafIO LEAF_IO = new TestLeafIO();
-
/**
*/
- TestLeafIO() {
- super(29_501, 1, ITEM_SIZE);
+ TestLeafIO(int itemSz) {
+ super(29_501, 1, itemSz);
}
/** {@inheritDoc} */
@@ -1056,13 +406,10 @@ public class CompressionProcessorTest extends GridCommonAbstractTest {
/**
*/
static class TestInnerIO extends BPlusInnerIO<byte[]> {
- /** */
- static TestInnerIO INNER_IO = new TestInnerIO();
-
/**
*/
- TestInnerIO() {
- super(29_502, 1, true, ITEM_SIZE);
+ TestInnerIO(int itemSz) {
+ super(29_502, 1, true, itemSz);
}
/** {@inheritDoc} */
@@ -1082,3 +429,4 @@ public class CompressionProcessorTest extends GridCommonAbstractTest {
}
}
}
+
diff --git a/modules/platforms/cpp/core/src/jni/os/linux/utils.cpp b/modules/platforms/cpp/core/src/jni/os/linux/utils.cpp
index dab7849c033..b6f455ad4c2 100644
--- a/modules/platforms/cpp/core/src/jni/os/linux/utils.cpp
+++ b/modules/platforms/cpp/core/src/jni/os/linux/utils.cpp
@@ -47,7 +47,7 @@ namespace ignite
const char* IGNITE_NATIVE_TEST_CLASSPATH = "IGNITE_NATIVE_TEST_CLASSPATH";
/** Excluded modules from test classpath. */
- const char* TEST_EXCLUDED_MODULES[] = { "rest-http" };
+ const char* TEST_EXCLUDED_MODULES[] = { "rest-http", "compress" };
/** Key indicating that the thread is attached. */
static pthread_key_t attachKey;
diff --git a/modules/platforms/cpp/core/src/jni/os/win/utils.cpp b/modules/platforms/cpp/core/src/jni/os/win/utils.cpp
index 40945d0ef6a..330f662aa04 100644
--- a/modules/platforms/cpp/core/src/jni/os/win/utils.cpp
+++ b/modules/platforms/cpp/core/src/jni/os/win/utils.cpp
@@ -43,7 +43,7 @@ namespace ignite
const char* IGNITE_NATIVE_TEST_CLASSPATH = "IGNITE_NATIVE_TEST_CLASSPATH";
/** Excluded modules from test classpath. */
- const char* TEST_EXCLUDED_MODULES[] = { "rest-http" };
+ const char* TEST_EXCLUDED_MODULES[] = { "rest-http", "compress" };
AttachHelper::~AttachHelper()
{