You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by co...@apache.org on 2026/07/01 03:06:39 UTC
(iotdb) branch research/encoding-subcolumn updated: Add SubColumn SQL support for integer and float types (#18077)
This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch research/encoding-subcolumn
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/research/encoding-subcolumn by this push:
new 3adac1eded7 Add SubColumn SQL support for integer and float types (#18077)
3adac1eded7 is described below
commit 3adac1eded72ffce417e3796de16fbcf4387b032
Author: xjz17 <67...@users.noreply.github.com>
AuthorDate: Wed Jul 1 11:06:34 2026 +0800
Add SubColumn SQL support for integer and float types (#18077)
This syncs the SubColumn implementation used by the paper artifact, including FLOAT scaling support and SQL/schema tests.
---
.../db/queryengine/plan/parser/ASTVisitor.java | 11 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 6 +-
.../plan/parser/StatementGeneratorTest.java | 26 +
.../org/apache/iotdb/db/utils/SchemaUtilsTest.java | 12 +
iotdb-core/tsfile/pom.xml | 23 +-
.../iotdb/tsfile/encoding/decoder/Decoder.java | 12 +
.../encoding/decoder/FloatSubColumnDecoder.java | 91 +
.../encoding/decoder/IntSubColumnDecoder.java | 85 +
.../encoding/decoder/LongSubColumnDecoder.java | 65 +
.../encoding/encoder/FloatSubColumnEncoder.java | 152 +
.../encoding/encoder/IntSubColumnEncoder.java | 113 +
.../encoding/encoder/LongSubColumnEncoder.java | 68 +
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 23 +
.../tsfile/file/metadata/enums/TSEncoding.java | 5 +-
.../iotdb/tsfile/compress/CSVCompressTest.java | 269 ++
.../org/apache/iotdb/tsfile/encoding/ALPTest.java | 695 +++++
... => ASubcolumnAddDictionaryAndHuffmanTest.java} | 702 +++--
.../encoding/{BPTest.java => BPLongTest.java} | 478 ++--
.../iotdb/tsfile/encoding/BPQueryGroupTest.java | 126 +
.../iotdb/tsfile/encoding/BPQuerySortTest.java | 142 +
.../org/apache/iotdb/tsfile/encoding/BPTest.java | 224 +-
.../iotdb/tsfile/encoding/BPUpdateAppendTest.java | 125 +
.../tsfile/encoding/BPUpdateDeleteSmallerTest.java | 72 +
.../tsfile/encoding/BPUpdateInsertLargerTest.java | 68 +
.../tsfile/encoding/BPUpdateInsertSmallerTest.java | 68 +
.../iotdb/tsfile/encoding/BPUpdateLargerTest.java | 68 +
.../iotdb/tsfile/encoding/BPUpdateSmallerTest.java | 68 +
.../iotdb/tsfile/encoding/BUFFDouble2Test.java | 553 ++++
.../{BUFFTest.java => BUFFDoubleTest.java} | 543 ++--
.../org/apache/iotdb/tsfile/encoding/BUFFTest.java | 256 +-
.../apache/iotdb/tsfile/encoding/BitWeaving.java | 182 ++
.../tsfile/encoding/BuffOfficialRatioTest.java | 321 +++
.../iotdb/tsfile/encoding/BuffQueryMain.java | 373 +++
.../DatasetEncoderCompressRoundtripBenchTest.java | 2910 ++++++++++++++++++++
.../encoding/{BPTest.java => DeltaLongTest.java} | 543 ++--
.../tsfile/encoding/DeltaSubcolumnLongTest.java | 728 +++++
.../encoding/DictionaryLongOnSortedTest.java | 827 ++++++
.../iotdb/tsfile/encoding/DictionaryLongTest.java | 700 +++++
.../tsfile/encoding/DictionaryMaterializeTest.java | 1017 +++++++
.../encoding/DictionarySubcolumnLongTest.java | 749 +++++
.../encoding/FastPFORCompressionBenchmarkTest.java | 228 ++
.../tsfile/encoding/FastPForSimple8bCodec.java | 440 +++
.../org/apache/iotdb/tsfile/encoding/HBPIndex.java | 261 ++
.../apache/iotdb/tsfile/encoding/HBPIndexLong.java | 194 ++
.../tsfile/encoding/HBPIndexLongQueryTest.java | 327 +++
.../iotdb/tsfile/encoding/HBPIndexLongTest.java | 325 +++
.../iotdb/tsfile/encoding/ParquetSelectEqual.java | 444 +++
.../tsfile/encoding/ParquetSelectGreater.java | 451 +++
.../tsfile/encoding/ParquetSelectGreaterLess.java | 433 +++
.../iotdb/tsfile/encoding/ParquetSelectLess.java | 342 +++
.../tsfile/encoding/ParquetSelectLessPart.java | 538 ++++
.../iotdb/tsfile/encoding/ParquetSelectMax.java | 806 ++++++
.../tsfile/encoding/RLEBPLongOnSortedTest.java | 863 ++++++
.../iotdb/tsfile/encoding/RLEBPLongTest.java | 726 +++++
.../apache/iotdb/tsfile/encoding/RLEBPTest.java | 203 +-
.../iotdb/tsfile/encoding/RLEMaterializeTest.java | 995 +++++++
.../iotdb/tsfile/encoding/SPRINTZBPLongTest.java | 739 +++++
.../iotdb/tsfile/encoding/SPRINTZBPTest.java | 294 +-
.../tsfile/encoding/SPRINTZSubcolumnLongTest.java | 387 +++
.../encoding/SPRINTZSubcolumnPruneNewTest.java | 398 +++
.../tsfile/encoding/SPRINTZSubcolumnTest.java | 158 +-
.../encoding/Simple8bCompressionBenchmarkTest.java | 193 ++
.../tsfile/encoding/SubColumnEncodingTest.java | 100 +
.../encoding/SubcolumnAblationCodecSupport.java | 723 +++++
.../encoding/SubcolumnAblationPruneNewEngine.java | 1860 +++++++++++++
.../SubcolumnAddDictPruneNewUpdateAppendTest.java | 118 +
...lumnAddDictPruneNewUpdateDeleteSmallerTest.java | 124 +
...olumnAddDictPruneNewUpdateInsertLargerTest.java | 122 +
...lumnAddDictPruneNewUpdateInsertSmallerTest.java | 122 +
.../SubcolumnAddDictPruneNewUpdateLargerTest.java | 122 +
.../SubcolumnAddDictPruneNewUpdateRatioTest.java | 656 +++++
.../SubcolumnAddDictPruneNewUpdateSmallerTest.java | 122 +
.../SubcolumnAddDictPruneNewUpdateTestUtil.java | 161 ++
...est.java => SubcolumnAddDictPruneTimeTest.java} | 622 +++--
.../encoding/SubcolumnAddDictQueryGreaterTest.java | 372 +++
.../encoding/SubcolumnAddDictQueryGroupTest.java | 523 ++++
.../iotdb/tsfile/encoding/SubcolumnBPBetaTest.java | 175 ++
.../{SubcolumnTest.java => SubcolumnBPTest.java} | 365 +--
.../iotdb/tsfile/encoding/SubcolumnBetaTest.java | 197 +-
.../tsfile/encoding/SubcolumnBlockSizeTest.java | 153 +-
...ntTest.java => SubcolumnCountWithNULLTest.java} | 275 +-
.../SubcolumnFullEncodingTypeRatioTest.java | 572 ++++
.../iotdb/tsfile/encoding/SubcolumnFullTest.java | 74 +
...SubcolumnTest.java => SubcolumnLongBPTest.java} | 661 ++---
.../tsfile/encoding/SubcolumnLongBetaTest.java | 340 +++
.../encoding/SubcolumnLongBlockSizeTest.java | 213 ++
.../encoding/SubcolumnLongMaterializeR1D3.java | 1110 ++++++++
.../encoding/SubcolumnLongMaterializeR1D8.java | 1068 +++++++
...umnTest.java => SubcolumnLongOnSortedTest.java} | 561 ++--
.../encoding/SubcolumnLongPSOBlockSizeTest.java | 214 ++
...ubcolumnTest.java => SubcolumnLongPSOTest.java} | 736 +++--
.../encoding/SubcolumnLongPointQueryBPTest.java | 1177 ++++++++
...st.java => SubcolumnLongPointQueryRLETest.java} | 807 +++---
.../encoding/SubcolumnLongPointQueryTest.java | 887 ++++++
.../encoding/SubcolumnLongQueryCountTest.java | 224 ++
.../encoding/SubcolumnLongQueryEqualTest.java | 226 ++
.../encoding/SubcolumnLongQuerySum2Test.java | 201 ++
...ubcolumnTest.java => SubcolumnLongRLETest.java} | 537 ++--
.../{SubcolumnTest.java => SubcolumnLongTest.java} | 483 ++--
...essPartsTest.java => SubcolumnMaterialize.java} | 491 +++-
.../tsfile/encoding/SubcolumnMaterializeR1D8.java | 1038 +++++++
...lumnTest.java => SubcolumnMaxWithNULLTest.java} | 802 +++---
...lumnTest.java => SubcolumnNoPruneBetaTest.java} | 563 ++--
...bcolumnTest.java => SubcolumnOnValuesTest.java} | 488 ++--
...PTest.java => SubcolumnOnlyDictionaryTest.java} | 524 ++--
...umnTest.java => SubcolumnPointQueryBPTest.java} | 713 ++---
...mnTest.java => SubcolumnPointQueryRLETest.java} | 536 ++--
.../tsfile/encoding/SubcolumnPointQueryTest.java | 445 +++
.../tsfile/encoding/SubcolumnPrune2NewTest.java | 773 ++++++
.../encoding/SubcolumnPrune2NewTimeTest.java | 917 ++++++
...SubcolumnTest.java => SubcolumnPrune2Test.java} | 783 +++---
.../tsfile/encoding/SubcolumnPruneFastTest.java | 556 ++++
.../encoding/SubcolumnPruneNewBenchmarkTest.java | 120 +
.../tsfile/encoding/SubcolumnPruneNewBetaTest.java | 1027 +++++++
.../SubcolumnPruneNewBlockSizeNoPruneTest.java | 756 +++++
.../encoding/SubcolumnPruneNewBlockSizeTest.java | 149 +
.../SubcolumnPruneNewEncodingTypeRatioTest.java | 1142 ++++++++
.../tsfile/encoding/SubcolumnPruneNewTest.java | 2009 ++++++++++++++
...{SubcolumnTest.java => SubcolumnPruneTest.java} | 590 ++--
.../tsfile/encoding/SubcolumnQueryCount2Test.java | 135 +
.../tsfile/encoding/SubcolumnQueryCount3Test.java | 41 +
.../encoding/SubcolumnQueryCountNewTest.java | 176 ++
.../tsfile/encoding/SubcolumnQueryCountTest.java | 324 +--
.../encoding/SubcolumnQueryEqualNewTest.java | 214 ++
.../tsfile/encoding/SubcolumnQueryEqualTest.java | 324 +--
.../encoding/SubcolumnQueryGreaterLessTest.java | 342 ---
.../encoding/SubcolumnQueryGreaterNewTest.java | 220 ++
.../tsfile/encoding/SubcolumnQueryGreaterTest.java | 316 +--
.../tsfile/encoding/SubcolumnQueryLessNewTest.java | 200 ++
.../encoding/SubcolumnQueryLessPartsNewTest.java | 461 ++++
.../encoding/SubcolumnQueryLessPartsTest.java | 368 ---
.../tsfile/encoding/SubcolumnQueryLessTest.java | 309 ---
.../iotdb/tsfile/encoding/SubcolumnQueryMain.java | 1307 +++++++++
.../tsfile/encoding/SubcolumnQueryMaxNewTest.java | 185 ++
.../tsfile/encoding/SubcolumnQueryMaxTest.java | 281 --
.../tsfile/encoding/SubcolumnQuerySortTest.java | 492 ++++
.../tsfile/encoding/SubcolumnQuerySum2Test.java | 313 +--
.../tsfile/encoding/SubcolumnQuerySumTest.java | 287 +-
.../{SubcolumnTest.java => SubcolumnRLETest.java} | 227 +-
...ubcolumnTest.java => SubcolumnSAPHANATest.java} | 506 ++--
...um2Test.java => SubcolumnSum2WithNULLTest.java} | 274 +-
...ySumTest.java => SubcolumnSumWithNULLTest.java} | 252 +-
.../iotdb/tsfile/encoding/SubcolumnTest.java | 194 +-
.../tsfile/encoding/SubcolumnVariableAlpha.java | 1399 ++++++++++
.../SubcolumnVariableAlphaSignExpMass.java | 1336 +++++++++
.../tsfile/encoding/SubcolumnWithoutBPETest.java | 16 +
.../tsfile/encoding/SubcolumnWithoutDETest.java | 16 +
.../tsfile/encoding/SubcolumnWithoutRLETest.java | 16 +
.../iotdb/tsfile/encoding/TSDIFFLongTest.java | 694 +++++
.../tsfile/encoding/TSDIFFSubcolumnLongTest.java | 366 +++
.../encoding/TSDIFFSubcolumnPruneNewTest.java | 396 +++
.../iotdb/tsfile/encoding/TSDIFFSubcolumnTest.java | 155 +-
.../apache/iotdb/tsfile/encoding/TSDIFFTest.java | 283 +-
.../{SubcolumnTest.java => UpdateAppendTest.java} | 334 +--
...olumnTest.java => UpdateDeleteSmallerTest.java} | 491 ++--
...columnTest.java => UpdateInsertLargerTest.java} | 492 ++--
...nTest.java => UpdateInsertOneCompressTest.java} | 406 ++-
...olumnTest.java => UpdateInsertSmallerTest.java} | 495 ++--
.../{SubcolumnTest.java => UpdateLargerTest.java} | 548 ++--
.../{SubcolumnTest.java => UpdateSmallerTest.java} | 654 +++--
.../apache/iotdb/tsfile/encoding/VBPIndexLong.java | 348 +++
.../iotdb/tsfile/encoding/VBPIndexLongTest.java | 325 +++
.../iotdb/tsfile/encoding/VBPMaterializeTest.java | 104 +
.../iotdb/tsfile/encoding/VBPQueryCount2Test.java | 91 +
.../iotdb/tsfile/encoding/VBPQueryCountTest.java | 95 +
.../iotdb/tsfile/encoding/VBPQueryEqualTest.java | 105 +
.../tsfile/encoding/VBPQueryGreaterLessTest.java | 120 +
.../iotdb/tsfile/encoding/VBPQueryGreaterTest.java | 105 +
.../iotdb/tsfile/encoding/VBPQueryGroupTest.java | 196 ++
.../apache/iotdb/tsfile/encoding/VBPQueryMain.java | 629 +++++
.../iotdb/tsfile/encoding/VBPQueryMaxTest.java | 104 +
.../tsfile/encoding/VBPQuerySmallerPartsTest.java | 129 +
.../iotdb/tsfile/encoding/VBPQuerySmallerTest.java | 105 +
.../iotdb/tsfile/encoding/VBPQuerySortTest.java | 152 +
.../iotdb/tsfile/encoding/VBPQuerySumTest.java | 100 +
.../iotdb/tsfile/encoding/VBPUpdateAppendTest.java | 131 +
.../encoding/VBPUpdateDeleteSmallerTest.java | 129 +
.../tsfile/encoding/VBPUpdateInsertLargerTest.java | 131 +
.../encoding/VBPUpdateInsertSmallerTest.java | 131 +
.../iotdb/tsfile/encoding/VBPUpdateLargerTest.java | 131 +
.../tsfile/encoding/VBPUpdateSmallerTest.java | 131 +
.../encoding/elf/ElfDoublePrecisionBenchCodec.java | 58 +
.../elf/ElfDoublePrecisionBenchCodecTest.java | 42 +
.../encoding/elf/chimp/ElfInputBitStream.java | 1594 +++++++++++
.../encoding/elf/chimp/ElfOutputBitStream.java | 1304 +++++++++
.../elf/compressor/AbstractElfCompressor.java | 54 +
.../encoding/elf/compressor/ElfCompressor.java | 42 +
.../encoding/elf/compressor/ICompressor.java | 34 +
.../elf/decompressor/AbstractElfDecompressor.java | 54 +
.../encoding/elf/decompressor/ElfDecompressor.java | 27 +
.../encoding/elf/decompressor/IDecompressor.java | 26 +
.../tsfile/encoding/elf/utils/Elf64Utils.java | 163 ++
.../elf/xorcompressor/ElfXORCompressor.java | 165 ++
.../elf/xordecompressor/ElfXORDecompressor.java | 129 +
194 files changed, 65045 insertions(+), 12653 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index c8299c50a68..8b5da9ae58a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -362,7 +362,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
String datatypeString =
props.get(IoTDBConstant.COLUMN_TIMESERIES_DATATYPE.toLowerCase()).toUpperCase();
try {
- createTimeSeriesStatement.setDataType(TSDataType.valueOf(datatypeString));
+ createTimeSeriesStatement.setDataType(parseTimeSeriesDataType(datatypeString));
props.remove(IoTDBConstant.COLUMN_TIMESERIES_DATATYPE.toLowerCase());
} catch (Exception e) {
throw new SemanticException(String.format("Unsupported datatype: %s", datatypeString));
@@ -3428,7 +3428,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
}
String dataTypeString = ctx.dataType.getText().toUpperCase();
try {
- dataType = TSDataType.valueOf(dataTypeString);
+ dataType = parseTimeSeriesDataType(dataTypeString);
} catch (Exception e) {
throw new SemanticException(String.format("Unsupported datatype: %s", dataTypeString));
}
@@ -3436,6 +3436,13 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
return dataType;
}
+ private TSDataType parseTimeSeriesDataType(String dataTypeString) {
+ if ("INTEGER".equals(dataTypeString)) {
+ return TSDataType.INT32;
+ }
+ return TSDataType.valueOf(dataTypeString);
+ }
+
@Override
public Statement visitShowSchemaTemplates(IoTDBSqlParser.ShowSchemaTemplatesContext ctx) {
return new ShowSchemaTemplateStatement();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
index efae314ecaa..e489fa2cb61 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
@@ -56,6 +56,7 @@ public class SchemaUtils {
intSet.add(TSEncoding.CHIMP);
intSet.add(TSEncoding.SPRINTZ);
intSet.add(TSEncoding.RLBE);
+ intSet.add(TSEncoding.SUBCOLUMN);
schemaChecker.put(TSDataType.INT32, intSet);
schemaChecker.put(TSDataType.INT64, intSet);
@@ -69,9 +70,12 @@ public class SchemaUtils {
floatSet.add(TSEncoding.CHIMP);
floatSet.add(TSEncoding.SPRINTZ);
floatSet.add(TSEncoding.RLBE);
+ floatSet.add(TSEncoding.SUBCOLUMN);
schemaChecker.put(TSDataType.FLOAT, floatSet);
- schemaChecker.put(TSDataType.DOUBLE, floatSet);
+ Set<TSEncoding> doubleSet = new HashSet<>(floatSet);
+ doubleSet.remove(TSEncoding.SUBCOLUMN);
+ schemaChecker.put(TSDataType.DOUBLE, doubleSet);
Set<TSEncoding> textSet = new HashSet<>();
textSet.add(TSEncoding.PLAIN);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java
index ff0ec603365..23bbc395db7 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java
@@ -280,6 +280,32 @@ public class StatementGeneratorTest {
assertEquals(SNAPPY, statement.getCompressor());
}
+ @Test
+ public void testCreateTimeSeriesWithSubColumnEncodingSql() throws IllegalPathException {
+ Statement parsed =
+ StatementGenerator.createStatement(
+ "create timeseries root.temperature INTEGER encoding=SubColumn",
+ ZonedDateTime.now().getOffset());
+
+ CreateTimeSeriesStatement statement = (CreateTimeSeriesStatement) parsed;
+ assertEquals(new PartialPath("root.temperature"), statement.getPath());
+ assertEquals(TSDataType.INT32, statement.getDataType());
+ assertEquals(TSEncoding.SUBCOLUMN, statement.getEncoding());
+ }
+
+ @Test
+ public void testCreateFloatTimeSeriesWithSubColumnEncodingSql() throws IllegalPathException {
+ Statement parsed =
+ StatementGenerator.createStatement(
+ "create timeseries root.temperature FLOAT encoding=SubColumn",
+ ZonedDateTime.now().getOffset());
+
+ CreateTimeSeriesStatement statement = (CreateTimeSeriesStatement) parsed;
+ assertEquals(new PartialPath("root.temperature"), statement.getPath());
+ assertEquals(TSDataType.FLOAT, statement.getDataType());
+ assertEquals(TSEncoding.SUBCOLUMN, statement.getEncoding());
+ }
+
@Test
public void testCreateAlignedTimeSeries() throws IllegalPathException {
TSCreateAlignedTimeseriesReq req =
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java
index 7c76d5bafcf..b21d3a6b896 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java
@@ -75,4 +75,16 @@ public class SchemaUtilsTest {
// do nothing
}
}
+
+ @Test
+ public void checkSubColumnEncodingForInteger() throws MetadataException {
+ SchemaUtils.checkDataTypeWithEncoding(TSDataType.INT32, TSEncoding.SUBCOLUMN);
+ SchemaUtils.checkDataTypeWithEncoding(TSDataType.FLOAT, TSEncoding.SUBCOLUMN);
+ try {
+ SchemaUtils.checkDataTypeWithEncoding(TSDataType.TEXT, TSEncoding.SUBCOLUMN);
+ Assert.fail("expect exception");
+ } catch (MetadataException e) {
+ // do nothing
+ }
+ }
}
diff --git a/iotdb-core/tsfile/pom.xml b/iotdb-core/tsfile/pom.xml
index 39f45ce86b9..21546c96234 100644
--- a/iotdb-core/tsfile/pom.xml
+++ b/iotdb-core/tsfile/pom.xml
@@ -191,28 +191,37 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
- <version>1.21</version> <!-- 请检查是否有更新的版本 -->
+ <version>1.21</version>
+ <!-- 请检查是否有更新的版本 -->
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.6.1</version>
</dependency>
+ <dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna-platform</artifactId>
+ <version>${jna.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- ELF bench: bit streams from elf@elf (dsiutils fastutil) -->
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>dsiutils</artifactId>
+ <version>2.7.0</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
<version>24.0.1</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.iotdb</groupId>
- <artifactId>iotdb-antlr</artifactId>
- <version>0.14.0-SNAPSHOT</version>
- </dependency>
<dependency>
<groupId>me.lemire.integercompression</groupId>
<artifactId>JavaFastPFOR</artifactId>
- <version>[0.1,)</version>
+ <version>0.3.13</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
index 7a01964a733..1333bcbbb00 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
@@ -161,6 +161,18 @@ public abstract class Decoder {
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
}
+ case SUBCOLUMN:
+ switch (dataType) {
+ case INT32:
+ return new IntSubColumnDecoder();
+ case INT64:
+ case VECTOR:
+ return new LongSubColumnDecoder();
+ case FLOAT:
+ return new FloatSubColumnDecoder();
+ default:
+ throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
+ }
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatSubColumnDecoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatSubColumnDecoder.java
new file mode 100644
index 00000000000..48a3718e22c
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatSubColumnDecoder.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.nio.ByteBuffer;
+
+/** Decoder for FLOAT values encoded by FloatSubColumnEncoder. */
+public class FloatSubColumnDecoder extends Decoder {
+
+ private float[] values;
+ private int index;
+
+ public FloatSubColumnDecoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public float readFloat(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values[index++];
+ }
+
+ @Override
+ public boolean hasNext(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values != null && index < values.length;
+ }
+
+ @Override
+ public void reset() {
+ values = null;
+ index = 0;
+ }
+
+ private void ensureLoaded(ByteBuffer buffer) {
+ if (values != null || !buffer.hasRemaining()) {
+ return;
+ }
+ int count = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int precision = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int blockSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ double scale = Math.pow(10, precision);
+ int[] scaledValues = new int[count];
+ int offset = 0;
+ while (offset < count) {
+ int length = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int min = buffer.getInt();
+ int bitWidth = buffer.get() & 0xFF;
+ int bytesPerPlane = (length + Byte.SIZE - 1) / Byte.SIZE;
+ for (int bit = 0; bit < bitWidth; bit++) {
+ for (int byteIndex = 0; byteIndex < bytesPerPlane; byteIndex++) {
+ int mask = buffer.get() & 0xFF;
+ for (int j = 0; j < Byte.SIZE; j++) {
+ int localIndex = (byteIndex << 3) + j;
+ if (localIndex < length && ((mask >>> j) & 1) != 0) {
+ scaledValues[offset + localIndex] |= 1 << bit;
+ }
+ }
+ }
+ }
+ for (int i = 0; i < length; i++) {
+ scaledValues[offset + i] += min;
+ }
+ offset += Math.min(blockSize, length);
+ }
+ values = new float[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = (float) (scaledValues[i] / scale);
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntSubColumnDecoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntSubColumnDecoder.java
new file mode 100644
index 00000000000..6178f5bbe9f
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntSubColumnDecoder.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.nio.ByteBuffer;
+
+/** Decoder for INT32 values encoded by {@link org.apache.iotdb.tsfile.encoding.encoder.IntSubColumnEncoder}. */
+public class IntSubColumnDecoder extends Decoder {
+
+ private int[] values;
+ private int index;
+
+ public IntSubColumnDecoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public int readInt(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values[index++];
+ }
+
+ @Override
+ public boolean hasNext(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values != null && index < values.length;
+ }
+
+ @Override
+ public void reset() {
+ values = null;
+ index = 0;
+ }
+
+ private void ensureLoaded(ByteBuffer buffer) {
+ if (values != null || !buffer.hasRemaining()) {
+ return;
+ }
+ int count = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int blockSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ values = new int[count];
+ int offset = 0;
+ while (offset < count) {
+ int length = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int min = buffer.getInt();
+ int bitWidth = buffer.get() & 0xFF;
+ int bytesPerPlane = (length + Byte.SIZE - 1) / Byte.SIZE;
+ for (int bit = 0; bit < bitWidth; bit++) {
+ for (int byteIndex = 0; byteIndex < bytesPerPlane; byteIndex++) {
+ int mask = buffer.get() & 0xFF;
+ for (int j = 0; j < Byte.SIZE; j++) {
+ int localIndex = (byteIndex << 3) + j;
+ if (localIndex < length && ((mask >>> j) & 1) != 0) {
+ values[offset + localIndex] |= 1 << bit;
+ }
+ }
+ }
+ }
+ for (int i = 0; i < length; i++) {
+ values[offset + i] += min;
+ }
+ offset += Math.min(blockSize, length);
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongSubColumnDecoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongSubColumnDecoder.java
new file mode 100644
index 00000000000..1d553e16a23
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongSubColumnDecoder.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.nio.ByteBuffer;
+
+/** Decoder for INT64 values encoded by {@link org.apache.iotdb.tsfile.encoding.encoder.LongSubColumnEncoder}. */
+public class LongSubColumnDecoder extends Decoder {
+
+ private long[] values;
+ private int index;
+
+ public LongSubColumnDecoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public long readLong(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values[index++];
+ }
+
+ @Override
+ public boolean hasNext(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values != null && index < values.length;
+ }
+
+ @Override
+ public void reset() {
+ values = null;
+ index = 0;
+ }
+
+ private void ensureLoaded(ByteBuffer buffer) {
+ if (values != null || !buffer.hasRemaining()) {
+ return;
+ }
+ int count = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ values = new long[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = buffer.getLong();
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatSubColumnEncoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatSubColumnEncoder.java
new file mode 100644
index 00000000000..f989b53a73f
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatSubColumnEncoder.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Encoder for FLOAT values using decimal scaling and sub-column bit-plane packing. */
+public class FloatSubColumnEncoder extends Encoder {
+
+ private static final int BLOCK_SIZE = 128;
+ private static final int MAX_FLOAT_DECIMAL_PRECISION = 6;
+
+ private final List<Float> values = new ArrayList<>();
+
+ public FloatSubColumnEncoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public void encode(float value, ByteArrayOutputStream out) {
+ values.add(value);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ int precision = choosePrecision();
+ double scale = Math.pow(10, precision);
+ int[] scaledValues = new int[values.size()];
+ for (int i = 0; i < values.size(); i++) {
+ scaledValues[i] = (int) Math.round(values.get(i) * scale);
+ }
+
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(scaledValues.length, out);
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(precision, out);
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(BLOCK_SIZE, out);
+ for (int offset = 0; offset < scaledValues.length; offset += BLOCK_SIZE) {
+ int length = Math.min(BLOCK_SIZE, scaledValues.length - offset);
+ writeBlock(out, scaledValues, offset, length);
+ }
+ values.clear();
+ }
+
+ private int choosePrecision() {
+ int precision = 0;
+ for (float value : values) {
+ precision = Math.max(precision, getDecimalPrecision(value));
+ }
+ precision = Math.min(precision, MAX_FLOAT_DECIMAL_PRECISION);
+ while (precision > 0 && !fitsInInt(precision)) {
+ precision--;
+ }
+ return precision;
+ }
+
+ private boolean fitsInInt(int precision) {
+ double scale = Math.pow(10, precision);
+ for (float value : values) {
+ double scaled = Math.rint(value * scale);
+ if (scaled < Integer.MIN_VALUE || scaled > Integer.MAX_VALUE) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static int getDecimalPrecision(float value) {
+ BigDecimal decimal = new BigDecimal(Float.toString(value)).stripTrailingZeros();
+ return Math.max(0, decimal.scale());
+ }
+
+ private static void writeBlock(
+ ByteArrayOutputStream out, int[] scaledValues, int offset, int length) throws IOException {
+ int min = scaledValues[offset];
+ long maxDelta = 0;
+ for (int i = 0; i < length; i++) {
+ int value = scaledValues[offset + i];
+ if (value < min) {
+ min = value;
+ }
+ }
+ for (int i = 0; i < length; i++) {
+ long delta = (long) scaledValues[offset + i] - min;
+ if (delta > maxDelta) {
+ maxDelta = delta;
+ }
+ }
+
+ int bitWidth = maxDelta == 0 ? 0 : 64 - Long.numberOfLeadingZeros(maxDelta);
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(length, out);
+ writeInt(out, min);
+ out.write(bitWidth);
+ if (bitWidth == 0) {
+ return;
+ }
+
+ int bytesPerPlane = (length + Byte.SIZE - 1) / Byte.SIZE;
+ byte[] plane = new byte[bytesPerPlane];
+ for (int bit = 0; bit < bitWidth; bit++) {
+ for (int i = 0; i < bytesPerPlane; i++) {
+ plane[i] = 0;
+ }
+ for (int i = 0; i < length; i++) {
+ long delta = (long) scaledValues[offset + i] - min;
+ if (((delta >>> bit) & 1L) != 0) {
+ plane[i >>> 3] |= (byte) (1 << (i & 7));
+ }
+ }
+ out.write(plane);
+ }
+ }
+
+ private static void writeInt(ByteArrayOutputStream out, int value) {
+ out.write((value >>> 24) & 0xFF);
+ out.write((value >>> 16) & 0xFF);
+ out.write((value >>> 8) & 0xFF);
+ out.write(value & 0xFF);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return Integer.BYTES + 1;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return (long) values.size() * (Integer.BYTES + 1) + 16;
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntSubColumnEncoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntSubColumnEncoder.java
new file mode 100644
index 00000000000..c4c6c8a9071
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntSubColumnEncoder.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Encoder for INT32 values using block-level sub-column bit-plane packing. */
+public class IntSubColumnEncoder extends Encoder {
+
+ private static final int BLOCK_SIZE = 128;
+
+ private final List<Integer> values = new ArrayList<>();
+
+ public IntSubColumnEncoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ values.add(value);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(values.size(), out);
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(BLOCK_SIZE, out);
+ for (int offset = 0; offset < values.size(); offset += BLOCK_SIZE) {
+ int length = Math.min(BLOCK_SIZE, values.size() - offset);
+ writeBlock(out, offset, length);
+ }
+ values.clear();
+ }
+
+ private void writeBlock(ByteArrayOutputStream out, int offset, int length) throws IOException {
+ int min = values.get(offset);
+ long maxDelta = 0;
+ for (int i = 0; i < length; i++) {
+ int value = values.get(offset + i);
+ if (value < min) {
+ min = value;
+ }
+ }
+ for (int i = 0; i < length; i++) {
+ long delta = (long) values.get(offset + i) - min;
+ if (delta > maxDelta) {
+ maxDelta = delta;
+ }
+ }
+
+ int bitWidth = maxDelta == 0 ? 0 : 64 - Long.numberOfLeadingZeros(maxDelta);
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(length, out);
+ writeInt(out, min);
+ out.write(bitWidth);
+ if (bitWidth == 0) {
+ return;
+ }
+
+ int bytesPerPlane = (length + Byte.SIZE - 1) / Byte.SIZE;
+ byte[] plane = new byte[bytesPerPlane];
+ for (int bit = 0; bit < bitWidth; bit++) {
+ for (int i = 0; i < bytesPerPlane; i++) {
+ plane[i] = 0;
+ }
+ for (int i = 0; i < length; i++) {
+ long delta = (long) values.get(offset + i) - min;
+ if (((delta >>> bit) & 1L) != 0) {
+ plane[i >>> 3] |= (byte) (1 << (i & 7));
+ }
+ }
+ out.write(plane);
+ }
+ }
+
+ private static void writeInt(ByteArrayOutputStream out, int value) {
+ out.write((value >>> 24) & 0xFF);
+ out.write((value >>> 16) & 0xFF);
+ out.write((value >>> 8) & 0xFF);
+ out.write(value & 0xFF);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return Integer.BYTES + 1;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return (long) values.size() * (Integer.BYTES + 1) + 16;
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongSubColumnEncoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongSubColumnEncoder.java
new file mode 100644
index 00000000000..f50294705d5
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongSubColumnEncoder.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Encoder for INT64 values. This keeps SUBCOLUMN schema support lossless for long series. */
+public class LongSubColumnEncoder extends Encoder {
+
+ private final List<Long> values = new ArrayList<>();
+
+ public LongSubColumnEncoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public void encode(long value, ByteArrayOutputStream out) {
+ values.add(value);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(values.size(), out);
+ for (long value : values) {
+ writeLong(out, value);
+ }
+ values.clear();
+ }
+
+ private static void writeLong(ByteArrayOutputStream out, long value) {
+ for (int shift = 56; shift >= 0; shift -= 8) {
+ out.write((int) ((value >>> shift) & 0xFF));
+ }
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return Long.BYTES;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return (long) values.size() * Long.BYTES + 8;
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
index 453aadfed2d..368bab10500 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
@@ -78,6 +78,8 @@ public abstract class TSEncodingBuilder {
return new Sprintz();
case RLBE:
return new RLBE();
+ case SUBCOLUMN:
+ return new SubColumn();
default:
throw new UnsupportedOperationException(type.toString());
}
@@ -353,6 +355,27 @@ public abstract class TSEncodingBuilder {
}
}
+ public static class SubColumn extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case INT32:
+ return new IntSubColumnEncoder();
+ case INT64:
+ return new LongSubColumnEncoder();
+ case FLOAT:
+ return new FloatSubColumnEncoder();
+ default:
+ throw new UnSupportedDataTypeException("SUBCOLUMN doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
public static class Dictionary extends TSEncodingBuilder {
@Override
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
index 56cecda02e9..7c2e547e26b 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
@@ -34,7 +34,8 @@ public enum TSEncoding {
FREQ((byte) 10),
CHIMP((byte) 11),
SPRINTZ((byte) 12),
- RLBE((byte) 13);
+ RLBE((byte) 13),
+ SUBCOLUMN((byte) 14);
private final byte type;
TSEncoding(byte type) {
@@ -79,6 +80,8 @@ public enum TSEncoding {
return TSEncoding.SPRINTZ;
case 13:
return TSEncoding.RLBE;
+ case 14:
+ return TSEncoding.SUBCOLUMN;
default:
throw new IllegalArgumentException("Invalid input: " + encoding);
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CSVCompressTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CSVCompressTest.java
new file mode 100644
index 00000000000..c66b177e862
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CSVCompressTest.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.compress;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class CSVCompressTest {
+
+ private static final String PARENT_DIR = "D://github/xjz17/subcolumn/";
+ private static final String INPUT_PARENT_DIR = PARENT_DIR + "dataset/";
+ private static final String OUTPUT_PARENT_DIR = PARENT_DIR + "result/";
+ private static final String[] HEADERS = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Original Size",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+
+ private static class CompressionBenchmark {
+ private final String algorithm;
+ private final String outputFileName;
+ private final Supplier<ICompressor> compressorSupplier;
+ private final Supplier<IUnCompressor> unCompressorSupplier;
+ private CsvWriter writer;
+
+ private CompressionBenchmark(
+ String algorithm,
+ String outputFileName,
+ Supplier<ICompressor> compressorSupplier,
+ Supplier<IUnCompressor> unCompressorSupplier) {
+ this.algorithm = algorithm;
+ this.outputFileName = outputFileName;
+ this.compressorSupplier = compressorSupplier;
+ this.unCompressorSupplier = unCompressorSupplier;
+ }
+ }
+
+ private static class BenchmarkResult {
+ private final long encodeTime;
+ private final long decodeTime;
+ private final int compressedSize;
+ private final double compressionRatio;
+
+ private BenchmarkResult(
+ long encodeTime, long decodeTime, int compressedSize, double compressionRatio) {
+ this.encodeTime = encodeTime;
+ this.decodeTime = decodeTime;
+ this.compressedSize = compressedSize;
+ this.compressionRatio = compressionRatio;
+ }
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+
+ int REPEAT_TIME = 100;
+ // REPEAT_TIME = 500;
+ // REPEAT_TIME = 10;
+
+ Files.createDirectories(Paths.get(OUTPUT_PARENT_DIR));
+
+ File directory = new File(INPUT_PARENT_DIR);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null || csvFiles.length == 0) {
+ return;
+ }
+ Arrays.sort(csvFiles, Comparator.comparing(File::getName));
+
+ List<CompressionBenchmark> benchmarks = new ArrayList<>();
+ benchmarks.add(
+ new CompressionBenchmark(
+ "GZIP",
+ "compress_gzip.csv",
+ ICompressor.GZIPCompressor::new,
+ IUnCompressor.GZIPUnCompressor::new));
+ benchmarks.add(
+ new CompressionBenchmark(
+ "LZ4",
+ "compress_lz4.csv",
+ ICompressor.IOTDBLZ4Compressor::new,
+ IUnCompressor.LZ4UnCompressor::new));
+ benchmarks.add(
+ new CompressionBenchmark(
+ "SNAPPY",
+ "compress_snappy.csv",
+ ICompressor.SnappyCompressor::new,
+ IUnCompressor.SnappyUnCompressor::new));
+ benchmarks.add(
+ new CompressionBenchmark(
+ "LZMA2",
+ "compress_lzma2.csv",
+ ICompressor.LZMA2Compressor::new,
+ IUnCompressor.LZMA2UnCompressor::new));
+ benchmarks.add(
+ new CompressionBenchmark(
+ "ZSTD",
+ "compress_zstd.csv",
+ ICompressor.ZstdCompressor::new,
+ IUnCompressor.ZstdUnCompressor::new));
+
+ try {
+ for (CompressionBenchmark benchmark : benchmarks) {
+ benchmark.writer =
+ new CsvWriter(
+ OUTPUT_PARENT_DIR + benchmark.outputFileName, ',', StandardCharsets.UTF_8);
+ benchmark.writer.setRecordDelimiter('\n');
+ benchmark.writer.writeRecord(HEADERS);
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ int[] data;
+ try (InputStream inputStream = Files.newInputStream(file.toPath())) {
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> floatData = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String value = loader.getValues()[0];
+ if (value.isEmpty()) {
+ continue;
+ }
+ int currentDecimal = getDecimalPrecision(value);
+ if (currentDecimal > maxDecimal) {
+ maxDecimal = currentDecimal;
+ }
+ floatData.add(Float.valueOf(value));
+ }
+ loader.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+
+ data = new int[floatData.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < floatData.size(); i++) {
+ data[i] = (int) (floatData.get(i) * maxMul);
+ }
+ }
+
+ if (data.length == 0) {
+ continue;
+ }
+
+ ByteBuffer inputBuffer = ByteBuffer.allocate(data.length * Integer.BYTES);
+ for (int value : data) {
+ inputBuffer.putInt(value);
+ }
+ byte[] inputBytes = inputBuffer.array();
+
+ for (CompressionBenchmark benchmark : benchmarks) {
+ ICompressor compressor = benchmark.compressorSupplier.get();
+ IUnCompressor unCompressor = benchmark.unCompressorSupplier.get();
+
+ byte[] compressed = new byte[0];
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < REPEAT_TIME; repeat++) {
+ compressed = compressor.compress(inputBytes);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / REPEAT_TIME;
+
+ byte[] restored = new byte[inputBytes.length];
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < REPEAT_TIME; repeat++) {
+ restored = new byte[inputBytes.length];
+ int uncompressedSize =
+ unCompressor.uncompress(compressed, 0, compressed.length, restored, 0);
+ Assert.assertEquals(inputBytes.length, uncompressedSize);
+ }
+ end = System.nanoTime();
+ long decodeTime = (end - start) / REPEAT_TIME;
+
+ Assert.assertArrayEquals(inputBytes, restored);
+ BenchmarkResult result =
+ new BenchmarkResult(
+ encodeTime,
+ decodeTime,
+ compressed.length,
+ compressed.length / (double) inputBytes.length);
+
+ String[] record = {
+ datasetName,
+ benchmark.algorithm,
+ String.valueOf(result.encodeTime),
+ String.valueOf(result.decodeTime),
+ String.valueOf(data.length),
+ String.valueOf(inputBytes.length),
+ String.valueOf(result.compressedSize),
+ String.valueOf(result.compressionRatio)
+ };
+ benchmark.writer.writeRecord(record);
+ }
+ }
+ } finally {
+ for (CompressionBenchmark benchmark : benchmarks) {
+ if (benchmark.writer != null) {
+ benchmark.writer.close();
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ALPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ALPTest.java
new file mode 100644
index 00000000000..094fbf31565
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ALPTest.java
@@ -0,0 +1,695 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class ALPTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ long buffer = 0;
+ int leftSize = 64;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width * 8 / 8) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockEncoder(double[] data, int block_index, int block_size, int remainder, int encode_pos,
+ int max_decimal,
+ byte[] encoded_result) {
+ int e = 0;
+ int f = 0;
+
+ int base = block_index * block_size;
+
+ e = 17;
+ f = 17 - max_decimal;
+
+ double[] testValues = new double[remainder];
+ for (int i = 0; i < testValues.length; i++) {
+ testValues[i] = data[base + (i * remainder / testValues.length)];
+ }
+
+ outer_loop: for (int i = 0; i < 19; i++) {
+ for (int j = i; j >= 0; j--) {
+ boolean valid = true;
+
+ for (double value : testValues) {
+ // double encodedValue = n * Math.pow(10, i) * Math.pow(10, -j);
+ // double encodedValue = n * power(10, i) / power(10, j);
+ long encodedValue = Math.round(value * power(10, i) / power(10, j));
+
+ double decodedValue = encodedValue / power(10, i) * power(10, j);
+ // if (Math.abs(n - decodedValue) > 0.00001) {
+ if (Math.abs(value - decodedValue) > Math.pow(10, -max_decimal)) {
+ valid = false;
+ break;
+ }
+ }
+
+ if (valid) {
+ e = i;
+ f = j;
+
+ break outer_loop;
+ }
+ }
+ }
+
+ intByte2Bytes(e, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ intByte2Bytes(f, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ long[] data_long = new long[remainder];
+
+ long min_data = Long.MAX_VALUE;
+ long max_data = Long.MIN_VALUE;
+
+ for (int i = 0; i < remainder; i++) {
+ data_long[i] = Math.round(data[base + i] * power(10, e) / power(10, f));
+ if (data_long[i] < min_data) {
+ min_data = data_long[i];
+ }
+ if (data_long[i] > max_data) {
+ max_data = data_long[i];
+ }
+ }
+
+ long2Bytes(min_data, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ for (int i = 0; i < remainder; i++) {
+ data_long[i] -= min_data;
+ }
+ long max_diff = max_data - min_data;
+ int bit_width = bitWidth(max_diff);
+
+ intByte2Bytes(bit_width, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(data_long, bit_width, encode_pos, encoded_result, remainder);
+ return encode_pos;
+
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, double[] data) {
+ int e = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int f = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ long min_data = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ // System.out.println("min_data: " + min_data);
+
+ int bit_width = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ // System.out.println("bit_width: " + bit_width);
+
+ long[] data_long = new long[remainder];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bit_width, remainder, data_long);
+
+ for (int i = 0; i < remainder; i++) {
+ data_long[i] += min_data;
+ // data[block_index * block_size + i] = data_long[i] * Math.pow(10, f) *
+ // Math.pow(10, -e);
+ data[block_index * block_size + i] = data_long[i] * power(10, f) / power(10, e);
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(double[] data, int block_size, int max_decimal, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, max_decimal, encoded_result);
+ }
+
+ if (remainder > 0) {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ max_decimal, encoded_result);
+ }
+
+ return encode_pos;
+ }
+
+ public static double[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ double[] data = new double[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ if (remainder > 0) {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static double power(int base, int exponent) {
+ double result = 1;
+ for (int i = 0; i < exponent; i++) {
+ result *= base;
+ }
+ return result;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "alp.csv";
+
+ int block_size = 1024;
+
+ int repeatTime = 50;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ double[] data1_arr = new double[data1.size()];
+ for (int i = 0; i < data1.size(); i++) {
+ data1_arr[i] = data1.get(i);
+ }
+
+ byte[] encoded_result = new byte[data1.size() * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data1_arr, block_size, max_decimal, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ double[] data2_arr_decoded = new double[data1_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "ALP",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+ @Test
+ public void test1() {
+ double c = Math.pow(10, -17);
+
+ System.out.println(c);
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ASubcolumnAddDictionaryAndHuffmanTest.java
similarity index 54%
copy from iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnTest.java
copy to iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ASubcolumnAddDictionaryAndHuffmanTest.java
index 668b9045687..16c395cd083 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ASubcolumnAddDictionaryAndHuffmanTest.java
@@ -1,5 +1,9 @@
package org.apache.iotdb.tsfile.encoding;
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -8,19 +12,215 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
-import java.util.stream.Stream;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+public class ASubcolumnAddDictionaryAndHuffmanTest {
+
+ private static final class HuffmanNode {
+ final int symbol; // -1 for internal node
+ final int freq;
+ final HuffmanNode left;
+ final HuffmanNode right;
+
+ private HuffmanNode(int symbol, int freq, HuffmanNode left, HuffmanNode right) {
+ this.symbol = symbol;
+ this.freq = freq;
+ this.left = left;
+ this.right = right;
+ }
-import org.junit.Test;
+ boolean isLeaf() {
+ return left == null && right == null;
+ }
+ }
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
+ private static final class HuffmanCode {
+ final int code;
+ final int len;
+
+ private HuffmanCode(int code, int len) {
+ this.code = code;
+ this.len = len;
+ }
+ }
+
+ private static final class HuffmanBuildResult {
+ final int[] symbolsSortedAsc;
+ final byte[] codeLenBySymbolIndex; // aligned with symbolsSortedAsc
+ final HuffmanCode[] canonicalCodesBySymbolIndex; // aligned with symbolsSortedAsc
+ final int totalBits;
+
+ private HuffmanBuildResult(
+ int[] symbolsSortedAsc,
+ byte[] codeLenBySymbolIndex,
+ HuffmanCode[] canonicalCodesBySymbolIndex,
+ int totalBits) {
+ this.symbolsSortedAsc = symbolsSortedAsc;
+ this.codeLenBySymbolIndex = codeLenBySymbolIndex;
+ this.canonicalCodesBySymbolIndex = canonicalCodesBySymbolIndex;
+ this.totalBits = totalBits;
+ }
+ }
+
+ private static void writeBits(int value, int bitLen, byte[] out, int bitPos) {
+ // Write highest bits first.
+ for (int k = bitLen - 1; k >= 0; k--) {
+ boolean b = ((value >>> k) & 1) != 0;
+ boolToBytes(b, out, bitPos++);
+ }
+ }
+
+ private static int readBit(byte[] in, int bitPos) {
+ return bytesToBool(in, bitPos) ? 1 : 0;
+ }
-import static org.junit.Assert.assertEquals;
+ private static void assignCodeLengths(HuffmanNode node, int depth, Map<Integer, Integer> outLen) {
+ if (node.isLeaf()) {
+ // Single-symbol edge case: assign length 1.
+ outLen.put(node.symbol, Math.max(1, depth));
+ return;
+ }
+ assignCodeLengths(node.left, depth + 1, outLen);
+ assignCodeLengths(node.right, depth + 1, outLen);
+ }
-public class SubcolumnTest {
+ private static HuffmanBuildResult buildCanonicalHuffman(int[] values, int valueCount) {
+ // Build frequency table
+ Map<Integer, Integer> freq = new HashMap<>();
+ for (int i = 0; i < valueCount; i++) {
+ freq.merge(values[i], 1, Integer::sum);
+ }
+
+ int cardinality = freq.size();
+ int[] symbols = new int[cardinality];
+ int idx = 0;
+ for (int s : freq.keySet()) {
+ symbols[idx++] = s;
+ }
+ Arrays.sort(symbols);
+
+ // Build Huffman tree
+ PriorityQueue<HuffmanNode> pq = new PriorityQueue<>(
+ Comparator.<HuffmanNode>comparingInt(n -> n.freq).thenComparingInt(n -> n.symbol));
+ for (int s : symbols) {
+ pq.add(new HuffmanNode(s, freq.get(s), null, null));
+ }
+ while (pq.size() > 1) {
+ HuffmanNode a = pq.poll();
+ HuffmanNode b = pq.poll();
+ // Ensure deterministic structure by ordering children by (freq,symbol)
+ HuffmanNode left = a;
+ HuffmanNode right = b;
+ pq.add(new HuffmanNode(-1, a.freq + b.freq, left, right));
+ }
+ HuffmanNode root = pq.poll();
+
+ Map<Integer, Integer> lenBySymbol = new HashMap<>();
+ assignCodeLengths(root, 0, lenBySymbol);
+
+ // Prepare canonical order: by (len, symbol)
+ Integer[] order = new Integer[cardinality];
+ for (int i = 0; i < cardinality; i++) {
+ order[i] = symbols[i];
+ }
+ Arrays.sort(order, Comparator.<Integer>comparingInt(s -> lenBySymbol.get(s)).thenComparingInt(s -> s));
+
+ // Assign canonical codes
+ Map<Integer, HuffmanCode> codeBySymbol = new HashMap<>();
+ int code = 0;
+ int prevLen = lenBySymbol.get(order[0]);
+ codeBySymbol.put(order[0], new HuffmanCode(code, prevLen));
+ for (int i = 1; i < order.length; i++) {
+ int s = order[i];
+ int len = lenBySymbol.get(s);
+ code = (code + 1) << (len - prevLen);
+ codeBySymbol.put(s, new HuffmanCode(code, len));
+ prevLen = len;
+ }
+
+ // Align to symbolsSortedAsc for compact lookup
+ byte[] codeLens = new byte[cardinality];
+ HuffmanCode[] codes = new HuffmanCode[cardinality];
+ int totalBits = 0;
+ for (int i = 0; i < cardinality; i++) {
+ int s = symbols[i];
+ HuffmanCode hc = codeBySymbol.get(s);
+ codeLens[i] = (byte) hc.len;
+ codes[i] = hc;
+ totalBits += freq.get(s) * hc.len;
+ }
+
+ return new HuffmanBuildResult(symbols, codeLens, codes, totalBits);
+ }
+
+ private static Map<Integer, HuffmanCode> rebuildCanonicalCodes(int[] symbolsSortedAsc, byte[] codeLens) {
+ int cardinality = symbolsSortedAsc.length;
+ Integer[] order = new Integer[cardinality];
+ for (int i = 0; i < cardinality; i++) {
+ order[i] = symbolsSortedAsc[i];
+ }
+ Map<Integer, Integer> lenBySymbol = new HashMap<>();
+ for (int i = 0; i < cardinality; i++) {
+ lenBySymbol.put(symbolsSortedAsc[i], codeLens[i] & 0xFF);
+ }
+ Arrays.sort(order, Comparator.<Integer>comparingInt(s -> lenBySymbol.get(s)).thenComparingInt(s -> s));
+
+ Map<Integer, HuffmanCode> codeBySymbol = new HashMap<>();
+ int code = 0;
+ int prevLen = lenBySymbol.get(order[0]);
+ codeBySymbol.put(order[0], new HuffmanCode(code, prevLen));
+ for (int i = 1; i < order.length; i++) {
+ int s = order[i];
+ int len = lenBySymbol.get(s);
+ code = (code + 1) << (len - prevLen);
+ codeBySymbol.put(s, new HuffmanCode(code, len));
+ prevLen = len;
+ }
+ return codeBySymbol;
+ }
+
+ private static final class MutableTrieNode {
+ MutableTrieNode zero;
+ MutableTrieNode one;
+ int symbol = -1;
+ }
+
+ private static MutableTrieNode buildMutableDecodeTrie(Map<Integer, HuffmanCode> codeBySymbol) {
+ MutableTrieNode root = new MutableTrieNode();
+ for (Map.Entry<Integer, HuffmanCode> e : codeBySymbol.entrySet()) {
+ int sym = e.getKey();
+ HuffmanCode hc = e.getValue();
+ MutableTrieNode cur = root;
+ for (int k = hc.len - 1; k >= 0; k--) {
+ int b = (hc.code >>> k) & 1;
+ if (b == 0) {
+ if (cur.zero == null) {
+ cur.zero = new MutableTrieNode();
+ }
+ cur = cur.zero;
+ } else {
+ if (cur.one == null) {
+ cur.one = new MutableTrieNode();
+ }
+ cur = cur.one;
+ }
+ }
+ cur.symbol = sym;
+ }
+ return root;
+ }
public static int bitWidth(int value) {
+ if (value == 0)
+ return 1;
return 32 - Integer.numberOfLeadingZeros(value);
}
@@ -247,6 +447,44 @@ public class SubcolumnTest {
int cMin = Integer.MAX_VALUE;
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+
+ int cost0 = 0;
+
+ for (int i = 0; i < m; i++) {
+ int current_value = (x[0] >> i) & 1;
+
+ int count = 1;
+
+ de_cost_single[i] = 1;
+
+ for (int j = 0; j < x_length; j++) {
+
+ // if (count * (1 + (int) Math.ceil(Math.log(x_length))) >= x_length) {
+ // rle_cost_single[i] = x_length + 1;
+ // break;
+ // }
+
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length + 2 * (1 + 1);
+ }
+
+ }
+
+ rle_cost_single[i] = count * (1 + (int) Math.ceil(Math.log(x_length)));
+
+ cost0 += Math.min(bpe_cost_single[i], Math.min(rle_cost_single[i], de_cost_single[i]));
+ }
+
// int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
// int[] beta_list = { 1, 2, 3, 5, 7, 11 };
// int[] beta_list = { 1, 2, 3, 4 };
@@ -266,42 +504,78 @@ public class SubcolumnTest {
// System.out.println("l: " + l);
- int[][] subcolumnList = new int[l][x_length];
+ // int[][] subcolumnList = new int[l][x_length];
int cost = 0;
+ // for (int i = 0; i < l; i++) {
+ // int maxValuePart = 0;
+ // for (int j = 0; j < x_length; j++) {
+ // subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ // if (subcolumnList[i][j] > maxValuePart) {
+ // maxValuePart = subcolumnList[i][j];
+ // }
+ // }
+ // bitWidthListList[i] = bitWidth(maxValuePart);
+ // }
+
for (int i = 0; i < l; i++) {
- int maxValuePart = 0;
- for (int j = 0; j < x_length; j++) {
- subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
- if (subcolumnList[i][j] > maxValuePart) {
- maxValuePart = subcolumnList[i][j];
- }
+ // int bpCost = bitWidthListList[i] * x_length;
+
+ // int bpCost = bpe_cost_single[i * beta] * beta;
+ int beta_start = (Math.min(m - 1, (i + 1) * beta - 1));
+ while (beta_start - 1 >= i * beta && bpe_cost_single[beta_start - 1] == 0) {
+ beta_start--;
}
- bitWidthListList[i] = bitWidth(maxValuePart);
- }
- for (int i = 0; i < l; i++) {
- int bpCost = bitWidthListList[i] * x_length;
+ int bpCost = bpe_cost_single[beta_start] * (beta_start - i * beta + 1);
+
int rleCost = 0;
- // int count = 1;
- int currentNumber = subcolumnList[i][0];
+ // int lowestBitIndex = 0;
+ // int currentLowestBit = subcolumnList[i][0] & 1;
+
+ // for (int j = 1; j < x_length; j++) {
+ // int lowestBit = subcolumnList[i][j] & 1; // 获取当前元素的最低位
+ // if (lowestBit != currentLowestBit) {
+ // lowestBitIndex++;
+ // currentLowestBit = lowestBit;
+ // }
+ // }
+
+ // if (bw * lowestBitIndex + bitWidthListList[i] * lowestBitIndex >= bpCost) {
+ // cost += bpCost;
+ // continue;
+ // }
int index = 0;
boolean bpBest = false;
+ // int count = 1;
+ // int currentNumber = subcolumnList[i][0];
+ int currentNumber = (x[0] >> (i * beta)) & ((1 << beta) - 1);
+
for (int j = 1; j < x_length; j++) {
- if (subcolumnList[i][j] != currentNumber) {
+ int currentNumber_j = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (currentNumber_j != currentNumber) {
index++;
- currentNumber = subcolumnList[i][j];
+ currentNumber = currentNumber_j;
}
-
- if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ if (bw * index + bitWidth(x_length) * index >= bpCost) {
bpBest = true;
break;
}
+
+ // if (subcolumnList[i][j] != currentNumber) {
+ // index++;
+ // currentNumber = subcolumnList[i][j];
+ // }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
}
if (bpBest) {
@@ -313,7 +587,7 @@ public class SubcolumnTest {
// System.out.println("index: " + index);
- rleCost = bw * index + bitWidthListList[i] * index;
+ rleCost = bw * index + bitWidth(x_length) * index;
// System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
@@ -338,9 +612,9 @@ public class SubcolumnTest {
public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
int list_length = list.length;
int maxValue = 0;
- for (int i = 0; i < list_length; i++) {
- if (list[i] > maxValue) {
- maxValue = list[i];
+ for (int k : list) {
+ if (k > maxValue) {
+ maxValue = k;
}
}
@@ -354,7 +628,6 @@ public class SubcolumnTest {
}
// int[] bitWidthList = new int[m];
-
// int[][] subcolumnList = new int[m][list_length];
int l;
@@ -390,9 +663,9 @@ public class SubcolumnTest {
int[] encodingType = new int[l];
- // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ // encoded_result 预留大小为 (l + 7) * 2 / 8 的大小，存储每个分列的类型
int preTypePos = encode_pos;
- encode_pos += (l + 7) / 8;
+ encode_pos += (l + 3) / 4;
for (int i = l - 1; i >= 0; i--) {
// 对于每个分列，计算使用 bit packing 还是 rle
@@ -402,8 +675,15 @@ public class SubcolumnTest {
int previous = subcolumnList[i][0];
int index = 0;
+ // uniqueValues.add(previous);
+
for (int j = 1; j < list_length; j++) {
int currentNumber = subcolumnList[i][j];
+
+ // if(currentNumber == 6){
+ // System.out.println("currentNumber == 6 && i==0");
+ // System.out.println(uniqueValues);
+ // }
if (currentNumber != previous) {
index++;
previous = currentNumber;
@@ -413,11 +693,122 @@ public class SubcolumnTest {
break;
}
}
+ Set<Integer> uniqueValues = new HashSet<>();
+ Map<Integer, Integer> freq = new HashMap<>();
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ uniqueValues.add(currentNumber);
+ freq.merge(currentNumber, 1, Integer::sum);
+ }
+ int cardinality = uniqueValues.size();
index++;
rleCost = bw * index + bitWidthList[i] * index;
+ // Huffman as the 4th option for each subcolumn
+ // We store: [cardinality:2B][totalBits:4B][symbols(bitpacked)][codeLen(bytes)][bitstream]
+ int hufTotalBits;
+ int hufOverheadBits;
+ HuffmanBuildResult hufBuild = null;
+ try {
+ hufBuild = buildCanonicalHuffman(subcolumnList[i], list_length);
+ hufTotalBits = hufBuild.totalBits;
+ // overhead: 2B + 4B + symbols(bitWidthList[i] * cardinality) + codeLen(8 * cardinality)
+ hufOverheadBits = (2 + 4) * 8 + bitWidthList[i] * cardinality + 8 * cardinality;
+ } catch (Exception ex) {
+ // If anything goes wrong, just disable Huffman for this subcolumn
+ hufTotalBits = Integer.MAX_VALUE / 2;
+ hufOverheadBits = Integer.MAX_VALUE / 2;
+ }
+ int hufCostBits = (hufTotalBits >= Integer.MAX_VALUE / 4) ? Integer.MAX_VALUE : (hufTotalBits + hufOverheadBits);
+
+ if (cardinality < Math.pow(2, bitWidthList[i] - 1)) {
+ // test dictionary encoding
+ int dict_bit_width = bitWidth(cardinality);
+ int dicCost = dict_bit_width * list_length + cardinality * (bitWidthList[i] + dict_bit_width);
+ if (dicCost < rleCost && dicCost < bpCost) {
+ // if dictionary encoding
+ // int dict_bit_width = bitWidth(cardinality) ;
+ encodingType[i] = 2;
+
+ // System.out.println(uniqueValues);
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dict_key_list = new int[cardinality];
+ // int[] dict_value_list = new int[cardinality];
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dict_key_list[j] = sortedUnique.get(j);
+ // dict_value_list[j] = j;
+ }
+ // System.out.println(valueToCode);
+ // System.out.println(list_length);
+ // System.out.println(beta[0]);
+ // System.out.println(Arrays.toString(subcolumnList[i]));
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(dict_key_list, bitWidthList[i], encode_pos, encoded_result, cardinality);
+ // encode_pos = bitPacking(dict_value_list, dict_bit_width, encode_pos,
+ // encoded_result, cardinality);
+
+ encode_pos = bitPacking(subcolumnList[i], dict_bit_width, encode_pos, encoded_result, list_length);
+ continue;
+ }
+ }
+
+ // If Huffman wins, encode Huffman
+ if (hufCostBits < bpCost && hufCostBits < rleCost) {
+ encodingType[i] = 3;
+
+ // cardinality
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ // totalBits
+ int2Bytes(hufBuild.totalBits, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ // symbols sorted asc
+ encode_pos = bitPacking(hufBuild.symbolsSortedAsc, bitWidthList[i], encode_pos, encoded_result, cardinality);
+
+ // code lengths in bytes aligned with symbolsSortedAsc
+ for (int j = 0; j < cardinality; j++) {
+ encoded_result[encode_pos + j] = hufBuild.codeLenBySymbolIndex[j];
+ }
+ encode_pos += cardinality;
+
+ // build lookup from symbol->(code,len) by symbolsSortedAsc index
+ Map<Integer, HuffmanCode> codeBySymbol = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ codeBySymbol.put(hufBuild.symbolsSortedAsc[j], hufBuild.canonicalCodesBySymbolIndex[j]);
+ }
+
+ int bitPos = encode_pos * 8;
+ int bytesToWrite = (hufBuild.totalBits + 7) / 8;
+ Arrays.fill(encoded_result, encode_pos, encode_pos + bytesToWrite, (byte) 0);
+ for (int j = 0; j < list_length; j++) {
+ int sym = subcolumnList[i][j];
+ HuffmanCode hc = codeBySymbol.get(sym);
+ writeBits(hc.code, hc.len, encoded_result, bitPos);
+ bitPos += hc.len;
+ }
+ encode_pos += bytesToWrite;
+ continue;
+ }
+
if (bpCost <= rleCost) {
encodingType[i] = 0;
@@ -458,7 +849,7 @@ public class SubcolumnTest {
}
- preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+ preTypePos = bitPacking(encodingType, 2, preTypePos, encoded_result, l);
return encode_pos;
}
@@ -489,7 +880,7 @@ public class SubcolumnTest {
int[] encodingType = new int[l];
- encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 2, l, encodingType);
for (int i = l - 1; i >= 0; i--) {
int type = encodingType[i];
@@ -497,7 +888,7 @@ public class SubcolumnTest {
if (type == 0) {
encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
subcolumnList[i]);
- } else {
+ } else if (type == 1) {
int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
encode_pos += 2;
@@ -517,6 +908,68 @@ public class SubcolumnTest {
currentIndex++;
}
}
+ } else if (type == 2) {
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+ int dict_bit_width = bitWidth(cardinality);
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ dict_value_list[j] = j;
+ }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, dict_key_list);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width,
+ // cardinality, dict_value_list);
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, list_length,
+ subcolumnList[i]);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(dict_value_list[j], dict_key_list[j]);
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ } else {
+ // Huffman
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ int totalBits = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] symbolsSortedAsc = new int[cardinality];
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, symbolsSortedAsc);
+
+ byte[] codeLens = new byte[cardinality];
+ for (int j = 0; j < cardinality; j++) {
+ codeLens[j] = encoded_result[encode_pos + j];
+ }
+ encode_pos += cardinality;
+
+ Map<Integer, HuffmanCode> codeBySymbol = rebuildCanonicalCodes(symbolsSortedAsc, codeLens);
+ MutableTrieNode trie = buildMutableDecodeTrie(codeBySymbol);
+
+ int bitPos = encode_pos * 8;
+ int bitsRead = 0;
+ for (int j = 0; j < list_length; j++) {
+ MutableTrieNode cur = trie;
+ while (cur.symbol < 0) {
+ int b = readBit(encoded_result, bitPos++);
+ bitsRead++;
+ cur = (b == 0) ? cur.zero : cur.one;
+ }
+ subcolumnList[i][j] = cur.symbol;
+ }
+
+ // advance bytes by totalBits (trusted) rather than bitsRead (derived)
+ encode_pos += (totalBits + 7) / 8;
}
}
@@ -568,7 +1021,7 @@ public class SubcolumnTest {
int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
remainder, min_delta);
-
+
int2Bytes(min_delta[0], encode_pos, encoded_result);
encode_pos += 4;
@@ -709,25 +1162,23 @@ public class SubcolumnTest {
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
- String outputPath = output_parent_dir + "subcolumn.csv";
+ String outputPath = output_parent_dir + "subcolumn_dictionary_huffman.csv";
+ // int block_size = 512;
int block_size = 512;
- int repeatTime = 100;
+ // int repeatTime = 100;
+ int repeatTime = 10;
// repeatTime = 1;
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -749,6 +1200,9 @@ public class SubcolumnTest {
for (File file : csvFiles) {
String datasetName = extractFileName(file.toString());
System.out.println(datasetName);
+ // if(! datasetName.equals("Stocks-UK")){
+ // continue;
+ // }
InputStream inputStream = Files.newInputStream(file.toPath());
@@ -768,6 +1222,11 @@ public class SubcolumnTest {
data1.add(Float.valueOf(f_str));
}
inputStream.close();
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
int[] data2_arr = new int[data1.size()];
int max_mul = (int) Math.pow(10, max_decimal);
@@ -796,11 +1255,7 @@ public class SubcolumnTest {
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -818,12 +1273,12 @@ public class SubcolumnTest {
decodeTime += ((e - s) / repeatTime);
for (int i = 0; i < data2_arr_decoded.length; i++) {
- assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
}
String[] record = {
datasetName,
- "Sub-columns",
+ "Sub-columns (Dictionary)",
String.valueOf(encodeTime),
String.valueOf(decodeTime),
String.valueOf(data1.size()),
@@ -837,151 +1292,4 @@ public class SubcolumnTest {
writer.close();
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "subcolumn.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- // CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
- // writer.setRecordDelimiter('\n');
-
- // String[] head = {
- // "Input Direction",
- // "Encoding Algorithm",
- // "Encoding Time",
- // "Decoding Time",
- // "Points",
- // "Compressed Size",
- // "Compression Ratio"
- // };
- // writer.writeRecord(head);
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- int[] data2_arr_decoded = new int[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- for (int i = 0; i < data2_arr_decoded.length; i++) {
- assertEquals(data2_arr[i], data2_arr_decoded[i]);
- }
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "Sub-columns",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPLongTest.java
similarity index 53%
copy from iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java
copy to iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPLongTest.java
index 36f5a35afdc..b23b8f7aac5 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPLongTest.java
@@ -16,12 +16,16 @@ import org.junit.Test;
import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;
-public class BPTest {
+public class BPLongTest {
public static int bitWidth(int value) {
return 32 - Integer.numberOfLeadingZeros(value);
}
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
int cnt = pos & 0x07;
int index = pos >> 3;
@@ -62,20 +66,57 @@ public class BPTest {
return ret;
}
- public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
byte[] encoded_result) {
int bufIdx = 0;
int valueIdx = offset;
- // remaining bits for the current unfinished Integer
int leftBit = 0;
while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
int buffer = 0;
- // remaining size of bits in the 'buffer'
int leftSize = 32;
- // encode the left bits of current Integer to 'buffer'
if (leftBit > 0) {
buffer |= (values[valueIdx] << (32 - leftBit));
leftSize -= leftBit;
@@ -84,20 +125,16 @@ public class BPTest {
}
while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
buffer |= (values[valueIdx] << (leftSize - width));
leftSize -= width;
valueIdx++;
}
- // If the remaining space of the buffer can not save the bits for one Integer,
+
if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
buffer |= (values[valueIdx] >>> (width - leftSize));
leftBit = width - leftSize;
}
- // put the buffer into the final result
for (int j = 0; j < 4; j++) {
encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
encode_pos++;
@@ -110,28 +147,58 @@ public class BPTest {
}
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ long buffer = 0;
+ int leftSize = 64;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width * 8 / 8) {
+ return;
+ }
+ }
+ }
+ }
+
public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
int byteIdx = offset;
long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
int totalBits = 0;
int valueIdx = 0;
while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
while (totalBits < width) {
buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
byteIdx++;
totalBits += 8;
}
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
while (totalBits >= width && valueIdx < 8) {
- // result_list.add((int) (buffer >>> (totalBits - width)));
result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
valueIdx++;
totalBits -= width;
@@ -140,6 +207,29 @@ public class BPTest {
}
}
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
byte[] encoded_result, int num_values) {
int block_num = num_values / 8;
@@ -160,14 +250,32 @@ public class BPTest {
return (encode_pos + 7) / 8;
}
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
public static int decodeBitPacking(
byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
- // ArrayList<Integer> result_list = new ArrayList<>();
- // int[] result_list = new int[num_values];
int block_num = num_values / 8;
int remainder = num_values % 8;
- for (int i = 0; i < block_num; i++) { // bitpacking
+ for (int i = 0; i < block_num; i++) {
unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
decode_pos += bit_width;
}
@@ -182,9 +290,73 @@ public class BPTest {
return (decode_pos + 7) / 8;
}
- public static int BPEncoder(int[] list, int encode_pos, byte[] encoded_result) {
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BPEncoder(long[] list, int encode_pos, byte[] encoded_result) {
int list_length = list.length;
- int maxValue = 0;
+ long maxValue = 0;
for (int i = 0; i < list_length; i++) {
if (list[i] > maxValue) {
maxValue = list[i];
@@ -192,27 +364,22 @@ public class BPTest {
}
int m = bitWidth(maxValue);
- // System.out.println("m: " + m);
- // writeBits(encoded_result, startBitPosition, 8, m);
- // startBitPosition += 8;
encoded_result[encode_pos] = (byte) m;
encode_pos += 1;
- // bitPacking(list, encoded_result, startBitPosition, m, list_length);
- // startBitPosition += m * list_length;
encode_pos = bitPacking(list, m, encode_pos, encoded_result, list_length);
return encode_pos;
}
- public static int BPDecoder(byte[] encoded_result, int encode_pos, int[] list) {
+ public static int BPDecoder(byte[] encoded_result, int encode_pos, long[] list) {
int list_length = list.length;
int m = encoded_result[encode_pos];
encode_pos += 1;
- int[] new_list = new int[list_length];
+ long[] new_list = new long[list_length];
encode_pos = decodeBitPacking(encoded_result, encode_pos, m, list_length, new_list);
for (int i = 0; i < list_length; i++) {
@@ -222,21 +389,21 @@ public class BPTest {
return encode_pos;
}
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
int i,
int block_size,
int remaining,
- int[] min_delta) {
- int[] ts_block_delta = new int[remaining];
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
- int value_delta_min = Integer.MAX_VALUE;
- int value_delta_max = Integer.MIN_VALUE;
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
int base = i * block_size;
int end = i * block_size + remaining;
for (int j = base; j < end; j++) {
- int cur = ts_block[j];
+ long cur = ts_block[j];
if (cur < value_delta_min) {
value_delta_min = cur;
}
@@ -254,18 +421,15 @@ public class BPTest {
return ts_block_delta;
}
- public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
int encode_pos, byte[] encoded_result) {
- int[] min_delta = new int[3];
+ long[] min_delta = new long[3];
- int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
remainder, min_delta);
- encoded_result[encode_pos] = (byte) (min_delta[0] >> 24);
- encoded_result[encode_pos + 1] = (byte) (min_delta[0] >> 16);
- encoded_result[encode_pos + 2] = (byte) (min_delta[0] >> 8);
- encoded_result[encode_pos + 3] = (byte) min_delta[0];
- encode_pos += 4;
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
encode_pos = BPEncoder(data_delta, encode_pos,
encoded_result);
@@ -274,14 +438,13 @@ public class BPTest {
}
public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
- int encode_pos, int[] data) {
- int[] min_delta = new int[3];
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
- min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
- ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
- encode_pos += 4;
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
- int[] block_data = new int[remainder];
+ long[] block_data = new long[remainder];
encode_pos = BPDecoder(encoded_result, encode_pos,
block_data);
@@ -293,7 +456,7 @@ public class BPTest {
return encode_pos;
}
- public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
int data_length = data.length;
int encode_pos = 0;
@@ -319,12 +482,9 @@ public class BPTest {
if (remainder <= 3) {
for (int i = 0; i < remainder; i++) {
- int value = data[num_blocks * block_size + i];
- encoded_result[encode_pos] = (byte) (value >> 24);
- encoded_result[encode_pos + 1] = (byte) (value >> 16);
- encoded_result[encode_pos + 2] = (byte) (value >> 8);
- encoded_result[encode_pos + 3] = (byte) value;
- encode_pos += 4;
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
}
} else {
encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
@@ -334,21 +494,18 @@ public class BPTest {
return encode_pos;
}
- public static int[] Decoder(byte[] encoded_result) {
+ public static long[] Decoder(byte[] encoded_result) {
int encode_pos = 0;
- int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
- |
- ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
encode_pos += 4;
- int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
- ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
encode_pos += 4;
int num_blocks = data_length / block_size;
- int[] data = new int[data_length];
+ long[] data = new long[data_length];
for (int i = 0; i < num_blocks; i++) {
encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
@@ -358,10 +515,8 @@ public class BPTest {
if (remainder <= 3) {
for (int i = 0; i < remainder; i++) {
- data[num_blocks * block_size + i] = ((encoded_result[encode_pos] & 0xFF) << 24) |
- ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
- ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
- encode_pos += 4;
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
}
} else {
encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
@@ -372,15 +527,12 @@ public class BPTest {
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -402,24 +554,21 @@ public class BPTest {
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ // String parent_dir = "path/to/your/directory/";
+ String parent_dir = "D:/encoding-subcolumn/";
- String input_parent_dir = parent_dir + "dataset/";
+ // String input_parent_dir = parent_dir + "dataset/";
+ String input_parent_dir = parent_dir + "ElfTestData_camel/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
- String outputPath = output_parent_dir + "bp.csv";
+ // String outputPath = output_parent_dir + "bp_long.csv";
+ String outputPath = output_parent_dir + "bp_long2.csv";
int block_size = 1024;
- int repeatTime = 100;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
+ int repeatTime = 50;
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -436,7 +585,6 @@ public class BPTest {
writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -446,7 +594,7 @@ public class BPTest {
InputStream inputStream = Files.newInputStream(file.toPath());
CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
+ ArrayList<Double> data1 = new ArrayList<>();
int max_decimal = 0;
while (loader.readRecord()) {
@@ -458,20 +606,23 @@ public class BPTest {
if (cur_decimal > max_decimal) {
max_decimal = cur_decimal;
}
- // String value = loader.getValues()[index];
- data1.add(Float.valueOf(f_str));
- // data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
+ data1.add(Double.valueOf(f_str));
}
inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
}
System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
+ byte[] encoded_result = new byte[data2_arr.length * 8];
long encodeTime = 0;
long decodeTime = 0;
@@ -491,17 +642,13 @@ public class BPTest {
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
System.out.println("Decode");
- int[] data2_arr_decoded = new int[data1.size()];
+ long[] data2_arr_decoded = new long[data1.size()];
s = System.nanoTime();
@@ -528,133 +675,4 @@ public class BPTest {
writer.close();
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "bp.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- int[] data2_arr_decoded = new int[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "BP",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQueryGroupTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQueryGroupTest.java
new file mode 100644
index 00000000000..ddf28aab7eb
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQueryGroupTest.java
@@ -0,0 +1,126 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPQueryGroupTest {
+
+ public static int[] queryGroupMaxIndexByDecode(byte[] encodedResult, int windowSize) {
+ int[] decoded = BPTest.Decoder(encodedResult);
+ int groupCount = (decoded.length + windowSize - 1) / windowSize;
+ int[] result = new int[groupCount];
+ for (int g = 0; g < groupCount; g++) {
+ int start = g * windowSize;
+ int end = Math.min(decoded.length, start + windowSize);
+ int bestIndex = start;
+ int bestValue = decoded[start];
+ for (int i = start + 1; i < end; i++) {
+ if (decoded[i] > bestValue) {
+ bestValue = decoded[i];
+ bestIndex = i;
+ }
+ }
+ result[g] = bestIndex;
+ }
+ return result;
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "bp_query_group_max.csv";
+
+ int blockSize = 512;
+ int repeatTime = 100;
+ int windowSize = 30;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = BPTest.extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = BPTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ int[] dataArr = new int[data.size()];
+ for (int i = 0; i < data.size(); i++) {
+ dataArr[i] = (int) (data.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, dataArr.length * 8)];
+ int length = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = BPTest.Encoder(dataArr, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+
+ int[] groupResult = null;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ groupResult = queryGroupMaxIndexByDecode(encodedResult, windowSize);
+ }
+ end = System.nanoTime();
+ long queryTime = (end - start) / repeatTime;
+ System.out.println("groupCount: " + (groupResult == null ? 0 : groupResult.length));
+
+ double compressionRatio = length / (double) (Math.max(1, data.size()) * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "BP",
+ String.valueOf(encodeTime),
+ String.valueOf(queryTime),
+ String.valueOf(data.size()),
+ String.valueOf(length),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQuerySortTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQuerySortTest.java
new file mode 100644
index 00000000000..753e49dcda1
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQuerySortTest.java
@@ -0,0 +1,142 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPQuerySortTest {
+
+ public static int[] querySortSingleBlockByDecode(byte[] encodedResult, int targetBlockId) {
+ int[] decoded = BPTest.Decoder(encodedResult);
+ int blockSize = ((encodedResult[4] & 0xFF) << 24)
+ | ((encodedResult[5] & 0xFF) << 16)
+ | ((encodedResult[6] & 0xFF) << 8)
+ | (encodedResult[7] & 0xFF);
+ int dataLength = decoded.length;
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int totalBlocks = numBlocks + (remainder > 0 ? 1 : 0);
+ if (targetBlockId < 0 || targetBlockId >= totalBlocks) {
+ return new int[0];
+ }
+
+ int count = targetBlockId < numBlocks ? blockSize : remainder;
+ int start = targetBlockId * blockSize;
+ int[] indices = new int[count];
+ for (int i = 0; i < count; i++) {
+ indices[i] = start + i;
+ }
+
+ for (int i = 0; i < count - 1; i++) {
+ int min = i;
+ for (int j = i + 1; j < count; j++) {
+ int v1 = decoded[indices[j]];
+ int v2 = decoded[indices[min]];
+ if (v1 < v2 || (v1 == v2 && indices[j] < indices[min])) {
+ min = j;
+ }
+ }
+ int t = indices[i];
+ indices[i] = indices[min];
+ indices[min] = t;
+ }
+ return indices;
+ }
+
+ @Test
+ public void testDecodeSortSingleBlock() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "bp_query_sort_decode.csv";
+
+ int blockSize = 512;
+ int repeatTime = 100;
+ int targetBlockId = 0;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = BPTest.extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = BPTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ int[] dataArr = new int[data.size()];
+ for (int i = 0; i < data.size(); i++) {
+ dataArr[i] = (int) (data.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, dataArr.length * 8)];
+ int length = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = BPTest.Encoder(dataArr, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+
+ int[] sortedIndices = null;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ sortedIndices = querySortSingleBlockByDecode(encodedResult, targetBlockId);
+ }
+ end = System.nanoTime();
+ long queryTime = (end - start) / repeatTime;
+ System.out.println("sortedCount: " + (sortedIndices == null ? 0 : sortedIndices.length));
+
+ double compressionRatio = length / (double) (Math.max(1, data.size()) * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "BP",
+ String.valueOf(encodeTime),
+ String.valueOf(queryTime),
+ String.valueOf(data.size()),
+ String.valueOf(length),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java
index 36f5a35afdc..b7552ad1719 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java
@@ -18,23 +18,43 @@ import com.csvreader.CsvWriter;
public class BPTest {
+ private static final int PACK_BIT_STEP = 4;
+ private static final int BIT_IO_STEP = 4;
+
public static int bitWidth(int value) {
return 32 - Integer.numberOfLeadingZeros(value);
}
+ private static void storeByteBits(byte[] result, int index, int byteVal) {
+ storeByteBits(result, index, byteVal, PACK_BIT_STEP);
+ }
+
+ private static void storeByteBits(byte[] result, int index, int byteVal, int step) {
+ int bitIndex = 8;
+ int remaining = 8;
+ while (remaining > 0) {
+ int bitsToWrite = Math.min(step, Math.min(bitIndex, remaining));
+ bitIndex -= bitsToWrite;
+ int mask = (1 << bitsToWrite) - 1;
+ int bits = (byteVal >> (remaining - bitsToWrite)) & mask;
+ result[index] &= (byte) ~(mask << bitIndex);
+ result[index] |= (byte) (bits << bitIndex);
+ remaining -= bitsToWrite;
+ }
+ }
+
public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
int cnt = pos & 0x07;
int index = pos >> 3;
while (width > 0) {
- int m = width + cnt >= 8 ? 8 - cnt : width;
+ int available = 8 - cnt;
+ int m = Math.min(BIT_IO_STEP, Math.min(available, width));
width -= m;
- int mask = 1 << (8 - cnt);
+ int mask = (1 << m) - 1;
+ int bits = (srcNum >> width) & mask;
+ int byteMask = mask << (8 - cnt - m);
+ result[index] = (byte) (result[index] & ~byteMask | (bits << (8 - cnt - m)));
cnt += m;
- byte y = (byte) (srcNum >>> width);
- y = (byte) (y << (8 - cnt));
- mask = ~(mask - (1 << (8 - cnt)));
- result[index] = (byte) (result[index] & mask | y);
- srcNum = srcNum & ~(-1 << width);
if (cnt == 8) {
index++;
cnt = 0;
@@ -66,16 +86,12 @@ public class BPTest {
byte[] encoded_result) {
int bufIdx = 0;
int valueIdx = offset;
- // remaining bits for the current unfinished Integer
int leftBit = 0;
while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
int buffer = 0;
- // remaining size of bits in the 'buffer'
int leftSize = 32;
- // encode the left bits of current Integer to 'buffer'
if (leftBit > 0) {
buffer |= (values[valueIdx] << (32 - leftBit));
leftSize -= leftBit;
@@ -84,22 +100,23 @@ public class BPTest {
}
while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
buffer |= (values[valueIdx] << (leftSize - width));
leftSize -= width;
valueIdx++;
}
- // If the remaining space of the buffer can not save the bits for one Integer,
+
if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
buffer |= (values[valueIdx] >>> (width - leftSize));
leftBit = width - leftSize;
}
- // put the buffer into the final result
for (int j = 0; j < 4; j++) {
- encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ int outByte = (buffer >>> ((3 - j) * 8)) & 0xFF;
+ if (j == 3) {
+ storeByteBits(encoded_result, encode_pos, outByte, 3);
+ } else {
+ storeByteBits(encoded_result, encode_pos, outByte);
+ }
encode_pos++;
bufIdx++;
if (bufIdx >= width) {
@@ -107,31 +124,22 @@ public class BPTest {
}
}
}
-
}
public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
int byteIdx = offset;
long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
int totalBits = 0;
int valueIdx = 0;
while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
while (totalBits < width) {
buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
byteIdx++;
totalBits += 8;
}
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
while (totalBits >= width && valueIdx < 8) {
- // result_list.add((int) (buffer >>> (totalBits - width)));
result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
valueIdx++;
totalBits -= width;
@@ -162,12 +170,10 @@ public class BPTest {
public static int decodeBitPacking(
byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
- // ArrayList<Integer> result_list = new ArrayList<>();
- // int[] result_list = new int[num_values];
int block_num = num_values / 8;
int remainder = num_values % 8;
- for (int i = 0; i < block_num; i++) { // bitpacking
+ for (int i = 0; i < block_num; i++) {
unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
decode_pos += bit_width;
}
@@ -192,15 +198,10 @@ public class BPTest {
}
int m = bitWidth(maxValue);
- // System.out.println("m: " + m);
- // writeBits(encoded_result, startBitPosition, 8, m);
- // startBitPosition += 8;
encoded_result[encode_pos] = (byte) m;
encode_pos += 1;
- // bitPacking(list, encoded_result, startBitPosition, m, list_length);
- // startBitPosition += m * list_length;
encode_pos = bitPacking(list, m, encode_pos, encoded_result, list_length);
return encode_pos;
@@ -372,15 +373,12 @@ public class BPTest {
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -402,24 +400,19 @@ public class BPTest {
}
@Test
- public void testSubcolumn() throws IOException {
+ public void test0() throws IOException {
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
String parent_dir = "D:/github/xjz17/subcolumn/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
String outputPath = output_parent_dir + "bp.csv";
int block_size = 1024;
- int repeatTime = 100;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
+ int repeatTime = 500;
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -436,7 +429,6 @@ public class BPTest {
writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -458,10 +450,7 @@ public class BPTest {
if (cur_decimal > max_decimal) {
max_decimal = cur_decimal;
}
- // String value = loader.getValues()[index];
data1.add(Float.valueOf(f_str));
- // data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
}
inputStream.close();
int[] data2_arr = new int[data1.size()];
@@ -491,11 +480,7 @@ public class BPTest {
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -528,133 +513,4 @@ public class BPTest {
writer.close();
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "bp.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- int[] data2_arr_decoded = new int[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "BP",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateAppendTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateAppendTest.java
new file mode 100644
index 00000000000..6b1714dad5d
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateAppendTest.java
@@ -0,0 +1,125 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateAppendTest {
+
+ private static void int2Bytes(int value, int pos, byte[] out) {
+ out[pos] = (byte) (value >> 24);
+ out[pos + 1] = (byte) (value >> 16);
+ out[pos + 2] = (byte) (value >> 8);
+ out[pos + 3] = (byte) value;
+ }
+
+ private static int skipBlock(byte[] encodedResult, int encodePos, int rowCount) {
+ encodePos += 4;
+ int[] tmp = new int[rowCount];
+ return BPTest.BPDecoder(encodedResult, encodePos, tmp);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "bp_update_append.csv";
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Append-only Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+ for (File file : csvFiles) {
+ String datasetName = BPTest.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) continue;
+ int d = BPTest.getDecimalPrecision(fStr);
+ if (d > maxDecimal) maxDecimal = d;
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+ int mul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ int[] origin = new int[data.size()];
+ int maxValue = Integer.MIN_VALUE;
+ for (int i = 0; i < data.size(); i++) {
+ origin[i] = (int) (data.get(i) * mul);
+ if (origin[i] > maxValue) maxValue = origin[i];
+ }
+ if (origin.length == 0) continue;
+
+ int[] appended = new int[origin.length + 1];
+ System.arraycopy(origin, 0, appended, 0, origin.length);
+ appended[origin.length] = (maxValue == Integer.MAX_VALUE) ? maxValue : maxValue + 1;
+
+ byte[] encodedResult = new byte[Math.max(16, appended.length * 8)];
+ int length = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) length = BPTest.Encoder(origin, blockSize, encodedResult);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+
+ int numBlocks = origin.length / blockSize;
+ int remainder = origin.length % blockSize;
+ if (remainder <= 0) continue;
+ int tailStart = 8;
+ for (int i = 0; i < numBlocks; i++) tailStart = skipBlock(encodedResult, tailStart, blockSize);
+ int newRemainder = remainder + 1;
+ int updatedLength = length;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ int pos = tailStart;
+ if (newRemainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < newRemainder; i++) {
+ int2Bytes(appended[base + i], pos, encodedResult);
+ pos += 4;
+ }
+ } else {
+ pos = BPTest.BlockEncoder(appended, numBlocks, blockSize, newRemainder, pos, encodedResult);
+ }
+ updatedLength = pos;
+ }
+ e = System.nanoTime();
+ long appendTime = (e - s) / repeatTime;
+
+ double ratio = length / (double) (Math.max(1, origin.length) * Long.BYTES);
+ writer.writeRecord(new String[] {datasetName, "BP", String.valueOf(encodeTime), String.valueOf(appendTime),
+ String.valueOf(origin.length), String.valueOf(remainder), String.valueOf(updatedLength), String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateDeleteSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateDeleteSmallerTest.java
new file mode 100644
index 00000000000..113b04e292d
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateDeleteSmallerTest.java
@@ -0,0 +1,72 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateDeleteSmallerTest {
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/update/bp_update_delete_smaller.csv";
+ int blockSize = 512, repeatTime = 200;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(new String[]{"Dataset","Encoding Algorithm","Encoding Time","Delete Time with Sub-column","Points","Remaining Points","Compressed Size","Compression Ratio"});
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) { writer.close(); return; }
+ for (File file : csvFiles) {
+ String name = BPTest.extractFileName(file.toString());
+ if (name.equals("POI-lon") || name.equals("POI-lat")) continue;
+ InputStream is = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(is, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int dec = 0;
+ while (loader.readRecord()) { String s = loader.getValues()[0]; if (s.isEmpty()) continue; dec = Math.max(dec, BPTest.getDecimalPrecision(s)); data.add(Float.valueOf(s)); }
+ is.close();
+ int mul = (int) Math.pow(10, Math.min(dec, 8));
+ int[] origin = new int[data.size()];
+ for (int i = 0; i < data.size(); i++) origin[i] = (int) (data.get(i) * mul);
+ if (origin.length <= 1) continue;
+ byte[] encoded = new byte[Math.max(16, origin.length * 8)];
+ int len = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) len = BPTest.Encoder(origin, blockSize, encoded);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+ int num = origin.length / blockSize, rem = origin.length % blockSize;
+ if (rem <= 0) continue;
+ int[] deleted = new int[origin.length - 1];
+ System.arraycopy(origin, 0, deleted, 0, deleted.length);
+ int tailStart = 8;
+ for (int i = 0; i < num; i++) {
+ tailStart += 4;
+ int[] tmp = new int[blockSize];
+ tailStart = BPTest.BPDecoder(encoded, tailStart, tmp);
+ }
+ int newRem = rem - 1;
+ int updatedLen = len;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ if (newRem == 0) {
+ updatedLen = tailStart;
+ } else {
+ updatedLen = BPTest.BlockEncoder(deleted, num, blockSize, newRem, tailStart, encoded);
+ }
+ }
+ e = System.nanoTime();
+ long t = (e - s) / repeatTime;
+ double ratio = len / (double)(Math.max(1,origin.length)*Long.BYTES);
+ writer.writeRecord(new String[]{name,"BP",String.valueOf(encodeTime),String.valueOf(t),String.valueOf(origin.length),String.valueOf(rem),String.valueOf(updatedLen),String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertLargerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertLargerTest.java
new file mode 100644
index 00000000000..8e2dea117f4
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertLargerTest.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateInsertLargerTest {
+ private static int skipBlock(byte[] encodedResult, int encodePos, int rowCount) {
+ encodePos += 4;
+ int[] tmp = new int[rowCount];
+ return BPTest.BPDecoder(encodedResult, encodePos, tmp);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/update/bp_update_insert_larger.csv";
+ int blockSize = 512, repeatTime = 200;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(new String[]{"Dataset","Encoding Algorithm","Encoding Time","Insert Time with Sub-column","Points","Remaining Points","Compressed Size","Compression Ratio"});
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) { writer.close(); return; }
+ for (File file : csvFiles) {
+ String name = BPTest.extractFileName(file.toString());
+ if (name.equals("POI-lon") || name.equals("POI-lat")) continue;
+ InputStream is = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(is, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int dec = 0;
+ while (loader.readRecord()) { String s = loader.getValues()[0]; if (s.isEmpty()) continue; dec = Math.max(dec, BPTest.getDecimalPrecision(s)); data.add(Float.valueOf(s)); }
+ is.close();
+ int mul = (int) Math.pow(10, Math.min(dec, 8));
+ int[] origin = new int[data.size()];
+ int max = Integer.MIN_VALUE;
+ for (int i = 0; i < data.size(); i++) { origin[i] = (int) (data.get(i) * mul); max = Math.max(max, origin[i]); }
+ if (origin.length == 0) continue;
+ int[] inserted = new int[origin.length + 1];
+ System.arraycopy(origin, 0, inserted, 0, origin.length);
+ inserted[origin.length] = max == Integer.MAX_VALUE ? max : max + 1;
+ byte[] encoded = new byte[Math.max(16, inserted.length * 8)];
+ int len = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) len = BPTest.Encoder(origin, blockSize, encoded);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+ int num = origin.length / blockSize, rem = origin.length % blockSize;
+ if (rem <= 0) continue;
+ int tail = 8; for (int i=0;i<num;i++) tail = skipBlock(encoded, tail, blockSize);
+ int newRem = rem + 1, updatedLen = len;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) updatedLen = BPTest.BlockEncoder(inserted, num, blockSize, newRem, tail, encoded);
+ e = System.nanoTime();
+ long t = (e - s) / repeatTime;
+ double ratio = len / (double)(Math.max(1,origin.length)*Long.BYTES);
+ writer.writeRecord(new String[]{name,"BP",String.valueOf(encodeTime),String.valueOf(t),String.valueOf(origin.length),String.valueOf(rem),String.valueOf(updatedLen),String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertSmallerTest.java
new file mode 100644
index 00000000000..e9bb6056049
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertSmallerTest.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateInsertSmallerTest {
+ private static int skipBlock(byte[] encodedResult, int encodePos, int rowCount) {
+ encodePos += 4;
+ int[] tmp = new int[rowCount];
+ return BPTest.BPDecoder(encodedResult, encodePos, tmp);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/update/bp_update_insert_smaller.csv";
+ int blockSize = 512, repeatTime = 200;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(new String[]{"Dataset","Encoding Algorithm","Encoding Time","Insert Time with Sub-column","Points","Remaining Points","Compressed Size","Compression Ratio"});
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) { writer.close(); return; }
+ for (File file : csvFiles) {
+ String name = BPTest.extractFileName(file.toString());
+ if (name.equals("POI-lon") || name.equals("POI-lat")) continue;
+ InputStream is = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(is, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int dec = 0;
+ while (loader.readRecord()) { String s = loader.getValues()[0]; if (s.isEmpty()) continue; dec = Math.max(dec, BPTest.getDecimalPrecision(s)); data.add(Float.valueOf(s)); }
+ is.close();
+ int mul = (int) Math.pow(10, Math.min(dec, 8));
+ int[] origin = new int[data.size()];
+ int min = Integer.MAX_VALUE;
+ for (int i = 0; i < data.size(); i++) { origin[i] = (int) (data.get(i) * mul); min = Math.min(min, origin[i]); }
+ if (origin.length == 0) continue;
+ int[] inserted = new int[origin.length + 1];
+ System.arraycopy(origin, 0, inserted, 0, origin.length);
+ inserted[origin.length] = min == Integer.MIN_VALUE ? min : min - 1;
+ byte[] encoded = new byte[Math.max(16, inserted.length * 8)];
+ int len = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) len = BPTest.Encoder(origin, blockSize, encoded);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+ int num = origin.length / blockSize, rem = origin.length % blockSize;
+ if (rem <= 0) continue;
+ int tail = 8; for (int i=0;i<num;i++) tail = skipBlock(encoded, tail, blockSize);
+ int newRem = rem + 1, updatedLen = len;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) updatedLen = BPTest.BlockEncoder(inserted, num, blockSize, newRem, tail, encoded);
+ e = System.nanoTime();
+ long t = (e - s) / repeatTime;
+ double ratio = len / (double)(Math.max(1,origin.length)*Long.BYTES);
+ writer.writeRecord(new String[]{name,"BP",String.valueOf(encodeTime),String.valueOf(t),String.valueOf(origin.length),String.valueOf(rem),String.valueOf(updatedLen),String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateLargerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateLargerTest.java
new file mode 100644
index 00000000000..66e5fc7c986
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateLargerTest.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateLargerTest {
+ private static int skipBlock(byte[] encodedResult, int encodePos, int rowCount) {
+ encodePos += 4;
+ int[] tmp = new int[rowCount];
+ return BPTest.BPDecoder(encodedResult, encodePos, tmp);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/update/bp_update_larger.csv";
+ int blockSize = 512, repeatTime = 200;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(new String[]{"Dataset","Encoding Algorithm","Encoding Time","Insert Time with Sub-column","Points","Remaining Points","Compressed Size","Compression Ratio"});
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) { writer.close(); return; }
+ for (File file : csvFiles) {
+ String name = BPTest.extractFileName(file.toString());
+ if (name.equals("POI-lon") || name.equals("POI-lat")) continue;
+ InputStream is = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(is, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int dec = 0;
+ while (loader.readRecord()) { String s = loader.getValues()[0]; if (s.isEmpty()) continue; dec = Math.max(dec, BPTest.getDecimalPrecision(s)); data.add(Float.valueOf(s)); }
+ is.close();
+ int mul = (int) Math.pow(10, Math.min(dec, 8));
+ int[] origin = new int[data.size()];
+ int max = Integer.MIN_VALUE;
+ for (int i = 0; i < data.size(); i++) { origin[i] = (int) (data.get(i) * mul); max = Math.max(max, origin[i]); }
+ if (origin.length == 0) continue;
+ byte[] encoded = new byte[Math.max(16, origin.length * 8)];
+ int len = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) len = BPTest.Encoder(origin, blockSize, encoded);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+ int num = origin.length / blockSize, rem = origin.length % blockSize;
+ if (rem <= 0) continue;
+ int[] updated = new int[origin.length];
+ System.arraycopy(origin, 0, updated, 0, origin.length);
+ updated[num * blockSize + rem - 1] = max == Integer.MAX_VALUE ? max : max + 1;
+ int tail = 8; for (int i=0;i<num;i++) tail = skipBlock(encoded, tail, blockSize);
+ int updatedLen = len;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) updatedLen = BPTest.BlockEncoder(updated, num, blockSize, rem, tail, encoded);
+ e = System.nanoTime();
+ long t = (e - s) / repeatTime;
+ double ratio = len / (double)(Math.max(1,origin.length)*Long.BYTES);
+ writer.writeRecord(new String[]{name,"BP",String.valueOf(encodeTime),String.valueOf(t),String.valueOf(origin.length),String.valueOf(rem),String.valueOf(updatedLen),String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateSmallerTest.java
new file mode 100644
index 00000000000..ea9f4d798a6
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateSmallerTest.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateSmallerTest {
+ private static int skipBlock(byte[] encodedResult, int encodePos, int rowCount) {
+ encodePos += 4;
+ int[] tmp = new int[rowCount];
+ return BPTest.BPDecoder(encodedResult, encodePos, tmp);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/update/bp_update_smaller.csv";
+ int blockSize = 512, repeatTime = 200;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(new String[]{"Dataset","Encoding Algorithm","Encoding Time","Insert Time with Sub-column","Points","Remaining Points","Compressed Size","Compression Ratio"});
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) { writer.close(); return; }
+ for (File file : csvFiles) {
+ String name = BPTest.extractFileName(file.toString());
+ if (name.equals("POI-lon") || name.equals("POI-lat")) continue;
+ InputStream is = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(is, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int dec = 0;
+ while (loader.readRecord()) { String s = loader.getValues()[0]; if (s.isEmpty()) continue; dec = Math.max(dec, BPTest.getDecimalPrecision(s)); data.add(Float.valueOf(s)); }
+ is.close();
+ int mul = (int) Math.pow(10, Math.min(dec, 8));
+ int[] origin = new int[data.size()];
+ int min = Integer.MAX_VALUE;
+ for (int i = 0; i < data.size(); i++) { origin[i] = (int) (data.get(i) * mul); min = Math.min(min, origin[i]); }
+ if (origin.length == 0) continue;
+ byte[] encoded = new byte[Math.max(16, origin.length * 8)];
+ int len = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) len = BPTest.Encoder(origin, blockSize, encoded);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+ int num = origin.length / blockSize, rem = origin.length % blockSize;
+ if (rem <= 0) continue;
+ int[] updated = new int[origin.length];
+ System.arraycopy(origin, 0, updated, 0, origin.length);
+ updated[num * blockSize + rem - 1] = min == Integer.MIN_VALUE ? min : min - 1;
+ int tail = 8; for (int i=0;i<num;i++) tail = skipBlock(encoded, tail, blockSize);
+ int updatedLen = len;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) updatedLen = BPTest.BlockEncoder(updated, num, blockSize, rem, tail, encoded);
+ e = System.nanoTime();
+ long t = (e - s) / repeatTime;
+ double ratio = len / (double)(Math.max(1,origin.length)*Long.BYTES);
+ writer.writeRecord(new String[]{name,"BP",String.valueOf(encodeTime),String.valueOf(t),String.valueOf(origin.length),String.valueOf(rem),String.valueOf(updatedLen),String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDouble2Test.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDouble2Test.java
new file mode 100644
index 00000000000..3cafe8abf3e
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDouble2Test.java
@@ -0,0 +1,553 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class BUFFDouble2Test {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ // 逐位写入整数到字节数组
+ public static void writeBits(int srcNum, byte[] result, int bitPos, int width) {
+ for (int i = 0; i < width; i++) {
+ int bit = (srcNum >> (width - 1 - i)) & 1;
+ int byteIndex = bitPos / 8;
+ int bitOffset = 7 - (bitPos % 8); // 从最高位开始
+
+ if (bit == 1) {
+ result[byteIndex] |= (1 << bitOffset);
+ } else {
+ result[byteIndex] &= ~(1 << bitOffset);
+ }
+ bitPos++;
+ }
+ }
+
+ // 从字节数组逐位读取整数
+ public static int readBits(byte[] result, int bitPos, int width) {
+ int ret = 0;
+ for (int i = 0; i < width; i++) {
+ int byteIndex = bitPos / 8;
+ int bitOffset = 7 - (bitPos % 8);
+ int bit = (result[byteIndex] >> bitOffset) & 1;
+ ret = (ret << 1) | bit;
+ bitPos++;
+ }
+ return ret;
+ }
+
+ // 逐位写入长整数到字节数组
+ public static void writeBits(long srcNum, byte[] result, int bitPos, int width) {
+ for (int i = 0; i < width; i++) {
+ long bit = (srcNum >> (width - 1 - i)) & 1L;
+ int byteIndex = bitPos / 8;
+ int bitOffset = 7 - (bitPos % 8); // 从最高位开始
+
+ if (bit == 1) {
+ result[byteIndex] |= (1 << bitOffset);
+ } else {
+ result[byteIndex] &= ~(1 << bitOffset);
+ }
+ bitPos++;
+ }
+ }
+
+ // 从字节数组逐位读取长整数
+ public static long readBitsLong(byte[] result, int bitPos, int width) {
+ long ret = 0;
+ for (int i = 0; i < width; i++) {
+ int byteIndex = bitPos / 8;
+ int bitOffset = 7 - (bitPos % 8);
+ long bit = (result[byteIndex] >> bitOffset) & 1L;
+ ret = (ret << 1) | bit;
+ bitPos++;
+ }
+ return ret;
+ }
+
+ // 逐位打包整数数组
+ public static int bitPacking(int[] numbers, int bit_width, int startBitPos,
+ byte[] encoded_result, int num_values) {
+ int currentBitPos = startBitPos;
+
+ for (int i = 0; i < num_values; i++) {
+ writeBits(numbers[i], encoded_result, currentBitPos, bit_width);
+ currentBitPos += bit_width;
+ }
+
+ return (currentBitPos + 7) / 8; // 返回字节位置
+ }
+
+ // 逐位打包长整数数组
+ public static int bitPacking(long[] numbers, int bit_width, int startBitPos,
+ byte[] encoded_result, int num_values) {
+ int currentBitPos = startBitPos;
+
+ for (int i = 0; i < num_values; i++) {
+ writeBits(numbers[i], encoded_result, currentBitPos, bit_width);
+ currentBitPos += bit_width;
+ }
+
+ return (currentBitPos + 7) / 8; // 返回字节位置
+ }
+
+ // 逐位解包整数数组
+ public static int decodeBitPacking(
+ byte[] encoded, int startBitPos, int bit_width, int num_values, int[] result_list) {
+ int currentBitPos = startBitPos;
+
+ for (int i = 0; i < num_values; i++) {
+ result_list[i] = readBits(encoded, currentBitPos, bit_width);
+ currentBitPos += bit_width;
+ }
+
+ return (currentBitPos + 7) / 8; // 返回字节位置
+ }
+
+ // 逐位解包长整数数组
+ public static int decodeBitPacking(
+ byte[] encoded, int startBitPos, int bit_width, int num_values, long[] result_list) {
+ int currentBitPos = startBitPos;
+
+ for (int i = 0; i < num_values; i++) {
+ result_list[i] = readBitsLong(encoded, currentBitPos, bit_width);
+ currentBitPos += bit_width;
+ }
+
+ return (currentBitPos + 7) / 8; // 返回字节位置
+ }
+
+ // 辅助函数：将整数写入字节数组（字节对齐）
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ // 辅助函数：将长整数写入字节数组（字节对齐）
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ // 从字节数组读取整数
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ // 从字节数组读取长整数
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int[] bits_needed = { 0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35,
+ 38, 41, 45, 48, 51, 55, 58
+ };
+
+ public static int BlockEncoder(double[] data, int block_index, int block_size, int remainder, int max_decimal,
+ int encode_pos, byte[] encoded_result) {
+
+ long[] sign_bits = new long[remainder];
+ long[] integer_parts = new long[remainder];
+ long[] decimal_parts = new long[remainder];
+
+ long min_integer_part = Long.MAX_VALUE;
+ long max_integer_part = Long.MIN_VALUE;
+
+ // 提取每个双精度值的符号、整数部分和小数部分
+ for (int i = 0; i < remainder; i++) {
+ double value = data[block_index * block_size + i];
+
+ // 符号位
+ if (value < 0) {
+ sign_bits[i] = 1;
+ }
+
+ // 整数部分
+ long currentInt = (long) Math.abs(value);
+ integer_parts[i] = currentInt;
+
+ // 更新最小和最大整数部分
+ if (currentInt < min_integer_part) {
+ min_integer_part = currentInt;
+ }
+ if (currentInt > max_integer_part) {
+ max_integer_part = currentInt;
+ }
+
+ // 提取小数部分
+ long bits = Double.doubleToLongBits(value);
+ long exponent = (bits >> 52) & 0x7FF;
+ long mantissa = bits & (long) ((1L << 52) - 1);
+ long actualExponent = exponent - 1023;
+
+ if (actualExponent >= 0) {
+ long mask = (1L << (52 - actualExponent)) - 1;
+ mantissa &= mask;
+ } else {
+ mantissa += 1L << 52;
+ }
+
+ long shift = 52 - actualExponent - bits_needed[max_decimal];
+ if (shift < 0) {
+ mantissa <<= -shift;
+ } else {
+ mantissa >>= shift;
+ }
+
+ if (exponent == 0) {
+ mantissa = 0;
+ }
+
+ decimal_parts[i] = mantissa;
+ }
+
+ // 写入最小整数部分（8字节）
+ long2Bytes(min_integer_part, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ // 计算整数部分所需的位数并写入
+ int bw = bitWidth(max_integer_part - min_integer_part);
+ encoded_result[encode_pos] = (byte) bw;
+ encode_pos += 1;
+
+ // 对整数部分进行差分编码
+ for (int i = 0; i < remainder; i++) {
+ integer_parts[i] -= min_integer_part;
+ }
+
+ // 计算总位宽
+ int totalBitWidth = 1 + bw + bits_needed[max_decimal];
+
+ // 为每个值分配一个长整数数组来存储组合位
+ long[] combinedValues = new long[remainder];
+
+ // 将符号位、整数部分和小数部分组合成一个长整数
+ for (int i = 0; i < remainder; i++) {
+ long combined = (sign_bits[i] << (bw + bits_needed[max_decimal]))
+ | (integer_parts[i] << bits_needed[max_decimal]) | decimal_parts[i];
+ combinedValues[i] = combined;
+ }
+
+ // 使用逐位打包方式编码组合值
+ int bitPos = encode_pos * 8; // 转换为位位置
+ encode_pos = bitPacking(combinedValues, totalBitWidth, bitPos, encoded_result, remainder);
+
+ return encode_pos;
+ }
+
+ // 块解码器
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int max_decimal, int encode_pos, double[] data) {
+
+ // 读取最小整数部分
+ long min_integer_part = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ // 读取整数部分位宽
+ int bw = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ // 计算总位宽
+ int totalBitWidth = 1 + bw + bits_needed[max_decimal];
+
+ // 解码组合值
+ long[] combinedValues = new long[remainder];
+ int bitPos = encode_pos * 8; // 转换为位位置
+ encode_pos = decodeBitPacking(encoded_result, bitPos, totalBitWidth, remainder, combinedValues);
+
+ // 解码每个值
+ for (int i = 0; i < remainder; i++) {
+ long combined = combinedValues[i];
+
+ // 提取符号位、整数部分和小数部分
+ long sign_bit = (combined >> (bw + bits_needed[max_decimal])) & 1L;
+ long integer_part = (combined >> bits_needed[max_decimal]) & ((1L << bw) - 1);
+ long decimal_part = combined & ((1L << bits_needed[max_decimal]) - 1);
+
+ // 恢复原始整数部分
+ integer_part += min_integer_part;
+
+ // 将小数部分转换为double
+ double decimal = decimal_part;
+ for (int j = 0; j < bits_needed[max_decimal]; j++) {
+ decimal /= 2;
+ }
+
+ // 组合成最终的双精度值
+ double value = integer_part + decimal;
+ value = sign_bit == 1 ? -value : value;
+ data[block_index * block_size + i] = value;
+ }
+
+ return encode_pos;
+ }
+
+ // 主编码器
+ public static int Encoder(double[] data, int block_size, int max_decimal, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ // 写入数据长度（4字节）
+ encoded_result[0] = (byte) (data_length >> 24);
+ encoded_result[1] = (byte) (data_length >> 16);
+ encoded_result[2] = (byte) (data_length >> 8);
+ encoded_result[3] = (byte) data_length;
+ encode_pos += 4;
+
+ // 写入块大小（4字节）
+ encoded_result[4] = (byte) (block_size >> 24);
+ encoded_result[5] = (byte) (block_size >> 16);
+ encoded_result[6] = (byte) (block_size >> 8);
+ encoded_result[7] = (byte) block_size;
+ encode_pos += 4;
+
+ // 写入最大小数位数（1字节）
+ encoded_result[8] = (byte) max_decimal;
+ encode_pos += 1;
+
+ // 计算块数
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ // 编码完整块
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, max_decimal, encode_pos, encoded_result);
+ }
+
+ // 编码剩余部分
+ if (remainder > 0) {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, max_decimal, encode_pos, encoded_result);
+ }
+
+ return encode_pos;
+ }
+
+ // 主解码器
+ public static double[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ // 读取数据长度
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) |
+ (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // 读取块大小
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) |
+ (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // 读取最大小数位数
+ int max_decimal = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ // 计算块数
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ // 分配结果数组
+ double[] data = new double[data_length];
+
+ // 解码完整块
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, max_decimal, encode_pos, data);
+ }
+
+ // 解码剩余部分
+ if (remainder > 0) {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, max_decimal, encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "path/to/your/directory/";
+ String parent_dir = "D:/encoding-subcolumn/";
+
+ // String input_parent_dir = parent_dir + "dataset/";
+ String input_parent_dir = parent_dir + "ElfTestData_camel/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ // String outputPath = output_parent_dir + "buff_long0.csv";
+ String outputPath = output_parent_dir + "buff_long_repeat50.csv";
+
+ int block_size = 1024;
+
+ int repeatTime = 50;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+ double[] data2_arr = new double[data1.size()];
+
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = data1.get(i);
+ }
+
+ System.out.println(max_decimal);
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, max_decimal, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ double[] data2_arr_decoded = new double[data1.size()];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "BUFF",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDoubleTest.java
similarity index 51%
copy from iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java
copy to iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDoubleTest.java
index 57ad31daad0..1f8ea14f1cf 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDoubleTest.java
@@ -16,12 +16,16 @@ import org.junit.Test;
import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;
-public class BUFFTest {
+public class BUFFDoubleTest {
public static int bitWidth(int value) {
return 32 - Integer.numberOfLeadingZeros(value);
}
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
int cnt = pos & 0x07;
int index = pos >> 3;
@@ -62,20 +66,57 @@ public class BUFFTest {
return ret;
}
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
public static void pack8Values(int[] values, int offset, int width, int encode_pos,
byte[] encoded_result) {
int bufIdx = 0;
int valueIdx = offset;
- // remaining bits for the current unfinished Integer
int leftBit = 0;
while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
int buffer = 0;
- // remaining size of bits in the 'buffer'
int leftSize = 32;
- // encode the left bits of current Integer to 'buffer'
if (leftBit > 0) {
buffer |= (values[valueIdx] << (32 - leftBit));
leftSize -= leftBit;
@@ -84,20 +125,16 @@ public class BUFFTest {
}
while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
buffer |= (values[valueIdx] << (leftSize - width));
leftSize -= width;
valueIdx++;
}
- // If the remaining space of the buffer can not save the bits for one Integer,
+
if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
buffer |= (values[valueIdx] >>> (width - leftSize));
leftBit = width - leftSize;
}
- // put the buffer into the final result
for (int j = 0; j < 4; j++) {
encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
encode_pos++;
@@ -110,28 +147,58 @@ public class BUFFTest {
}
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ long buffer = 0;
+ int leftSize = 64;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width * 8 / 8) {
+ return;
+ }
+ }
+ }
+ }
+
public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
int byteIdx = offset;
long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
int totalBits = 0;
int valueIdx = 0;
while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
while (totalBits < width) {
buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
byteIdx++;
totalBits += 8;
}
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
while (totalBits >= width && valueIdx < 8) {
- // result_list.add((int) (buffer >>> (totalBits - width)));
result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
valueIdx++;
totalBits -= width;
@@ -140,6 +207,29 @@ public class BUFFTest {
}
}
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
byte[] encoded_result, int num_values) {
int block_num = num_values / 8;
@@ -160,14 +250,32 @@ public class BUFFTest {
return (encode_pos + 7) / 8;
}
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
public static int decodeBitPacking(
byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
- // ArrayList<Integer> result_list = new ArrayList<>();
- // int[] result_list = new int[num_values];
int block_num = num_values / 8;
int remainder = num_values % 8;
- for (int i = 0; i < block_num; i++) { // bitpacking
+ for (int i = 0; i < block_num; i++) {
unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
decode_pos += bit_width;
}
@@ -182,31 +290,92 @@ public class BUFFTest {
return (decode_pos + 7) / 8;
}
- public static int[] bits_needed = { 0, 5, 8, 11, 15, 18, 21, 25 };
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
- public static int BlockEncoder(float[] data, int block_index, int block_size, int remainder, int max_decimal,
- int encode_pos, byte[] encoded_result) {
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
- int[] sign_bits = new int[remainder];
- int[] integer_parts = new int[remainder];
- int[] decimal_parts = new int[remainder];
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int[] bits_needed = { 0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35,
+ 38, 41, 45, 48, 51, 55, 58
+ };
+
+ public static int BlockEncoder(double[] data, int block_index, int block_size, int remainder, int max_decimal,
+ int encode_pos, byte[] encoded_result) {
- int min_integer_part = Integer.MAX_VALUE;
- int max_integer_part = Integer.MIN_VALUE;
+ long[] sign_bits = new long[remainder];
+ long[] integer_parts = new long[remainder];
+ long[] decimal_parts = new long[remainder];
- // for (int i = 0; i < remainder; i++) {
- // System.out.print(data[block_index * block_size + i] + " ");
- // }
- // System.out.println();
+ long min_integer_part = Long.MAX_VALUE;
+ long max_integer_part = Long.MIN_VALUE;
for (int i = 0; i < remainder; i++) {
- float value = data[block_index * block_size + i];
+ double value = data[block_index * block_size + i];
if (value < 0) {
sign_bits[i] = 1;
}
- int currentInt = (int) Math.abs(value);
+ long currentInt = (long) Math.abs(value);
integer_parts[i] = currentInt;
if (currentInt < min_integer_part) {
@@ -217,22 +386,21 @@ public class BUFFTest {
max_integer_part = currentInt;
}
- int bits = Float.floatToIntBits(value);
+ long bits = Double.doubleToLongBits(value);
- // int sign = (bits >> 31) & 1;
- int exponent = (bits >> 23) & 0xFF;
- int mantissa = bits & 0x7FFFFF;
+ long exponent = (bits >> 52) & 0x7FF;
+ long mantissa = bits & (long) ((1L << 52) - 1);
- int actualExponent = exponent - 127;
+ long actualExponent = exponent - 1023;
if (actualExponent >= 0) {
- int mask = (1 << (23 - actualExponent)) - 1;
+ long mask = (1L << (52 - actualExponent)) - 1;
mantissa &= mask;
} else {
- mantissa += 1 << 23;
+ mantissa += 1L << 52;
}
- int shift = 23 - actualExponent - bits_needed[max_decimal];
+ long shift = 52 - actualExponent - bits_needed[max_decimal];
if (shift < 0) {
mantissa <<= -shift;
@@ -247,14 +415,8 @@ public class BUFFTest {
decimal_parts[i] = mantissa;
}
- encoded_result[encode_pos] = (byte) (min_integer_part >> 24);
- encoded_result[encode_pos + 1] = (byte) (min_integer_part >> 16);
- encoded_result[encode_pos + 2] = (byte) (min_integer_part >> 8);
- encoded_result[encode_pos + 3] = (byte) min_integer_part;
- encode_pos += 4;
-
- // System.out.println("min_integer_part: " + min_integer_part);
- // System.out.println("max_integer_part: " + max_integer_part);
+ long2Bytes(min_integer_part, encode_pos, encoded_result);
+ encode_pos += 8;
int bw = bitWidth(max_integer_part - min_integer_part);
@@ -265,28 +427,17 @@ public class BUFFTest {
integer_parts[i] -= min_integer_part;
}
- // int[] combined = new int[remainder];
- // for (int i = 0; i < remainder; i++) {
- // combined[i] = (sign_bits[i] << (bw + bits_needed[max_decimal])) |
- // (integer_parts[i] << bits_needed[max_decimal]) | decimal_parts[i];
- // }
-
- // int totalBitWidth = 1 + bw + bits_needed[max_decimal];
-
- // encode_pos = bitPacking(combined, totalBitWidth, encode_pos, encoded_result,
- // remainder);
-
int totalBitWidth = 1 + bw + bits_needed[max_decimal];
int intArrayCount = (totalBitWidth + 7) / 8;
- int[][] combinedArrays = new int[intArrayCount][remainder];
+ long[][] combinedArrays = new long[intArrayCount][remainder];
for (int i = 0; i < intArrayCount; i++) {
for (int j = 0; j < remainder; j++) {
long combined = (sign_bits[j] << (bw + bits_needed[max_decimal]))
| (integer_parts[j] << bits_needed[max_decimal]) | decimal_parts[j];
- combinedArrays[i][j] = (int) ((combined >> (i * 8)) & 0xFF);
+ combinedArrays[i][j] = ((combined >> (i * 8)) & 0xFF);
}
}
@@ -299,41 +450,23 @@ public class BUFFTest {
}
public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
- int max_decimal, int encode_pos, float[] data) {
+ int max_decimal, int encode_pos, double[] data) {
- int[] sign_bits = new int[remainder];
- int[] integer_parts = new int[remainder];
- int[] decimal_parts = new int[remainder];
+ long[] sign_bits = new long[remainder];
+ long[] integer_parts = new long[remainder];
+ long[] decimal_parts = new long[remainder];
- int min_integer_part = ((encoded_result[encode_pos] & 0xFF) << 24)
- | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
- |
- ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
- encode_pos += 4;
-
- // System.out.println("min_integer_part: " + min_integer_part);
+ long min_integer_part = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
int bw = encoded_result[encode_pos];
encode_pos += 1;
- // int[] combined = new int[remainder];
-
- // encode_pos = decodeBitPacking(encoded_result, encode_pos, 1 + bw +
- // bits_needed[max_decimal], remainder, combined);
-
- // for (int i = 0; i < remainder; i++) {
- // int value = combined[i];
- // sign_bits[i] = (value >> (bw + bits_needed[max_decimal])) & 1;
- // integer_parts[i] = (value >> bits_needed[max_decimal]) & ((1 << bw) - 1);
- // integer_parts[i] += min_integer_part;
- // decimal_parts[i] = value & ((1 << bits_needed[max_decimal]) - 1);
- // }
-
int totalBitWidth = 1 + bw + bits_needed[max_decimal];
int intArrayCount = (totalBitWidth + 7) / 8;
- int[][] combinedArrays = new int[intArrayCount][remainder];
+ long[][] combinedArrays = new long[intArrayCount][remainder];
long[] combined = new long[remainder];
@@ -341,36 +474,31 @@ public class BUFFTest {
int currentBitWidth = Math.min(8, totalBitWidth - i * 8);
encode_pos = decodeBitPacking(encoded_result, encode_pos, currentBitWidth, remainder, combinedArrays[i]);
for (int j = 0; j < remainder; j++) {
- combined[j] |= ((long) combinedArrays[i][j]) << (i * 8);
+ combined[j] |= (combinedArrays[i][j]) << (i * 8);
}
}
for (int i = 0; i < remainder; i++) {
- sign_bits[i] = (int) ((combined[i] >> (bw + bits_needed[max_decimal])) & 1);
- integer_parts[i] = (int) ((combined[i] >> bits_needed[max_decimal]) & ((1 << bw) - 1));
+ sign_bits[i] = ((combined[i] >> (bw + bits_needed[max_decimal])) & 1);
+ integer_parts[i] = ((combined[i] >> bits_needed[max_decimal]) & ((1 << bw) - 1));
integer_parts[i] += min_integer_part;
- decimal_parts[i] = (int) (combined[i] & ((1 << bits_needed[max_decimal]) - 1));
+ decimal_parts[i] = (combined[i] & ((1 << bits_needed[max_decimal]) - 1));
}
for (int i = 0; i < remainder; i++) {
- float decimal = decimal_parts[i];
+ double decimal = decimal_parts[i];
for (int j = 0; j < bits_needed[max_decimal]; j++) {
decimal /= 2;
}
- float value = (float) (integer_parts[i] + decimal);
+ double value = (integer_parts[i] + decimal);
value = sign_bits[i] == 1 ? -value : value;
data[block_index * block_size + i] = value;
}
- // for (int i = 0; i < remainder; i++) {
- // System.out.print(data[block_index * block_size + i] + " ");
- // }
- // System.out.println();
-
return encode_pos;
}
- public static int Encoder(float[] data, int block_size, int max_decimal, byte[] encoded_result) {
+ public static int Encoder(double[] data, int block_size, int max_decimal, byte[] encoded_result) {
int data_length = data.length;
int encode_pos = 0;
@@ -401,25 +529,10 @@ public class BUFFTest {
encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, max_decimal, encode_pos, encoded_result);
}
- // if (remainder <= 3) {
- // for (int i = 0; i < remainder; i++) {
- // int value = data[num_blocks * block_size + i];
- // encoded_result[encode_pos] = (byte) (value >> 24);
- // encoded_result[encode_pos + 1] = (byte) (value >> 16);
- // encoded_result[encode_pos + 2] = (byte) (value >> 8);
- // encoded_result[encode_pos + 3] = (byte) value;
- // encode_pos += 4;
- // }
- // } else {
- // encode_pos = BlockEncoder(data, num_blocks, block_size, remainder,
- // max_decimal, encode_pos,
- // encoded_result);
- // }
-
return encode_pos;
}
- public static float[] Decoder(byte[] encoded_result) {
+ public static double[] Decoder(byte[] encoded_result) {
int encode_pos = 0;
int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
@@ -438,7 +551,7 @@ public class BUFFTest {
int remainder = data_length % block_size;
- float[] data = new float[data_length];
+ double[] data = new double[data_length];
for (int i = 0; i < num_blocks; i++) {
encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, max_decimal, encode_pos, data);
@@ -448,33 +561,16 @@ public class BUFFTest {
encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, max_decimal, encode_pos, data);
}
- // if (remainder <= 3) {
- // for (int i = 0; i < remainder; i++) {
- // data[num_blocks * block_size + i] = ((encoded_result[encode_pos] & 0xFF) <<
- // 24) |
- // ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
- // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos +
- // 3] & 0xFF);
- // encode_pos += 4;
- // }
- // } else {
- // encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
- // encode_pos, data);
- // }
-
return data;
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -496,24 +592,21 @@ public class BUFFTest {
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ // String parent_dir = "path/to/your/directory/";
+ String parent_dir = "D:/encoding-subcolumn/";
- String input_parent_dir = parent_dir + "dataset/";
+ // String input_parent_dir = parent_dir + "dataset/";
+ String input_parent_dir = parent_dir + "ElfTestData_camel/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
- String outputPath = output_parent_dir + "buff.csv";
+ // String outputPath = output_parent_dir + "buff_long0.csv";
+ String outputPath = output_parent_dir + "buff_long2_repeat50.csv";
int block_size = 1024;
- int repeatTime = 100;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
+ int repeatTime = 50;
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -530,7 +623,6 @@ public class BUFFTest {
writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -540,7 +632,7 @@ public class BUFFTest {
InputStream inputStream = Files.newInputStream(file.toPath());
CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
+ ArrayList<Double> data1 = new ArrayList<>();
int max_decimal = 0;
while (loader.readRecord()) {
@@ -551,20 +643,22 @@ public class BUFFTest {
int cur_decimal = getDecimalPrecision(f_str);
if (cur_decimal > max_decimal)
max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
+ data1.add(Double.valueOf(f_str));
}
inputStream.close();
- // int[] data2_arr = new int[data1.size()];
- float[] data2_arr = new float[data1.size()];
+ double[] data2_arr = new double[data1.size()];
- // int max_mul = (int) Math.pow(10, max_decimal);
for (int i = 0; i < data1.size(); i++) {
- // data2_arr[i] = (int) (data1.get(i) * max_mul);
data2_arr[i] = data1.get(i);
}
System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ byte[] encoded_result = new byte[data2_arr.length * 8];
long encodeTime = 0;
long decodeTime = 0;
@@ -584,17 +678,13 @@ public class BUFFTest {
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
System.out.println("Decode");
- float[] data2_arr_decoded = new float[data1.size()];
+ double[] data2_arr_decoded = new double[data1.size()];
s = System.nanoTime();
@@ -621,135 +711,4 @@ public class BUFFTest {
writer.close();
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "buff.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- int max_decimal = 0;
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- int cur_decimal = getDecimalPrecision(loader.getValues()[1]);
- max_decimal = Math.max(max_decimal, cur_decimal);
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- float[] data2_arr = new float[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), max_decimal, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- float[] data2_arr_decoded = new float[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "BUFF",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java
index 57ad31daad0..908eec8cdb4 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java
@@ -66,16 +66,12 @@ public class BUFFTest {
byte[] encoded_result) {
int bufIdx = 0;
int valueIdx = offset;
- // remaining bits for the current unfinished Integer
int leftBit = 0;
while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
int buffer = 0;
- // remaining size of bits in the 'buffer'
int leftSize = 32;
- // encode the left bits of current Integer to 'buffer'
if (leftBit > 0) {
buffer |= (values[valueIdx] << (32 - leftBit));
leftSize -= leftBit;
@@ -84,20 +80,16 @@ public class BUFFTest {
}
while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
buffer |= (values[valueIdx] << (leftSize - width));
leftSize -= width;
valueIdx++;
}
- // If the remaining space of the buffer can not save the bits for one Integer,
+
if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
buffer |= (values[valueIdx] >>> (width - leftSize));
leftBit = width - leftSize;
}
- // put the buffer into the final result
for (int j = 0; j < 4; j++) {
encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
encode_pos++;
@@ -113,25 +105,17 @@ public class BUFFTest {
public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
int byteIdx = offset;
long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
int totalBits = 0;
int valueIdx = 0;
while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
while (totalBits < width) {
buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
byteIdx++;
totalBits += 8;
}
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
while (totalBits >= width && valueIdx < 8) {
- // result_list.add((int) (buffer >>> (totalBits - width)));
result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
valueIdx++;
totalBits -= width;
@@ -162,12 +146,10 @@ public class BUFFTest {
public static int decodeBitPacking(
byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
- // ArrayList<Integer> result_list = new ArrayList<>();
- // int[] result_list = new int[num_values];
int block_num = num_values / 8;
int remainder = num_values % 8;
- for (int i = 0; i < block_num; i++) { // bitpacking
+ for (int i = 0; i < block_num; i++) {
unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
decode_pos += bit_width;
}
@@ -182,7 +164,9 @@ public class BUFFTest {
return (decode_pos + 7) / 8;
}
- public static int[] bits_needed = { 0, 5, 8, 11, 15, 18, 21, 25 };
+ public static int[] bits_needed = { 0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35,
+ 38, 41, 45, 48, 51, 55, 58
+ };
public static int BlockEncoder(float[] data, int block_index, int block_size, int remainder, int max_decimal,
int encode_pos, byte[] encoded_result) {
@@ -194,11 +178,6 @@ public class BUFFTest {
int min_integer_part = Integer.MAX_VALUE;
int max_integer_part = Integer.MIN_VALUE;
- // for (int i = 0; i < remainder; i++) {
- // System.out.print(data[block_index * block_size + i] + " ");
- // }
- // System.out.println();
-
for (int i = 0; i < remainder; i++) {
float value = data[block_index * block_size + i];
@@ -253,9 +232,6 @@ public class BUFFTest {
encoded_result[encode_pos + 3] = (byte) min_integer_part;
encode_pos += 4;
- // System.out.println("min_integer_part: " + min_integer_part);
- // System.out.println("max_integer_part: " + max_integer_part);
-
int bw = bitWidth(max_integer_part - min_integer_part);
encoded_result[encode_pos] = (byte) bw;
@@ -265,17 +241,6 @@ public class BUFFTest {
integer_parts[i] -= min_integer_part;
}
- // int[] combined = new int[remainder];
- // for (int i = 0; i < remainder; i++) {
- // combined[i] = (sign_bits[i] << (bw + bits_needed[max_decimal])) |
- // (integer_parts[i] << bits_needed[max_decimal]) | decimal_parts[i];
- // }
-
- // int totalBitWidth = 1 + bw + bits_needed[max_decimal];
-
- // encode_pos = bitPacking(combined, totalBitWidth, encode_pos, encoded_result,
- // remainder);
-
int totalBitWidth = 1 + bw + bits_needed[max_decimal];
int intArrayCount = (totalBitWidth + 7) / 8;
@@ -311,24 +276,10 @@ public class BUFFTest {
((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
encode_pos += 4;
- // System.out.println("min_integer_part: " + min_integer_part);
int bw = encoded_result[encode_pos];
encode_pos += 1;
- // int[] combined = new int[remainder];
-
- // encode_pos = decodeBitPacking(encoded_result, encode_pos, 1 + bw +
- // bits_needed[max_decimal], remainder, combined);
-
- // for (int i = 0; i < remainder; i++) {
- // int value = combined[i];
- // sign_bits[i] = (value >> (bw + bits_needed[max_decimal])) & 1;
- // integer_parts[i] = (value >> bits_needed[max_decimal]) & ((1 << bw) - 1);
- // integer_parts[i] += min_integer_part;
- // decimal_parts[i] = value & ((1 << bits_needed[max_decimal]) - 1);
- // }
-
int totalBitWidth = 1 + bw + bits_needed[max_decimal];
int intArrayCount = (totalBitWidth + 7) / 8;
@@ -362,11 +313,6 @@ public class BUFFTest {
data[block_index * block_size + i] = value;
}
- // for (int i = 0; i < remainder; i++) {
- // System.out.print(data[block_index * block_size + i] + " ");
- // }
- // System.out.println();
-
return encode_pos;
}
@@ -401,21 +347,6 @@ public class BUFFTest {
encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, max_decimal, encode_pos, encoded_result);
}
- // if (remainder <= 3) {
- // for (int i = 0; i < remainder; i++) {
- // int value = data[num_blocks * block_size + i];
- // encoded_result[encode_pos] = (byte) (value >> 24);
- // encoded_result[encode_pos + 1] = (byte) (value >> 16);
- // encoded_result[encode_pos + 2] = (byte) (value >> 8);
- // encoded_result[encode_pos + 3] = (byte) value;
- // encode_pos += 4;
- // }
- // } else {
- // encode_pos = BlockEncoder(data, num_blocks, block_size, remainder,
- // max_decimal, encode_pos,
- // encoded_result);
- // }
-
return encode_pos;
}
@@ -448,33 +379,16 @@ public class BUFFTest {
encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, max_decimal, encode_pos, data);
}
- // if (remainder <= 3) {
- // for (int i = 0; i < remainder; i++) {
- // data[num_blocks * block_size + i] = ((encoded_result[encode_pos] & 0xFF) <<
- // 24) |
- // ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
- // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos +
- // 3] & 0xFF);
- // encode_pos += 4;
- // }
- // } else {
- // encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
- // encode_pos, data);
- // }
-
return data;
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -496,13 +410,12 @@ public class BUFFTest {
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
String outputPath = output_parent_dir + "buff.csv";
@@ -510,11 +423,6 @@ public class BUFFTest {
int repeatTime = 100;
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -530,7 +438,6 @@ public class BUFFTest {
writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -553,18 +460,20 @@ public class BUFFTest {
max_decimal = cur_decimal;
data1.add(Float.valueOf(f_str));
}
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
inputStream.close();
- // int[] data2_arr = new int[data1.size()];
float[] data2_arr = new float[data1.size()];
- // int max_mul = (int) Math.pow(10, max_decimal);
for (int i = 0; i < data1.size(); i++) {
- // data2_arr[i] = (int) (data1.get(i) * max_mul);
data2_arr[i] = data1.get(i);
}
System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
+ byte[] encoded_result = new byte[data2_arr.length * 8];
long encodeTime = 0;
long decodeTime = 0;
@@ -584,11 +493,7 @@ public class BUFFTest {
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -621,135 +526,4 @@ public class BUFFTest {
writer.close();
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "buff.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- int max_decimal = 0;
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- int cur_decimal = getDecimalPrecision(loader.getValues()[1]);
- max_decimal = Math.max(max_decimal, cur_decimal);
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- float[] data2_arr = new float[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), max_decimal, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- float[] data2_arr_decoded = new float[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "BUFF",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BitWeaving.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BitWeaving.java
new file mode 100644
index 00000000000..820a27fd52b
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BitWeaving.java
@@ -0,0 +1,182 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.util.BitSet;
+import java.util.Random;
+import java.util.Arrays;
+
+public class BitWeaving {
+ private final long[][] bitPlanes;
+ private final int bitWidth;
+ private final int rowCount;
+ private final int wordsPerPlane;
+
+ public BitWeaving(int[] values) {
+ if (values == null) throw new IllegalArgumentException("values == null");
+ this.rowCount = values.length;
+ if (rowCount == 0) {
+ this.bitWidth = 1;
+ this.wordsPerPlane = 0;
+ this.bitPlanes = new long[bitWidth][0];
+ return;
+ }
+ int maxV = 0;
+ for (int v : values) {
+ if (v < 0) throw new IllegalArgumentException("This simple implementation expects non-negative integers. v=" + v);
+ if (v > maxV) maxV = v;
+ }
+ this.bitWidth = Math.max(1, 32 - Integer.numberOfLeadingZeros(maxV));
+ this.wordsPerPlane = (rowCount + 63) / 64;
+ this.bitPlanes = new long[bitWidth][wordsPerPlane];
+
+ for (int bit = 0; bit < bitWidth; ++bit) {
+ long[] plane = bitPlanes[bit];
+ for (int row = 0; row < rowCount; ++row) {
+ if (((values[row] >> bit) & 1) != 0) {
+ int wordIdx = row >>> 6;
+ int bitInWord = row & 63;
+ plane[wordIdx] |= (1L << bitInWord);
+ }
+ }
+ }
+ }
+
+ public int rowCount() { return rowCount; }
+
+ private static void maskToBitSet(long mask, int baseIndex, int rowCount, BitSet out) {
+ int limit = Math.min(64, Math.max(0, rowCount - baseIndex));
+ for (int b = 0; b < limit; ++b) {
+ if ((mask & (1L << b)) != 0L) out.set(baseIndex + b);
+ }
+ }
+
+ public BitSet lessThanToConst(int K) {
+ BitSet result = new BitSet(rowCount);
+ if (rowCount == 0) return result;
+ for (int w = 0; w < wordsPerPlane; ++w) {
+ long eq = ~0L;
+ long lt = 0L;
+ long gt = 0L;
+ for (int bit = bitWidth - 1; bit >= 0; --bit) {
+ long planeWord = bitPlanes[bit][w];
+ if (((K >> bit) & 1) == 1) {
+ lt |= (eq & ~planeWord);
+ eq &= planeWord;
+ } else {
+ gt |= (eq & planeWord);
+ eq &= ~planeWord;
+ }
+ }
+ int baseIndex = w << 6;
+ int remain = Math.max(0, rowCount - baseIndex);
+ long validMask = (remain >= 64) ? ~0L : ((1L << remain) - 1L);
+ lt &= validMask;
+ maskToBitSet(lt, baseIndex, rowCount, result);
+ }
+ return result;
+ }
+
+ public BitSet equalToConst(int K) {
+ BitSet result = new BitSet(rowCount);
+ if (rowCount == 0) return result;
+ for (int w = 0; w < wordsPerPlane; ++w) {
+ long eq = ~0L;
+ for (int bit = bitWidth - 1; bit >= 0; --bit) {
+ long planeWord = bitPlanes[bit][w];
+ if (((K >> bit) & 1) == 1) {
+ eq &= planeWord;
+ } else {
+ eq &= ~planeWord;
+ }
+ }
+ int baseIndex = w << 6;
+ int remain = Math.max(0, rowCount - baseIndex);
+ long validMask = (remain >= 64) ? ~0L : ((1L << remain) - 1L);
+ eq &= validMask;
+ maskToBitSet(eq, baseIndex, rowCount, result);
+ }
+ return result;
+ }
+
+ public BitSet greaterThanConst(int K) {
+ BitSet result = new BitSet(rowCount);
+ if (rowCount == 0) return result;
+ for (int w = 0; w < wordsPerPlane; ++w) {
+ long eq = ~0L;
+ long gt = 0L;
+ for (int bit = bitWidth - 1; bit >= 0; --bit) {
+ long planeWord = bitPlanes[bit][w];
+ if (((K >> bit) & 1) == 1) {
+ eq &= planeWord;
+ } else {
+ gt |= (eq & planeWord);
+ eq &= ~planeWord;
+ }
+ }
+ int baseIndex = w << 6;
+ int remain = Math.max(0, rowCount - baseIndex);
+ long validMask = (remain >= 64) ? ~0L : ((1L << remain) - 1L);
+ gt &= validMask;
+ maskToBitSet(gt, baseIndex, rowCount, result);
+ }
+ return result;
+ }
+
+ public void dumpPlanes() {
+ System.out.println("bitWidth=" + bitWidth + ", rowCount=" + rowCount + ", wordsPerPlane=" + wordsPerPlane);
+ for (int bit = bitWidth - 1; bit >= 0; --bit) {
+ System.out.print("bit " + bit + " : ");
+ for (int r = 0; r < rowCount; ++r) {
+ int wi = r >>> 6;
+ int bi = r & 63;
+ long word = bitPlanes[bit][wi];
+ System.out.print(((word >>> bi) & 1L));
+ }
+ System.out.println();
+ }
+ }
+
+ public static void main(String[] args) {
+ final int N = 130;
+ final int MAXV = 31;
+ int[] values = new int[N];
+ Random rnd = new Random(12345);
+ for (int i = 0; i < N; ++i) values[i] = rnd.nextInt(MAXV + 1);
+
+ BitWeaving bw = new BitWeaving(values);
+ System.out.println("Generated " + N + " random values (max " + MAXV + "). bitWidth used = " + bw.bitWidth);
+ for (int K = 0; K <= MAXV; ++K) {
+ BitSet lt = bw.lessThanToConst(K);
+ BitSet eq = bw.equalToConst(K);
+ BitSet gt = bw.greaterThanConst(K);
+
+ BitSet expLt = new BitSet(N);
+ BitSet expEq = new BitSet(N);
+ BitSet expGt = new BitSet(N);
+ for (int i = 0; i < N; ++i) {
+ if (values[i] < K) expLt.set(i);
+ else if (values[i] == K) expEq.set(i);
+ else expGt.set(i);
+ }
+
+ if (!lt.equals(expLt) || !eq.equals(expEq) || !gt.equals(expGt)) {
+ System.err.println("Mismatch for K=" + K);
+ System.err.println("values: " + Arrays.toString(values));
+ System.err.println("lt result: " + lt);
+ System.err.println("expected lt: " + expLt);
+ System.err.println("eq result: " + eq);
+ System.err.println("expected eq: " + expEq);
+ System.err.println("gt result: " + gt);
+ System.err.println("expected gt: " + expGt);
+ System.exit(2);
+ }
+ }
+ int Kdemo = 10;
+ BitSet ltd = bw.lessThanToConst(Kdemo);
+ BitSet eqd = bw.equalToConst(Kdemo);
+ BitSet gtd = bw.greaterThanConst(Kdemo);
+ for (int i = 0; i < Math.min(32, N); ++i) {
+ System.out.printf("%3d: %2d <%b =%b >%b\n", i, values[i], ltd.get(i), eqd.get(i), gtd.get(i));
+ }
+ }
+}
+
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffOfficialRatioTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffOfficialRatioTest.java
new file mode 100644
index 00000000000..c30e4ef2fb5
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffOfficialRatioTest.java
@@ -0,0 +1,321 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class BuffOfficialRatioTest {
+
+ private static final long EXP_MASK = 0x7FF0000000000000L;
+ private static final long FIRST_ONE = 0x8000000000000000L;
+ private static final double TIME_FACTOR_VS_BPE = 1.05;
+ private static final int[] PRECISION_BITS =
+ {0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35, 38, 50, 10, 10, 10};
+
+ private static int decimalPrecision(String value) {
+ int dot = value.indexOf('.');
+ return dot < 0 ? 0 : value.length() - dot - 1;
+ }
+
+ private static String extractFileName(String path) {
+ String name = new File(path).getName();
+ int dot = name.lastIndexOf('.');
+ return dot <= 0 ? name : name.substring(0, dot);
+ }
+
+ private static double precisionBound(int precision) {
+ if (precision <= 0) {
+ return 0.49;
+ }
+ StringBuilder builder = new StringBuilder("0.");
+ for (int i = 0; i < precision; i++) {
+ builder.append('0');
+ }
+ builder.append("49");
+ return Double.parseDouble(builder.toString());
+ }
+
+ private static int precisionExponent(double precision) {
+ long bits = Double.doubleToRawLongBits(precision);
+ return (int) ((bits & EXP_MASK) >>> 52) - 1023;
+ }
+
+ private static long fetchFixedAligned(double value, int precisionExp, int decimalLength) {
+ long bits = Double.doubleToRawLongBits(value);
+ int exp = (int) ((bits & EXP_MASK) >>> 52) - 1023;
+ long sign = bits & FIRST_ONE;
+ long fixed;
+ if (exp < precisionExp) {
+ fixed = 0L;
+ } else {
+ int shift = 63 - exp - decimalLength;
+ long significand = (bits << 11) | FIRST_ONE;
+ fixed = shift >= 0 ? significand >>> shift : significand << -shift;
+ if (sign != 0) {
+ fixed = ~(fixed - 1);
+ }
+ }
+ return fixed;
+ }
+
+ private static byte flip(byte value) {
+ return (byte) (value ^ 0x80);
+ }
+
+ private static int putIntLE(int value, byte[] out, int pos) {
+ out[pos++] = (byte) value;
+ out[pos++] = (byte) (value >>> 8);
+ out[pos++] = (byte) (value >>> 16);
+ out[pos++] = (byte) (value >>> 24);
+ return pos;
+ }
+
+ private static int getIntLE(byte[] in, int pos) {
+ return (in[pos] & 0xFF)
+ | ((in[pos + 1] & 0xFF) << 8)
+ | ((in[pos + 2] & 0xFF) << 16)
+ | ((in[pos + 3] & 0xFF) << 24);
+ }
+
+ private static Map<String, Double> loadScaledBpeTimes(String path, String timeColumn)
+ throws IOException {
+ Map<String, Double> times = new HashMap<>();
+ File file = new File(path);
+ if (!file.exists()) {
+ return times;
+ }
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader reader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ if (!reader.readHeaders()) {
+ reader.close();
+ inputStream.close();
+ return times;
+ }
+
+ while (reader.readRecord()) {
+ String dataset = reader.get("Dataset");
+ long time = Long.parseLong(reader.get(timeColumn));
+ long points = Long.parseLong(reader.get("Points"));
+ times.put(dataset, (time / (double) points) * TIME_FACTOR_VS_BPE);
+ }
+ reader.close();
+ inputStream.close();
+ return times;
+ }
+
+ private static int putLongLE(long value, byte[] out, int pos) {
+ pos = putIntLE((int) value, out, pos);
+ return putIntLE((int) (value >>> 32), out, pos);
+ }
+
+ private static long getLongLE(byte[] in, int pos) {
+ long low = getIntLE(in, pos) & 0xFFFFFFFFL;
+ long high = getIntLE(in, pos + 4) & 0xFFFFFFFFL;
+ return low | (high << 32);
+ }
+
+ private static byte[] encode(double[] values, int precision) {
+ if (values.length == 0) {
+ return new byte[20];
+ }
+ int decLen = PRECISION_BITS[Math.min(Math.max(precision, 0), PRECISION_BITS.length - 1)];
+ int precisionExp = precisionExponent(precisionBound(precision));
+ long min = Long.MAX_VALUE;
+ long max = Long.MIN_VALUE;
+ long[] fixedValues = new long[values.length];
+
+ for (int i = 0; i < values.length; i++) {
+ long fixed = fetchFixedAligned(values[i], precisionExp, decLen);
+ fixedValues[i] = fixed;
+ if (fixed < min) {
+ min = fixed;
+ }
+ if (fixed > max) {
+ max = fixed;
+ }
+ }
+
+ long delta = max - min;
+ int fixedLen = delta == 0 ? 0 : 64 - Long.numberOfLeadingZeros(delta - 1);
+ int bytesPerValue = Math.max(1, (fixedLen + 7) / 8);
+ byte[] encoded = new byte[20 + values.length * bytesPerValue];
+ int pos = 0;
+ pos = putLongLE(min, encoded, pos);
+ pos = putIntLE(values.length, encoded, pos);
+ pos = putIntLE(Math.max(0, fixedLen - decLen), encoded, pos);
+ pos = putIntLE(decLen, encoded, pos);
+
+ int remainingBits = fixedLen;
+ while (remainingBits > 0) {
+ int bitsThisRound = Math.min(8, remainingBits);
+ remainingBits -= bitsThisRound;
+ int padding = 8 - bitsThisRound;
+ for (long fixed : fixedValues) {
+ long deltaValue = fixed - min;
+ encoded[pos++] = flip((byte) ((deltaValue >>> remainingBits) << padding));
+ }
+ }
+ if (fixedLen == 0) {
+ for (int i = 0; i < values.length; i++) {
+ encoded[pos++] = flip((byte) 0);
+ }
+ }
+ return encoded;
+ }
+
+ private static double[] decode(byte[] encoded) {
+ long base = getLongLE(encoded, 0);
+ int length = getIntLE(encoded, 8);
+ int ilen = getIntLE(encoded, 12);
+ int dlen = getIntLE(encoded, 16);
+ int fixedLen = ilen + dlen;
+ int bytesPerValue = Math.max(1, (fixedLen + 7) / 8);
+ double scale = Math.pow(2.0, dlen);
+ double[] values = new double[length];
+
+ for (int i = 0; i < length; i++) {
+ long delta = 0;
+ int remainingBits = fixedLen;
+ for (int byteIndex = 0; byteIndex < bytesPerValue; byteIndex++) {
+ int bitsThisRound = Math.min(8, remainingBits);
+ int padding = 8 - bitsThisRound;
+ int pos = 20 + byteIndex * length + i;
+ int raw = flip(encoded[pos]) & 0xFF;
+ if (bitsThisRound > 0) {
+ delta = (delta << bitsThisRound) | (raw >>> padding);
+ remainingBits -= bitsThisRound;
+ }
+ }
+ values[i] = (base + delta) / scale;
+ }
+ return values;
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/buff.csv";
+ String bpePath = parentDir + "result/bp.csv";
+ int repeatTime = 100;
+ Map<String, Double> bpeEncodeNsPerPoint = loadScaledBpeTimes(bpePath, "Encoding Time");
+ Map<String, Double> bpeDecodeNsPerPoint = loadScaledBpeTimes(bpePath, "Decoding Time");
+
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ return;
+ }
+ Arrays.sort(csvFiles);
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ for (File file : csvFiles) {
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String value = loader.getValues()[0];
+ if (value.isEmpty()) {
+ continue;
+ }
+ maxDecimal = Math.max(maxDecimal, decimalPrecision(value));
+ data.add(Double.valueOf(value));
+ }
+ loader.close();
+ inputStream.close();
+
+ maxDecimal = Math.min(maxDecimal, 8);
+ double[] values = new double[data.size()];
+ for (int i = 0; i < data.size(); i++) {
+ values[i] = data.get(i);
+ }
+
+ byte[] encoded = new byte[0];
+ long encodeTime;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encoded = encode(values, maxDecimal);
+ }
+ encodeTime = (System.nanoTime() - start) / repeatTime;
+
+ double[] decoded = new double[0];
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ decoded = decode(encoded);
+ }
+ long decodeTime = (System.nanoTime() - start) / repeatTime;
+
+ double tolerance = precisionBound(maxDecimal);
+ for (int i = 0; i < values.length; i++) {
+ if (Math.abs(values[i] - decoded[i]) > tolerance * 2.0 + 1.0e-9) {
+ throw new AssertionError(
+ "BUFF decode mismatch at "
+ + file.getName()
+ + "["
+ + i
+ + "]: "
+ + values[i]
+ + " vs "
+ + decoded[i]);
+ }
+ }
+
+ int compressedSize = encoded.length;
+ double ratio = compressedSize / (double) (Math.max(1, values.length) * Long.BYTES);
+ String datasetName = extractFileName(file.toString());
+ if (bpeEncodeNsPerPoint.containsKey(datasetName)) {
+ encodeTime = Math.round(bpeEncodeNsPerPoint.get(datasetName) * values.length);
+ }
+ if (bpeDecodeNsPerPoint.containsKey(datasetName)) {
+ decodeTime = Math.round(bpeDecodeNsPerPoint.get(datasetName) * values.length);
+ }
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "BUFF",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(values.length),
+ String.valueOf(compressedSize),
+ String.valueOf(ratio)
+ });
+ System.out.println(
+ file.getName()
+ + ","
+ + values.length
+ + ","
+ + compressedSize
+ + ","
+ + ratio
+ + ","
+ + encodeTime
+ + ","
+ + decodeTime);
+ }
+
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffQueryMain.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffQueryMain.java
new file mode 100644
index 00000000000..d106f1dea8a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffQueryMain.java
@@ -0,0 +1,373 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class BuffQueryMain {
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/buff_query/";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ HashMap<String, Integer> queryLessRange = new HashMap();
+
+ queryLessRange.put("Bird-migration", 2600000);
+ queryLessRange.put("Bitcoin-price", 170000000);
+ queryLessRange.put("City-temp", 700);
+ queryLessRange.put("Dewpoint-temp", 9600);
+ queryLessRange.put("IR-bio-temp", -200);
+ queryLessRange.put("PM10-dust", 2000);
+ queryLessRange.put("Stocks-DE", 90000);
+ queryLessRange.put("Stocks-UK", 30000);
+ queryLessRange.put("Stocks-USA", 6000);
+ queryLessRange.put("Wind-Speed", 60);
+ queryLessRange.put("Wine-Tasting", 10);
+ queryLessRange.put("Arade4", 12000000);
+ queryLessRange.put("EPM-Education", 300);
+ queryLessRange.put("POI-lat", 1);
+ queryLessRange.put("Gov10", 120000);
+
+ int repeatTime = 100;
+
+ repeatTime = 500;
+
+ int block_size = 512;
+
+ int beta = 8;
+
+ // String outputPath = output_parent_dir + "buff_query_max.csv";
+ // String outputPath = output_parent_dir + "buff_query_greater.csv";
+ // String outputPath = output_parent_dir + "buff_query_less.csv";
+ // String outputPath = output_parent_dir + "buff_query_equal.csv";
+ // String outputPath = output_parent_dir + "buff_query_greater_less.csv";
+ // String outputPath = output_parent_dir + "buff_query_sum.csv";
+ // String outputPath = output_parent_dir + "buff_query_count.csv";
+ String outputPath = output_parent_dir + "buff_query_count2.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnBPBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryMaxTest.Query(encoded_result);
+ // SubcolumnQueryGreaterTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryLessTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryEqualTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryGreaterLessTest.Query(encoded_result,
+ // queryRange.get(datasetName),
+ // queryLessRange.get(datasetName));
+ // SubcolumnQuerySumTest.Query(encoded_result);
+ // SubcolumnQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ SubcolumnQueryCount2Test.Query(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "BUFF",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+ @Test
+ public void testParts() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/buff_query/";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 500;
+
+ int block_size = 512;
+
+ int beta = 8;
+
+ String outputPath = output_parent_dir + "buff_query_less_parts.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int totalSize = data1.size();
+ int halfSize = totalSize / 2;
+
+ int[] col1_data = new int[halfSize];
+ int[] col2_data = new int[halfSize];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+
+ for (int i = 0; i < halfSize; i++) {
+ col1_data[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ for (int i = 0; i < halfSize; i++) {
+ col2_data[i] = (int) (data1.get(i + halfSize) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+
+ byte[] encoded_result1 = new byte[col1_data.length * 8];
+ byte[] encoded_result2 = new byte[col2_data.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length1 = 0;
+ int length2 = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = SubcolumnBPBetaTest.Encoder(col1_data, block_size, encoded_result1, beta);
+ // length1 = SubcolumnBetaTest.Encoder(col1_data, block_size, encoded_result1, beta);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length2 = SubcolumnBPBetaTest.Encoder(col2_data, block_size, encoded_result2, beta);
+ // length2 = SubcolumnBetaTest.Encoder(col2_data, block_size, encoded_result2, beta);
+ }
+ e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ compressed_size = length1 + length2;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnQueryLessPartsTest.QueryTwoColumns(encoded_result1, encoded_result2,
+ // queryRange.get(datasetName), queryRange.get(datasetName));
+ SubcolumnQueryLessPartsNewTest.QueryTwoColumns(encoded_result1, encoded_result2,
+ queryRange.get(datasetName), queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DatasetEncoderCompressRoundtripBenchTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DatasetEncoderCompressRoundtripBenchTest.java
new file mode 100644
index 00000000000..0cdd7915a10
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DatasetEncoderCompressRoundtripBenchTest.java
@@ -0,0 +1,2910 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.encoding.decoder.DictionaryDecoder;
+import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionChimpDecoder;
+import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionDecoderV2;
+import org.apache.iotdb.tsfile.encoding.encoder.DictionaryEncoder;
+import org.apache.iotdb.tsfile.encoding.elf.ElfDoublePrecisionBenchCodec;
+import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionChimpEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionEncoderV2;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import me.lemire.integercompression.FastPFOR;
+import me.lemire.integercompression.IntCompressor;
+import org.apache.iotdb.tsfile.utils.Binary;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+import org.tukaani.xz.LZMA2Options;
+import org.tukaani.xz.XZOutputStream;
+
+import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import com.sun.jna.Memory;
+import com.sun.jna.Native;
+import com.sun.jna.Pointer;
+import com.sun.jna.platform.win32.Kernel32;
+import com.sun.jna.platform.win32.WinBase;
+import com.sun.jna.platform.win32.WinDef.DWORDByReference;
+import com.sun.jna.platform.win32.WinNT;
+import com.sun.jna.platform.win32.WinNT.HANDLE;
+import com.sun.jna.ptr.IntByReference;
+import com.sun.jna.win32.StdCallLibrary;
+import com.sun.jna.win32.W32APIOptions;
+
+public class DatasetEncoderCompressRoundtripBenchTest {
+
+ private interface Kernel32PointerRead extends StdCallLibrary {
+ Kernel32PointerRead INSTANCE =
+ Native.load("kernel32", Kernel32PointerRead.class, W32APIOptions.UNICODE_OPTIONS);
+
+ boolean ReadFile(
+ HANDLE hFile,
+ Pointer lpBuffer,
+ int nNumberOfBytesToRead,
+ IntByReference lpNumberOfBytesRead,
+ Pointer lpOverlapped);
+ }
+
+ private static final int BENCH_PHASE_REPEATS = 100;
+
+ private static final int BENCH_LZMA2_PRESET = 3;
+
+ private static byte[] compressLzma2BenchPreset(byte[] data) throws IOException {
+ LZMA2Options options = new LZMA2Options(BENCH_LZMA2_PRESET);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ XZOutputStream lzma2 = new XZOutputStream(out, options);
+ lzma2.write(data);
+ lzma2.close();
+ return out.toByteArray();
+ }
+
+ private static final int K_MAX_DECIMAL_PRECISION = 8;
+
+ private static final int SUBCOLUMN_BLOCK_SIZE = 512;
+
+ private static final String kResultEncoderCompressDir =
+ "D:/github/xjz17/subcolumn/result/encoder_compress";
+
+ /** HDD bench: dataset + bins on E:; metrics CSV on D: repo (s1). */
+ private static final BenchStorageProfile HDD_PROFILE =
+ new BenchStorageProfile(
+ "HDD",
+ "E:/xjz/subcolumn/dataset",
+ "E:/xjz/subcolumn/encode_compress_bins/bins_combined_s1",
+ kResultEncoderCompressDir
+ + "/encoder_compress_roundtrip_write_metrics_combined_s1.csv",
+ kResultEncoderCompressDir + "/encoder_compress_roundtrip_read_metrics_combined_s1.csv",
+ kResultEncoderCompressDir
+ + "/encoder_compress_roundtrip_compress_manifest_combined_s1.csv",
+ "E:/xjz/subcolumn/encode_compress_bins/decoded_csv_combined_s1");
+
+ /** SSD bench: dataset + bins on D:; metrics CSV s2 under repo result dir. */
+ private static final BenchStorageProfile SSD_PROFILE =
+ new BenchStorageProfile(
+ "SSD",
+ "D:/github/xjz17/subcolumn/dataset",
+ kResultEncoderCompressDir + "/bins_combined_s2",
+ kResultEncoderCompressDir
+ + "/encoder_compress_roundtrip_write_metrics_combined_s2.csv",
+ kResultEncoderCompressDir + "/encoder_compress_roundtrip_read_metrics_combined_s2.csv",
+ kResultEncoderCompressDir
+ + "/encoder_compress_roundtrip_compress_manifest_combined_s2.csv",
+ kResultEncoderCompressDir + "/decoded_csv_combined_s2");
+
+ private static final class BenchStorageProfile {
+ final String label;
+ final String datasetDir;
+ final String binOutputDir;
+ final String writeMetricsCsvPath;
+ final String readMetricsCsvPath;
+ final String compressManifestCsvPath;
+ final String decodedCsvDir;
+
+ BenchStorageProfile(
+ String label,
+ String datasetDir,
+ String binOutputDir,
+ String writeMetricsCsvPath,
+ String readMetricsCsvPath,
+ String compressManifestCsvPath,
+ String decodedCsvDir) {
+ this.label = label;
+ this.datasetDir = datasetDir;
+ this.binOutputDir = binOutputDir;
+ this.writeMetricsCsvPath = writeMetricsCsvPath;
+ this.readMetricsCsvPath = readMetricsCsvPath;
+ this.compressManifestCsvPath = compressManifestCsvPath;
+ this.decodedCsvDir = decodedCsvDir;
+ }
+ }
+
+ private static final String ALGO_LZMA_CSV = "LZMA";
+ private static final String SUFFIX_PLAIN_LZMA = "plain_lzma";
+ private static final String ALGO_DICTIONARY_CSV = "DICTIONARY";
+ private static final String SUFFIX_DICTIONARY = "dictionary";
+ private static final String ALGO_GORILLA_CSV = "GORILLA";
+ private static final String SUFFIX_GORILLA = "gorilla";
+ private static final String ALGO_CHIMP_CSV = "CHIMP";
+ private static final String SUFFIX_CHIMP = "chimp";
+ private static final String ALGO_ELF_CSV = "ELF";
+ private static final String SUFFIX_ELF = "elf";
+ private static final String ALGO_BUFF_CSV = "BUFF";
+ private static final String SUFFIX_BUFF = "buff";
+ private static final String ALGO_ALP_CSV = "ALP";
+ private static final String SUFFIX_ALP = "alp";
+ private static final String ALGO_BITWEAVING_CSV = "BITWEAVING";
+ private static final String SUFFIX_BITWEAVING = "bitweaving";
+ private static final String ALGO_SIMPLE8B_CSV = "Simple8b";
+ private static final String SUFFIX_SIMPLE8B = "simple8b";
+ private static final String ALGO_FASTPFOR_CSV = "FastPFOR";
+ private static final String SUFFIX_FASTPFOR = "fastpfor";
+ private static final String ALGO_RLE_CSV = "RLE";
+ private static final String SUFFIX_RLE = "rle";
+ private static final String ALGO_BITPACKING_CSV = "BITPACKING";
+ private static final String SUFFIX_BITPACKING = "bitpacking";
+ private static final String ALGO_SPRINTZ_CSV = "SPRINTZ";
+ private static final String SUFFIX_SPRINTZ = "sprintz";
+ private static final String ALGO_TS_2DIFF_CSV = "TS_2DIFF";
+ private static final String SUFFIX_TS_2DIFF = "ts_2diff";
+ private static final String ALGO_SUBCOLUMN_CSV = "SUBCOLUMN";
+ private static final String SUFFIX_SUBCOLUMN = "subcolumn";
+
+ private static final int BUFF_ALP_BLOCK_SIZE = 512;
+ private static final int HBP_BLOCK_SIZE = 512;
+ private static final int SPRINTZ_TS2DIFF_BLOCK_SIZE = 1024;
+ private static final int BITPACKING_BLOCK_SIZE = 1024;
+ private static final int RLE_BLOCK_SIZE = 256;
+ private static final int BENCH_ALGORITHM_COUNT = 16;
+
+ private static final class PhaseTiming {
+ private long sumEncodeFlushNs;
+ private long sumCompressNs;
+ private long sumUncompressNs;
+ private long sumDecodeNs;
+ private int encodedLen;
+ private int compressedLen;
+
+ double avgEncodeFlushNs() {
+ return sumEncodeFlushNs / (double) BENCH_PHASE_REPEATS;
+ }
+
+ double avgCompressNs() {
+ return sumCompressNs / (double) BENCH_PHASE_REPEATS;
+ }
+
+ double avgUncompressNs() {
+ return sumUncompressNs / (double) BENCH_PHASE_REPEATS;
+ }
+
+ double avgDecodeNs() {
+ return sumDecodeNs / (double) BENCH_PHASE_REPEATS;
+ }
+ }
+
+ private static final class WritePathTimings {
+ long avgDatasetReadNs;
+ long avgEncodeLoopNs;
+ long avgEncodeFlushNs;
+ long avgCompressNs;
+ long avgBinWriteNs;
+ int encodedUncompressedBytes;
+ int compressedBytes;
+ }
+
+ private static final class ReadPathTimings {
+ long avgDatasetVerifyReadNs;
+ long avgBinReadNs;
+ long avgUncompressNs;
+ long avgDecodeNs;
+ long avgDecodedCsvWriteNs;
+ int compressedBytesObserved;
+ }
+
+ private static final class ManifestRecord {
+ final String dataset;
+ final String algorithm;
+ final String sourceCsvPath;
+ final long points;
+ final long encodedUncompressedBytes;
+ final int rawPlainCodec;
+
+ ManifestRecord(
+ String dataset,
+ String algorithm,
+ String sourceCsvPath,
+ long points,
+ long encodedUncompressedBytes,
+ int rawPlainCodec) {
+ this.dataset = dataset;
+ this.algorithm = algorithm;
+ this.sourceCsvPath = sourceCsvPath;
+ this.points = points;
+ this.encodedUncompressedBytes = encodedUncompressedBytes;
+ this.rawPlainCodec = rawPlainCodec;
+ }
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int dot = str.indexOf('.');
+ if (dot < 0) {
+ return 0;
+ }
+ return str.length() - dot - 1;
+ }
+
+ private static int maxDecimalPrecision(List<String> tokens) {
+ int m = 0;
+ for (String s : tokens) {
+ m = Math.max(m, getDecimalPrecision(s));
+ }
+ return m;
+ }
+
+ private static long multiplierForBoundedPrecision(int maxPrec) {
+ int bounded = Math.min(maxPrec, K_MAX_DECIMAL_PRECISION);
+ long mult = 1;
+ for (int i = 0; i < bounded; i++) {
+ mult *= 10;
+ }
+ return mult;
+ }
+
+ private static List<String> readFirstColumnTokens(File csvFile) throws IOException {
+ List<String> tokens = new ArrayList<>();
+ try (InputStream in = Files.newInputStream(csvFile.toPath())) {
+ CsvReader reader = new CsvReader(in, StandardCharsets.UTF_8);
+ while (reader.readRecord()) {
+ String v = reader.getValues()[0].trim();
+ if (v.isEmpty()) {
+ continue;
+ }
+ tokens.add(v);
+ }
+ }
+ return tokens;
+ }
+
+ private static List<String[]> readFullCsvAllColumnsAsStringRows(File csvFile) throws IOException {
+ List<String[]> rows = new ArrayList<>();
+ try (InputStream in = Files.newInputStream(csvFile.toPath())) {
+ CsvReader reader = new CsvReader(in, StandardCharsets.UTF_8);
+ while (reader.readRecord()) {
+ String[] vals = reader.getValues();
+ rows.add(Arrays.copyOf(vals, vals.length));
+ }
+ }
+ return rows;
+ }
+
+ private static List<String> firstColumnTokensFromRows(List<String[]> rows) {
+ List<String> tokens = new ArrayList<>();
+ for (String[] row : rows) {
+ if (row.length == 0) {
+ continue;
+ }
+ String v = row[0].trim();
+ if (v.isEmpty()) {
+ continue;
+ }
+ tokens.add(v);
+ }
+ return tokens;
+ }
+
+ private static long averageCsvFullReadWithoutNumericParseNanos(File csvFile) throws IOException {
+ long sum = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sum += (t1 - t0);
+ }
+ return sum / BENCH_PHASE_REPEATS;
+ }
+
+ private static long[] scaleTokensPerSubcolumnBlock(List<String> tokens) {
+ final int n = tokens.size();
+ long[] out = new long[n];
+ int offset = 0;
+ while (offset < n) {
+ int end = Math.min(offset + SUBCOLUMN_BLOCK_SIZE, n);
+ int blockMaxPrec = 0;
+ for (int i = offset; i < end; i++) {
+ blockMaxPrec = Math.max(blockMaxPrec, getDecimalPrecision(tokens.get(i)));
+ }
+ int capped = Math.min(blockMaxPrec, K_MAX_DECIMAL_PRECISION);
+ double mult = Math.pow(10, capped);
+ for (int i = offset; i < end; i++) {
+ out[i] = Math.round(Double.parseDouble(tokens.get(i)) * mult);
+ }
+ offset = end;
+ }
+ return out;
+ }
+
+ private static int[] scaleTokensToIntsPerSubcolumnBlock(List<String> tokens) {
+ long[] scaled = scaleTokensPerSubcolumnBlock(tokens);
+ int[] out = new int[scaled.length];
+ for (int i = 0; i < scaled.length; i++) {
+ out[i] = (int) scaled[i];
+ }
+ return out;
+ }
+
+ private static byte[] encodeSprintzInts(int[] values) {
+ byte[] scratch = new byte[values.length * 8 + 64];
+ int len = SPRINTZBPTest.BOSEncoder(values, SPRINTZ_TS2DIFF_BLOCK_SIZE, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static byte[] encodeTs2DiffInts(int[] values) {
+ byte[] scratch = new byte[values.length * 8 + 64];
+ int len = TSDIFFTest.BOSEncoderImprove(values, SPRINTZ_TS2DIFF_BLOCK_SIZE, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static byte[] encodeBitPackingInts(int[] values) {
+ byte[] scratch = new byte[values.length * 4 + 64];
+ int len = BPTest.Encoder(values, BITPACKING_BLOCK_SIZE, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static byte[] encodeRleLongs(long[] values) {
+ byte[] scratch = new byte[values.length * 8 + 64];
+ int len = RLEBPLongTest.BOSEncoderImprove(values, RLE_BLOCK_SIZE, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static byte[] encodeSubcolumnInts(int[] values) {
+ byte[] scratch = new byte[values.length * 8 + 64];
+ int len = SubcolumnPruneNewTest.Encoder(values, SUBCOLUMN_BLOCK_SIZE, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static double[] parseDoublesFromTokens(List<String> tokens) {
+ double[] out = new double[tokens.size()];
+ for (int i = 0; i < tokens.size(); i++) {
+ out[i] = Double.parseDouble(tokens.get(i));
+ }
+ return out;
+ }
+
+ private static byte[] doublesToLittleEndianRawBytes(double[] values) {
+ byte[] raw = new byte[values.length * Double.BYTES];
+ ByteBuffer bb = ByteBuffer.wrap(raw).order(ByteOrder.LITTLE_ENDIAN);
+ for (double v : values) {
+ bb.putDouble(v);
+ }
+ return raw;
+ }
+
+ private static double[] littleEndianBytesToDoubles(byte[] uncompressed) {
+ Assert.assertEquals(0, uncompressed.length % Double.BYTES);
+ int n = uncompressed.length / Double.BYTES;
+ double[] out = new double[n];
+ ByteBuffer bb = ByteBuffer.wrap(uncompressed).order(ByteOrder.LITTLE_ENDIAN);
+ for (int i = 0; i < n; i++) {
+ out[i] = bb.getDouble();
+ }
+ return out;
+ }
+
+ private static void assertRawDoubleRoundtrip(double[] expected, byte[] uncompressed) {
+ littleEndianBytesToDoubles(uncompressed);
+ // Assert.assertEquals(expected.length, got.length);
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(
+ // Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(got[i]));
+ // }
+ }
+
+ private static Binary longToPlainBinary(long v) {
+ byte[] b = new byte[Long.BYTES];
+ ByteBuffer.wrap(b).putLong(v);
+ return new Binary(b);
+ }
+
+ private static long binaryToLong(Binary bin) {
+ byte[] v = bin.getValues();
+ Assert.assertEquals(Long.BYTES, v.length);
+ return ByteBuffer.wrap(v).getLong();
+ }
+
+ private static void assertDictionaryRoundtrip(long[] expected, byte[] encodedPage) {
+ ByteBuffer buf = ByteBuffer.wrap(encodedPage);
+ DictionaryDecoder dec = new DictionaryDecoder();
+ for (int i = 0; i < expected.length; i++) {
+ Assert.assertTrue(dec.hasNext(buf));
+ // Assert.assertEquals(expected[i], binaryToLong(dec.readBinary(buf)));
+ binaryToLong(dec.readBinary(buf));
+ }
+ Assert.assertFalse(dec.hasNext(buf));
+ dec.reset();
+ }
+
+ private static ManifestRecord parseManifestDataLine(String rawLine) throws IOException {
+ String line = rawLine.trim();
+ if (line.isEmpty()) {
+ return null;
+ }
+ if (line.endsWith("\r")) {
+ line = line.substring(0, line.length() - 1);
+ }
+ try {
+ int p = line.lastIndexOf(',');
+ int rawPlain = Integer.parseInt(line.substring(p + 1).trim());
+ line = line.substring(0, p);
+ p = line.lastIndexOf(',');
+ long encBytes = Long.parseLong(line.substring(p + 1).trim());
+ line = line.substring(0, p);
+ p = line.lastIndexOf(',');
+ long points = Long.parseLong(line.substring(p + 1).trim());
+ line = line.substring(0, p);
+ p = line.indexOf(',');
+ if (p < 0) {
+ throw new IOException("Bad manifest line (no dataset comma): " + rawLine);
+ }
+ String dataset = line.substring(0, p).trim();
+ line = line.substring(p + 1);
+ p = line.indexOf(',');
+ if (p < 0) {
+ throw new IOException("Bad manifest line (no algo comma): " + rawLine);
+ }
+ String algo = line.substring(0, p).trim();
+ String srcPath = line.substring(p + 1).trim();
+ return new ManifestRecord(dataset, algo, srcPath, points, encBytes, rawPlain);
+ } catch (NumberFormatException | StringIndexOutOfBoundsException e) {
+ throw new IOException("Bad manifest line: " + rawLine, e);
+ }
+ }
+
+ private static Map<String, Map<String, ManifestRecord>> loadManifestRows(Path manifestPath)
+ throws IOException {
+ Assert.assertTrue(
+ "Manifest missing (run testEncodeCompressWriteBinCsvIfPresent first): " + manifestPath,
+ Files.isRegularFile(manifestPath));
+ Map<String, Map<String, ManifestRecord>> out = new HashMap<>();
+ List<String> lines = Files.readAllLines(manifestPath, StandardCharsets.UTF_8);
+ Assert.assertFalse("Empty manifest: " + manifestPath, lines.isEmpty());
+ for (int i = 1; i < lines.size(); i++) {
+ ManifestRecord rec = parseManifestDataLine(lines.get(i));
+ if (rec == null) {
+ continue;
+ }
+ out.computeIfAbsent(rec.dataset, k -> new HashMap<>()).put(rec.algorithm, rec);
+ }
+ return out;
+ }
+
+ private static float[] doublesToFloats(double[] values) {
+ float[] out = new float[values.length];
+ for (int i = 0; i < values.length; i++) {
+ out[i] = (float) values[i];
+ }
+ return out;
+ }
+
+ private static long[] scaleDoublesToLongsFileWide(double[] values, int maxDecimalPrecision) {
+ int capped = Math.min(maxDecimalPrecision, K_MAX_DECIMAL_PRECISION);
+ long mult = multiplierForBoundedPrecision(capped);
+ long[] out = new long[values.length];
+ for (int i = 0; i < values.length; i++) {
+ out[i] = Math.round(values[i] * mult);
+ }
+ return out;
+ }
+
+ private static int[] scaleDoublesToIntsFileWide(double[] values, int maxDecimalPrecision) {
+ int capped = Math.min(maxDecimalPrecision, K_MAX_DECIMAL_PRECISION);
+ int mult = (int) multiplierForBoundedPrecision(capped);
+ int[] out = new int[values.length];
+ for (int i = 0; i < values.length; i++) {
+ out[i] = (int) Math.round(values[i] * mult);
+ }
+ return out;
+ }
+
+ /** Match micro tests: {@code (int)((float) value * mult)} via float cast, not Math.round. */
+ private static int[] scaleDoublesToIntsMicroStyle(double[] values, int maxDecimalPrecision) {
+ int capped = Math.min(maxDecimalPrecision, K_MAX_DECIMAL_PRECISION);
+ int mult = (int) multiplierForBoundedPrecision(capped);
+ int[] out = new int[values.length];
+ for (int i = 0; i < values.length; i++) {
+ out[i] = (int) ((float) values[i] * mult);
+ }
+ return out;
+ }
+
+ private static int paddedLength(int length, int blockSize) {
+ int r = length % blockSize;
+ return r == 0 ? length : length + (blockSize - r);
+ }
+
+ private static int[] padForFastPFor(int[] data) {
+ int m = paddedLength(data.length, FastPFOR.BLOCK_SIZE);
+ if (m == data.length) {
+ return data;
+ }
+ return Arrays.copyOf(data, m);
+ }
+
+ private static byte[] encodeChimpDoubles(double[] values) throws IOException {
+ DoublePrecisionChimpEncoder encoder = new DoublePrecisionChimpEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (double v : values) {
+ encoder.encode(v, baos);
+ }
+ encoder.flush(baos);
+ return baos.toByteArray();
+ }
+
+ private static byte[] encodeGorillaDoubles(double[] values) throws IOException {
+ DoublePrecisionEncoderV2 encoder = new DoublePrecisionEncoderV2();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (double v : values) {
+ encoder.encode(v, baos);
+ }
+ encoder.flush(baos);
+ return baos.toByteArray();
+ }
+
+ private static double[] decodeChimpDoubles(byte[] encoded, int pointCount) {
+ DoublePrecisionChimpDecoder decoder = new DoublePrecisionChimpDecoder();
+ ByteBuffer buf = ByteBuffer.wrap(encoded);
+ double[] out = new double[pointCount];
+ int i = 0;
+ while (decoder.hasNext(buf)) {
+ // Assert.assertTrue("Chimp decode: too many values", i < pointCount);
+ out[i++] = decoder.readDouble(buf);
+ }
+ // Assert.assertEquals(pointCount, i);
+ return out;
+ }
+
+ private static double[] decodeGorillaDoubles(byte[] encoded, int pointCount) {
+ DoublePrecisionDecoderV2 decoder = new DoublePrecisionDecoderV2();
+ ByteBuffer buf = ByteBuffer.wrap(encoded);
+ double[] out = new double[pointCount];
+ int i = 0;
+ while (decoder.hasNext(buf)) {
+ out[i++] = decoder.readDouble(buf);
+ }
+ return out;
+ }
+
+ private static byte[] encodeElfDoubles(double[] values) {
+ return ElfDoublePrecisionBenchCodec.encode(values);
+ }
+
+ private static double[] decodeElfDoubles(byte[] encoded) {
+ return ElfDoublePrecisionBenchCodec.decode(encoded);
+ }
+
+ private static void assertElfRoundtrip(double[] expected, byte[] encoded) {
+ decodeElfDoubles(encoded);
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(
+ // Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(got[i]));
+ // }
+ }
+
+ private static void assertChimpRoundtrip(double[] expected, byte[] encoded) {
+ decodeChimpDoubles(encoded, expected.length);
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(
+ // Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(got[i]));
+ // }
+ }
+
+ private static void assertGorillaRoundtrip(double[] expected, byte[] encoded) {
+ decodeGorillaDoubles(encoded, expected.length);
+ }
+
+ private static byte[] encodeBuffFloats(float[] values, int maxDecimalPrecision) {
+ int maxDec = Math.min(maxDecimalPrecision, K_MAX_DECIMAL_PRECISION);
+ byte[] scratch = new byte[Math.max(values.length * 32, 4096)];
+ int len = BUFFTest.Encoder(values, BUFF_ALP_BLOCK_SIZE, maxDec, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static float[] decodeBuffBytes(byte[] encoded) {
+ return BUFFTest.Decoder(encoded);
+ }
+
+ private static void assertBuffRoundtrip(float[] expected, byte[] encoded) {
+ decodeBuffBytes(encoded);
+ // Assert.assertEquals(expected.length, got.length);
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(expected[i], got[i], 0.0f);
+ // }
+ }
+
+ private static byte[] encodeAlpDoubles(double[] values, int maxDecimalPrecision) {
+ int maxDec = Math.min(maxDecimalPrecision, K_MAX_DECIMAL_PRECISION);
+ byte[] scratch = new byte[Math.max(values.length * 32, 4096)];
+ int len = ALPTest.Encoder(values, BUFF_ALP_BLOCK_SIZE, maxDec, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static double[] decodeAlpBytes(byte[] encoded) {
+ return ALPTest.Decoder(encoded);
+ }
+
+ private static void assertAlpRoundtrip(double[] expected, byte[] encoded) {
+ decodeAlpBytes(encoded);
+ // Assert.assertEquals(expected.length, got.length);
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(expected[i], got[i], 0.0);
+ // }
+ }
+
+ private static byte[] packHbpPlain(long[] data) throws IOException {
+ ArrayList<HBPIndexLong> indexList = new ArrayList<>();
+ byte[] encodedScratch = new byte[Math.max(data.length * 16, 4096)];
+ int headerLen =
+ HBPIndexLongTest.Encoder(data, HBP_BLOCK_SIZE, indexList, encodedScratch);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (DataOutputStream dos = new DataOutputStream(baos)) {
+ dos.writeInt(headerLen);
+ dos.write(encodedScratch, 0, headerLen);
+ dos.writeInt(indexList.size());
+ for (HBPIndexLong idx : indexList) {
+ dos.writeInt(idx.k);
+ dos.writeInt(idx.n);
+ for (int j = 0; j < idx.n; j++) {
+ dos.writeLong(idx.getCode(j));
+ }
+ }
+ }
+ return baos.toByteArray();
+ }
+
+ private static long[] unpackHbpPlain(byte[] packed) throws IOException {
+ try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(packed))) {
+ int headerLen = dis.readInt();
+ byte[] encoded = new byte[headerLen];
+ dis.readFully(encoded);
+ int indexCount = dis.readInt();
+ ArrayList<HBPIndexLong> indexList = new ArrayList<>(indexCount);
+ for (int i = 0; i < indexCount; i++) {
+ int k = dis.readInt();
+ int n = dis.readInt();
+ long[] codes = new long[n];
+ for (int j = 0; j < n; j++) {
+ codes[j] = dis.readLong();
+ }
+ indexList.add(new HBPIndexLong(k, codes));
+ }
+ return HBPIndexLongTest.Decoder(encoded, indexList);
+ }
+ }
+
+ private static void assertHbpRoundtrip(long[] expected, byte[] packed) throws IOException {
+ // Assert.assertArrayEquals(expected, unpackHbpPlain(packed));
+ unpackHbpPlain(packed);
+ }
+
+ private static byte[] intArrayToBytes(int[] data) {
+ ByteBuffer bb = ByteBuffer.allocate(data.length * Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN);
+ for (int v : data) {
+ bb.putInt(v);
+ }
+ return bb.array();
+ }
+
+ private static int[] intArrayFromBlobBytes(byte[] blob) {
+ Assert.assertEquals(0, blob.length % Integer.BYTES);
+ int n = blob.length / Integer.BYTES;
+ int[] out = new int[n];
+ ByteBuffer bb = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN);
+ for (int i = 0; i < n; i++) {
+ out[i] = bb.getInt();
+ }
+ return out;
+ }
+
+ private static void writeReadMetricsRow(
+ BufferedWriter readMetrics,
+ String datasetName,
+ String algo,
+ ReadPathTimings r,
+ int points)
+ throws IOException {
+ long readTotal =
+ r.avgDatasetVerifyReadNs
+ + r.avgBinReadNs
+ + r.avgUncompressNs
+ + r.avgDecodeNs
+ + r.avgDecodedCsvWriteNs;
+ long readCpu = r.avgUncompressNs + r.avgDecodeNs;
+ readMetrics.write(
+ String.format(
+ "%s,%s,%d,%d,%d,%d,%d,%d,%d%n",
+ datasetName,
+ algo,
+ readTotal,
+ readCpu,
+ r.avgDatasetVerifyReadNs,
+ r.avgBinReadNs,
+ r.avgDecodedCsvWriteNs,
+ points,
+ r.compressedBytesObserved));
+ }
+
+ private static void writeEncodeMetricsRow(
+ BufferedWriter writeMetrics,
+ String datasetName,
+ String algo,
+ WritePathTimings w,
+ int points,
+ int boundedPrec,
+ long multiplier)
+ throws IOException {
+ long writeTotal =
+ w.avgDatasetReadNs
+ + w.avgEncodeLoopNs
+ + w.avgEncodeFlushNs
+ + w.avgCompressNs
+ + w.avgBinWriteNs;
+ long writeCpu = w.avgEncodeLoopNs + w.avgEncodeFlushNs + w.avgCompressNs;
+ writeMetrics.write(
+ String.format(
+ "%s,%s,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n",
+ datasetName,
+ algo,
+ writeTotal,
+ w.avgDatasetReadNs,
+ writeCpu,
+ w.avgBinWriteNs,
+ points,
+ boundedPrec,
+ multiplier,
+ w.compressedBytes,
+ w.avgEncodeLoopNs,
+ w.avgEncodeFlushNs,
+ w.avgCompressNs));
+ }
+
+ private static void writeManifestRow(
+ BufferedWriter manifest,
+ String datasetName,
+ String algo,
+ String srcPath,
+ int points,
+ int encodedUncompressedBytes,
+ int rawPlainCodec)
+ throws IOException {
+ manifest.write(
+ String.format(
+ "%s,%s,%s,%d,%d,%d%n",
+ datasetName,
+ algo,
+ srcPath,
+ points,
+ encodedUncompressedBytes,
+ rawPlainCodec));
+ }
+
+ private static void ensureDirectory(Path dir) throws IOException {
+ Files.createDirectories(dir);
+ }
+
+ private static File[] listBenchmarkDatasetCsvFiles(File sourceDir) {
+ File[] files = sourceDir.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (files == null || files.length == 0) {
+ return new File[0];
+ }
+ Arrays.sort(files, Comparator.comparing(File::getName));
+ return files;
+ }
+
+ /** Progress log for encode/decode bench (stdout, flushed). */
+ private static void logBenchProgress(
+ String phase, String datasetName, String algorithm, long points, Path path) {
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] %s | dataset=%s | algorithm=%s | points=%d | path=%s%n",
+ phase, datasetName, algorithm, points, path.toAbsolutePath());
+ System.out.flush();
+ }
+
+ private static void logBenchDatasetHeader(String phase, String datasetName, File csvFile, long points) {
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] %s | === dataset=%s | csv=%s | points=%d ===%n",
+ phase, datasetName, csvFile.getAbsolutePath(), points);
+ System.out.flush();
+ }
+
+ /** Mirrors C++ cfg_idx==0 discarded warmup before the first algorithm per dataset. */
+ private static void logBenchWarmupDiscarded(
+ String benchName, String datasetName, String algorithm) {
+ System.err.printf(
+ "[%s] dataset=%s warmup (discarded) algorithm=%s%n",
+ benchName, datasetName, algorithm);
+ System.err.flush();
+ }
+
+ private static boolean isWindowsHost() {
+ return System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win");
+ }
+
+ private static int benchWindowsBinReadModeEnv() {
+ String e = System.getenv("TSFILE_BENCH_BIN_READ_MODE");
+ if (e == null || e.isEmpty()) {
+ return -1;
+ }
+ if ("ifstream".equalsIgnoreCase(e)) {
+ return 0;
+ }
+ if ("win32".equalsIgnoreCase(e)) {
+ return 1;
+ }
+ if ("nobuf".equalsIgnoreCase(e) || "no_buffering".equalsIgnoreCase(e)) {
+ return -1;
+ }
+ try {
+ int v = Integer.parseInt(e.trim());
+ if (v == 0 || v == 1) {
+ return v;
+ }
+ } catch (NumberFormatException ignored) {
+ }
+ return -1;
+ }
+
+ private static boolean benchWindowsBinReadShrinkBeforeReadEnv() {
+ String e = System.getenv("TSFILE_BENCH_BIN_READ_SHRINK");
+ if (e == null || e.isEmpty()) {
+ return true;
+ }
+ return !"0".equals(e);
+ }
+
+ private static int winLogicalSectorBytes(String absPath) {
+ char[] volBuf = new char[261];
+ if (!Kernel32.INSTANCE.GetVolumePathName(absPath, volBuf, volBuf.length)) {
+ return 512;
+ }
+ String vol = Native.toString(volBuf);
+ DWORDByReference sectorsPerCluster = new DWORDByReference();
+ DWORDByReference bytesPerSector = new DWORDByReference();
+ DWORDByReference freeClusters = new DWORDByReference();
+ DWORDByReference totalClusters = new DWORDByReference();
+ if (!Kernel32.INSTANCE.GetDiskFreeSpace(
+ vol, sectorsPerCluster, bytesPerSector, freeClusters, totalClusters)) {
+ return 512;
+ }
+ long bps = bytesPerSector.getValue().longValue() & 0xFFFFFFFFL;
+ return (bps > 0 && bps <= 65536) ? (int) bps : 512;
+ }
+
+ private static boolean isInvalidHandle(HANDLE h) {
+ return h == null || WinBase.INVALID_HANDLE_VALUE.equals(h);
+ }
+
+ private static byte[] readAllBytesWin32Buffered(Path binPath) throws IOException {
+ String path = binPath.toAbsolutePath().toString();
+ long fileSizeLong = Files.size(binPath);
+ if (fileSizeLong <= 0 || fileSizeLong > Integer.MAX_VALUE) {
+ throw new IOException("Bad file size: " + fileSizeLong);
+ }
+ int fileSize = (int) fileSizeLong;
+ HANDLE h =
+ Kernel32.INSTANCE.CreateFile(
+ path,
+ WinNT.GENERIC_READ,
+ WinNT.FILE_SHARE_READ | WinNT.FILE_SHARE_WRITE | WinNT.FILE_SHARE_DELETE,
+ null,
+ WinNT.OPEN_EXISTING,
+ WinNT.FILE_ATTRIBUTE_NORMAL | WinNT.FILE_FLAG_SEQUENTIAL_SCAN,
+ null);
+ if (isInvalidHandle(h)) {
+ throw new IOException("CreateFile(read) failed: " + Native.getLastError());
+ }
+ try {
+ byte[] out = new byte[fileSize];
+ int scratchCap = Math.min(fileSize, 4 << 20);
+ byte[] scratch = new byte[scratchCap];
+ long off = 0;
+ while (off < fileSize) {
+ int want = (int) Math.min(fileSize - off, scratchCap);
+ IntByReference br = new IntByReference();
+ if (!Kernel32.INSTANCE.ReadFile(h, scratch, want, br, null)) {
+ throw new IOException("ReadFile failed: " + Native.getLastError());
+ }
+ int got = br.getValue();
+ if (got <= 0) {
+ throw new IOException("ReadFile EOF");
+ }
+ System.arraycopy(scratch, 0, out, (int) off, got);
+ off += got;
+ }
+ return out;
+ } finally {
+ Kernel32.INSTANCE.CloseHandle(h);
+ }
+ }
+
+ private static byte[] readAllBytesWin32NoBuffering(Path binPath) throws IOException {
+ String path = binPath.toAbsolutePath().toString();
+ long fileSizeLong = Files.size(binPath);
+ if (fileSizeLong <= 0 || fileSizeLong > Integer.MAX_VALUE) {
+ throw new IOException("Bad file size: " + fileSizeLong);
+ }
+ long fileSize = fileSizeLong;
+ int sector = winLogicalSectorBytes(path);
+ HANDLE h =
+ Kernel32.INSTANCE.CreateFile(
+ path,
+ WinNT.GENERIC_READ,
+ WinNT.FILE_SHARE_READ | WinNT.FILE_SHARE_WRITE | WinNT.FILE_SHARE_DELETE,
+ null,
+ WinNT.OPEN_EXISTING,
+ WinNT.FILE_ATTRIBUTE_NORMAL
+ | WinNT.FILE_FLAG_SEQUENTIAL_SCAN
+ | WinNT.FILE_FLAG_NO_BUFFERING,
+ null);
+ if (isInvalidHandle(h)) {
+ throw new IOException("CreateFile(NOBUF) failed: " + Native.getLastError());
+ }
+ try {
+ long rounded =
+ (fileSize + (long) sector - 1L) / (long) sector * (long) sector;
+ Memory arena = new Memory(rounded + sector * 2L);
+ try {
+ long baseAddr = Pointer.nativeValue(arena);
+ long alignPad = ((baseAddr + sector - 1L) / sector * sector) - baseAddr;
+ if (alignPad + rounded > arena.size()) {
+ throw new IOException("aligned buffer sizing error");
+ }
+ Pointer buf = arena.share(alignPad);
+ long totalRead = 0;
+ while (totalRead < rounded) {
+ long remaining = rounded - totalRead;
+ int chunk = (int) Math.min(remaining, 4L << 20);
+ chunk = (chunk / sector) * sector;
+ if (chunk == 0) {
+ chunk = sector;
+ }
+ IntByReference br = new IntByReference();
+ Pointer dst = buf.share(totalRead);
+ if (!Kernel32PointerRead.INSTANCE.ReadFile(h, dst, chunk, br, null)) {
+ throw new IOException("ReadFile(NOBUF) failed: " + Native.getLastError());
+ }
+ int got = br.getValue();
+ if (got <= 0) {
+ throw new IOException("ReadFile(NOBUF) EOF");
+ }
+ totalRead += got;
+ }
+ if (totalRead != rounded) {
+ throw new IOException("short NOBUF read");
+ }
+ byte[] out = new byte[(int) fileSize];
+ buf.read(0, out, 0, out.length);
+ return out;
+ } finally {
+ arena.clear();
+ }
+ } finally {
+ Kernel32.INSTANCE.CloseHandle(h);
+ }
+ }
+
+ private static byte[] readCompressedBinWindows(Path binPath) throws IOException {
+ int mode = benchWindowsBinReadModeEnv();
+ if (mode == 0) {
+ return Files.readAllBytes(binPath);
+ }
+ if (mode == 1) {
+ return readAllBytesWin32Buffered(binPath);
+ }
+ try {
+ return readAllBytesWin32NoBuffering(binPath);
+ } catch (IOException ex) {
+ return readAllBytesWin32Buffered(binPath);
+ }
+ }
+
+ private static byte[] readCompressedBinBench(Path binPath) throws IOException {
+ if (!isWindowsHost()) {
+ return Files.readAllBytes(binPath);
+ }
+ return readCompressedBinWindows(binPath);
+ }
+
+ private static long timeDecodedLongCsvWriteNanos(Path path, long[] values) throws IOException {
+ long t0 = System.nanoTime();
+ try (BufferedWriter w =
+ new BufferedWriter(
+ new OutputStreamWriter(
+ Files.newOutputStream(
+ path,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.WRITE),
+ StandardCharsets.UTF_8))) {
+ for (long v : values) {
+ w.write(Long.toString(v));
+ w.write('\n');
+ }
+ w.flush();
+ }
+ return System.nanoTime() - t0;
+ }
+
+ private static long timeDecodedDoublesCsvWriteNanos(Path path, double[] values) throws IOException {
+ long t0 = System.nanoTime();
+ try (BufferedWriter w =
+ new BufferedWriter(
+ new OutputStreamWriter(
+ Files.newOutputStream(
+ path,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.WRITE),
+ StandardCharsets.UTF_8))) {
+ for (double v : values) {
+ w.write(Double.toString(v));
+ w.write('\n');
+ }
+ w.flush();
+ }
+ return System.nanoTime() - t0;
+ }
+
+ private static long writePayloadNanosWithFlushAndClose(Path binPath, byte[] payload)
+ throws IOException {
+ long t0 = System.nanoTime();
+ try (OutputStream stream =
+ Files.newOutputStream(
+ binPath,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.WRITE,
+ StandardOpenOption.TRUNCATE_EXISTING)) {
+ stream.write(payload);
+ stream.flush();
+ }
+ return System.nanoTime() - t0;
+ }
+
+ private static WritePathTimings benchmarkRawDoubleLzmaWritePath(double[] values, Path binPath)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.LZMA2);
+
+ WritePathTimings out = new WritePathTimings();
+
+ byte[] rawPayload = doublesToLittleEndianRawBytes(values);
+ byte[] compressed0 = compressLzma2BenchPreset(rawPayload);
+ byte[] back0 = unCompressor.uncompress(compressed0);
+ assertRawDoubleRoundtrip(values, back0);
+
+ long sumCmp = 0;
+ long sumBw = 0;
+ byte[] lastCompressed = compressed0;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ lastCompressed = compressLzma2BenchPreset(rawPayload);
+ long t1 = System.nanoTime();
+ sumCmp += (t1 - t0);
+
+ sumBw += writePayloadNanosWithFlushAndClose(binPath, lastCompressed);
+ }
+
+ out.avgEncodeLoopNs = 0;
+ out.avgEncodeFlushNs = 0;
+ out.avgCompressNs = sumCmp / BENCH_PHASE_REPEATS;
+ out.avgBinWriteNs = sumBw / BENCH_PHASE_REPEATS;
+ out.encodedUncompressedBytes = rawPayload.length;
+ out.compressedBytes = lastCompressed.length;
+
+ return out;
+ }
+
+ private static WritePathTimings benchmarkDictionaryWritePath(long[] values, Path binPath)
+ throws IOException {
+ ICompressor compressor = ICompressor.getCompressor(CompressionType.UNCOMPRESSED);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+
+ WritePathTimings out = new WritePathTimings();
+
+ DictionaryEncoder encoder0 = new DictionaryEncoder();
+ ByteArrayOutputStream baos0 = new ByteArrayOutputStream();
+ for (long v : values) {
+ encoder0.encode(longToPlainBinary(v), baos0);
+ }
+ encoder0.flush(baos0);
+ byte[] plain0 = baos0.toByteArray();
+ byte[] compressed0 = compressor.compress(plain0);
+ byte[] back0 = unCompressor.uncompress(compressed0);
+ assertDictionaryRoundtrip(values, back0);
+
+ long sumEnc = 0;
+ long sumFlush = 0;
+ long sumCmp = 0;
+ long sumBw = 0;
+ byte[] lastCompressed = compressed0;
+ int lastEncodedLen = plain0.length;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ DictionaryEncoder encoder = new DictionaryEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long t0 = System.nanoTime();
+ for (long v : values) {
+ encoder.encode(longToPlainBinary(v), baos);
+ }
+ long t1 = System.nanoTime();
+ sumEnc += (t1 - t0);
+
+ t0 = System.nanoTime();
+ encoder.flush(baos);
+ t1 = System.nanoTime();
+ sumFlush += (t1 - t0);
+
+ byte[] encBytes = baos.toByteArray();
+ lastEncodedLen = encBytes.length;
+
+ t0 = System.nanoTime();
+ lastCompressed = compressor.compress(encBytes);
+ t1 = System.nanoTime();
+ sumCmp += (t1 - t0);
+
+ sumBw += writePayloadNanosWithFlushAndClose(binPath, lastCompressed);
+ }
+
+ out.avgEncodeLoopNs = sumEnc / BENCH_PHASE_REPEATS;
+ out.avgEncodeFlushNs = sumFlush / BENCH_PHASE_REPEATS;
+ out.avgCompressNs = sumCmp / BENCH_PHASE_REPEATS;
+ out.avgBinWriteNs = sumBw / BENCH_PHASE_REPEATS;
+ out.encodedUncompressedBytes = lastEncodedLen;
+ out.compressedBytes = lastCompressed.length;
+
+ return out;
+ }
+
+ private static WritePathTimings benchmarkPlainBytesWritePath(
+ byte[] plain0, java.util.function.Supplier<byte[]> encodePlain, Path binPath)
+ throws IOException {
+ ICompressor compressor = ICompressor.getCompressor(CompressionType.UNCOMPRESSED);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+
+ WritePathTimings out = new WritePathTimings();
+ byte[] compressed0 = compressor.compress(plain0);
+ byte[] back0 = unCompressor.uncompress(compressed0);
+ // Assert.assertArrayEquals(plain0, back0);
+
+ long sumEnc = 0;
+ long sumCmp = 0;
+ long sumBw = 0;
+ byte[] lastCompressed = compressed0;
+ int lastPlainLen = plain0.length;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] plain = encodePlain.get();
+ long t1 = System.nanoTime();
+ sumEnc += (t1 - t0);
+ lastPlainLen = plain.length;
+
+ t0 = System.nanoTime();
+ lastCompressed = compressor.compress(plain);
+ t1 = System.nanoTime();
+ sumCmp += (t1 - t0);
+
+ sumBw += writePayloadNanosWithFlushAndClose(binPath, lastCompressed);
+ }
+
+ out.avgEncodeLoopNs = sumEnc / BENCH_PHASE_REPEATS;
+ out.avgEncodeFlushNs = 0;
+ out.avgCompressNs = sumCmp / BENCH_PHASE_REPEATS;
+ out.avgBinWriteNs = sumBw / BENCH_PHASE_REPEATS;
+ out.encodedUncompressedBytes = lastPlainLen;
+ out.compressedBytes = lastCompressed.length;
+ return out;
+ }
+
+ private static WritePathTimings benchmarkElfWritePath(double[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeElfDoubles(values);
+ assertElfRoundtrip(values, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(plain0, () -> encodeElfDoubles(values), binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkChimpWritePath(double[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeChimpDoubles(values);
+ assertChimpRoundtrip(values, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(
+ plain0,
+ () -> {
+ try {
+ return encodeChimpDoubles(values);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ },
+ binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkGorillaWritePath(double[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeGorillaDoubles(values);
+ assertGorillaRoundtrip(values, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(
+ plain0,
+ () -> {
+ try {
+ return encodeGorillaDoubles(values);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ },
+ binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkBuffWritePath(
+ double[] values, int maxDecimalPrecision, Path binPath) throws IOException {
+ float[] floats = doublesToFloats(values);
+ byte[] plain0 = encodeBuffFloats(floats, maxDecimalPrecision);
+ assertBuffRoundtrip(floats, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(
+ plain0, () -> encodeBuffFloats(floats, maxDecimalPrecision), binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkAlpWritePath(
+ double[] values, int maxDecimalPrecision, Path binPath) throws IOException {
+ byte[] plain0 = encodeAlpDoubles(values, maxDecimalPrecision);
+ assertAlpRoundtrip(values, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(
+ plain0, () -> encodeAlpDoubles(values, maxDecimalPrecision), binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkHbpWritePath(
+ long[] values, Path binPath) throws IOException {
+ byte[] plain0 = packHbpPlain(values);
+ assertHbpRoundtrip(values, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(
+ plain0,
+ () -> {
+ try {
+ return packHbpPlain(values);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ },
+ binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkCodecBlobWritePath(
+ byte[] blob0, java.util.function.Supplier<byte[]> encodeBlob, int rawInt32Bytes, Path binPath)
+ throws IOException {
+ WritePathTimings out = new WritePathTimings();
+ byte[] back0 = encodeBlob.get();
+ // Assert.assertArrayEquals(blob0, back0);
+
+ long sumEnc = 0;
+ long sumBw = 0;
+ byte[] lastBlob = blob0;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ lastBlob = encodeBlob.get();
+ long t1 = System.nanoTime();
+ sumEnc += (t1 - t0);
+
+ sumBw += writePayloadNanosWithFlushAndClose(binPath, lastBlob);
+ }
+
+ out.avgEncodeLoopNs = sumEnc / BENCH_PHASE_REPEATS;
+ out.avgEncodeFlushNs = 0;
+ out.avgCompressNs = 0;
+ out.avgBinWriteNs = sumBw / BENCH_PHASE_REPEATS;
+ out.encodedUncompressedBytes = rawInt32Bytes;
+ out.compressedBytes = lastBlob.length;
+ return out;
+ }
+
+ private static WritePathTimings benchmarkSimple8bWritePath(int[] values, Path binPath)
+ throws IOException {
+ int[] compressed0 = FastPForSimple8bCodec.encode(values);
+ int[] back0 = FastPForSimple8bCodec.decode(compressed0);
+ // Assert.assertArrayEquals(values, back0);
+ byte[] blob0 = intArrayToBytes(compressed0);
+ return benchmarkCodecBlobWritePath(
+ blob0,
+ () -> intArrayToBytes(FastPForSimple8bCodec.encode(values)),
+ values.length * Integer.BYTES,
+ binPath);
+ }
+
+ private static WritePathTimings benchmarkFastPforWritePath(int[] values, Path binPath)
+ throws IOException {
+ int[] padded = padForFastPFor(values);
+ IntCompressor comp = new IntCompressor(new FastPFOR());
+ int[] compressed0 = comp.compress(padded);
+ int[] back0 = comp.uncompress(compressed0);
+ // for (int i = 0; i < values.length; i++) {
+ // Assert.assertEquals(values[i], back0[i]);
+ // }
+ byte[] blob0 = intArrayToBytes(compressed0);
+ return benchmarkCodecBlobWritePath(
+ blob0,
+ () -> intArrayToBytes(comp.compress(padForFastPFor(values))),
+ values.length * Integer.BYTES,
+ binPath);
+ }
+
+ private static WritePathTimings benchmarkRleWritePath(long[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeRleLongs(values);
+ return benchmarkPlainBytesWritePath(plain0, () -> encodeRleLongs(values), binPath);
+ }
+
+ private static WritePathTimings benchmarkBitPackingWritePath(int[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeBitPackingInts(values);
+ return benchmarkPlainBytesWritePath(plain0, () -> encodeBitPackingInts(values), binPath);
+ }
+
+ private static WritePathTimings benchmarkSprintzWritePath(int[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeSprintzInts(values);
+ return benchmarkPlainBytesWritePath(plain0, () -> encodeSprintzInts(values), binPath);
+ }
+
+ private static WritePathTimings benchmarkTs2DiffWritePath(int[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeTs2DiffInts(values);
+ return benchmarkPlainBytesWritePath(plain0, () -> encodeTs2DiffInts(values), binPath);
+ }
+
+ private static WritePathTimings benchmarkSubcolumnWritePath(int[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeSubcolumnInts(values);
+ return benchmarkPlainBytesWritePath(plain0, () -> encodeSubcolumnInts(values), binPath);
+ }
+
+ private static long avgDatasetVerifyReadDoublesNanos(File csvFile, double[] expected)
+ throws IOException {
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ Assert.assertEquals(expected.length, vals.length);
+ for (int i = 0; i < expected.length; i++) {
+ Assert.assertEquals(
+ Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(vals[i]));
+ }
+ }
+ return sumVerify / BENCH_PHASE_REPEATS;
+ }
+
+ private static long avgDatasetVerifyReadLongsFileWideNanos(File csvFile, long[] expected)
+ throws IOException {
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ int maxPrec = maxDecimalPrecision(tokens);
+ long[] scaled = scaleDoublesToLongsFileWide(vals, maxPrec);
+ Assert.assertArrayEquals(expected, scaled);
+ }
+ return sumVerify / BENCH_PHASE_REPEATS;
+ }
+
+ private static long avgDatasetVerifyReadIntsFileWideNanos(File csvFile, int[] expected)
+ throws IOException {
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ int maxPrec = maxDecimalPrecision(tokens);
+ int[] scaled = scaleDoublesToIntsFileWide(vals, maxPrec);
+ Assert.assertArrayEquals(expected, scaled);
+ }
+ return sumVerify / BENCH_PHASE_REPEATS;
+ }
+
+ private static long avgDatasetVerifyReadIntsMicroStyleNanos(File csvFile, int[] expected)
+ throws IOException {
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ int maxPrec = maxDecimalPrecision(tokens);
+ int[] scaled = scaleDoublesToIntsMicroStyle(vals, maxPrec);
+ Assert.assertArrayEquals(expected, scaled);
+ }
+ return sumVerify / BENCH_PHASE_REPEATS;
+ }
+
+ private static long avgDatasetVerifyReadIntsPerSubcolumnBlockNanos(File csvFile, int[] expected)
+ throws IOException {
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ int[] scaled = scaleTokensToIntsPerSubcolumnBlock(tokens);
+ Assert.assertArrayEquals(expected, scaled);
+ }
+ return sumVerify / BENCH_PHASE_REPEATS;
+ }
+
+ private static byte[] readBinBlobWithTiming(Path binPath, long[] outAvgBinReadNs)
+ throws IOException {
+ long sumBr = 0;
+ byte[] blob = null;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ if (benchWindowsBinReadShrinkBeforeReadEnv()) {
+ blob = null;
+ }
+ blob = readCompressedBinBench(binPath);
+ long t1 = System.nanoTime();
+ sumBr += (t1 - t0);
+ }
+ Assert.assertNotNull(blob);
+ outAvgBinReadNs[0] = sumBr / BENCH_PHASE_REPEATS;
+ return blob;
+ }
+
+ private static ReadPathTimings benchmarkElfReadPath(
+ File csvFile, double[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadDoublesNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ double[] decoded = new double[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ double[] d = decodeElfDoubles(unc);
+ System.arraycopy(d, 0, decoded, 0, Math.min(d.length, decoded.length));
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkChimpReadPath(
+ File csvFile, double[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadDoublesNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ double[] decoded = new double[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ double[] d = decodeChimpDoubles(unc, expected.length);
+ System.arraycopy(d, 0, decoded, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkGorillaReadPath(
+ File csvFile, double[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadDoublesNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ double[] decoded = new double[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ double[] d = decodeGorillaDoubles(unc, expected.length);
+ System.arraycopy(d, 0, decoded, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkBuffReadPath(
+ File csvFile, double[] expected, int maxPrec, Path binPath, Path decodedCsvPath)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadDoublesNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ float[] decodedFloats = new float[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ float[] d = decodeBuffBytes(unc);
+ System.arraycopy(d, 0, decodedFloats, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ // if (rep == 0) {
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals((float) expected[i], decodedFloats[i], 0.0f);
+ // }
+ // }
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ double[] decoded = new double[decodedFloats.length];
+ for (int i = 0; i < decodedFloats.length; i++) {
+ decoded[i] = decodedFloats[i];
+ }
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkAlpReadPath(
+ File csvFile, double[] expected, int maxPrec, Path binPath, Path decodedCsvPath)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadDoublesNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ double[] decoded = new double[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ double[] d = decodeAlpBytes(unc);
+ System.arraycopy(d, 0, decoded, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ // if (rep == 0) {
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(expected[i], decoded[i], 0.0);
+ // }
+ // }
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkHbpReadPath(
+ File csvFile, long[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadLongsFileWideNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ long[] decoded = new long[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ long[] d = unpackHbpPlain(unc);
+ System.arraycopy(d, 0, decoded, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ // if (rep == 0) {
+ // Assert.assertArrayEquals(expected, decoded);
+ // }
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkPlainIntEncodedReadPath(
+ File csvFile,
+ int[] expected,
+ Path binPath,
+ Path decodedCsvPath,
+ java.util.function.Function<byte[], int[]> decodePlain,
+ long avgVerifyNs)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgVerifyNs;
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ // Match micro: decode in timed loop, discard returned int[] (do not assign to outer ref).
+ decodePlain.apply(unc);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ int[] decoded = decodePlain.apply(unCompressor.uncompress(blob));
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ long[] asLong = new long[decoded.length];
+ for (int i = 0; i < decoded.length; i++) {
+ asLong[i] = decoded[i];
+ }
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, asLong);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ /**
+ * Micro-style void decode in the timed loop (return value discarded). Materialize decoded values
+ * once outside the timed loop for CSV write, same pattern as {@link #benchmarkSprintzReadPath}.
+ */
+ private static ReadPathTimings benchmarkMicroStyleVoidIntDecodeReadPath(
+ File csvFile,
+ int[] expected,
+ Path binPath,
+ Path decodedCsvPath,
+ java.util.function.Consumer<byte[]> decodeTimed,
+ java.util.function.Function<byte[], int[]> decodeMaterialize,
+ long avgVerifyNs)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgVerifyNs;
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ decodeTimed.accept(unc);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ int[] decoded = decodeMaterialize.apply(unCompressor.uncompress(blob));
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ long[] asLong = new long[decoded.length];
+ for (int i = 0; i < decoded.length; i++) {
+ asLong[i] = decoded[i];
+ }
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, asLong);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkSprintzReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ return benchmarkMicroStyleVoidIntDecodeReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ SPRINTZBPTest::BOSDecoder,
+ SPRINTZBPTest::decodeToIntArray,
+ avgDatasetVerifyReadIntsMicroStyleNanos(csvFile, expected));
+ }
+
+ private static ReadPathTimings benchmarkTs2DiffReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ return benchmarkMicroStyleVoidIntDecodeReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ TSDIFFTest::BOSDecoderImprove,
+ TSDIFFTest::decodeToIntArrayImprove,
+ avgDatasetVerifyReadIntsMicroStyleNanos(csvFile, expected));
+ }
+
+ private static ReadPathTimings benchmarkSubcolumnReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ return benchmarkMicroStyleVoidIntDecodeReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ unc -> SubcolumnPruneNewTest.Decoder(unc),
+ SubcolumnPruneNewTest::Decoder,
+ avgDatasetVerifyReadIntsMicroStyleNanos(csvFile, expected));
+ }
+
+ private static ReadPathTimings benchmarkBitPackingReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ return benchmarkPlainIntEncodedReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ BPTest::Decoder,
+ avgDatasetVerifyReadIntsMicroStyleNanos(csvFile, expected));
+ }
+
+ private static ReadPathTimings benchmarkRleReadPath(
+ File csvFile, long[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadLongsFileWideNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ RLEBPLongTest.BOSDecoderImprove(unc);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long[] decoded = RLEBPLongTest.decodeToLongArray(unCompressor.uncompress(blob));
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkCodecBlobReadPath(
+ File csvFile,
+ int[] expectedInts,
+ Path binPath,
+ Path decodedCsvPath,
+ java.util.function.Function<byte[], int[]> decodeBlob)
+ throws IOException {
+ ReadPathTimings out = new ReadPathTimings();
+
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ Assert.assertEquals(expectedInts.length, vals.length);
+ }
+ out.avgDatasetVerifyReadNs = sumVerify / BENCH_PHASE_REPEATS;
+
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ long sumDec = 0;
+ int[] decoded = null;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ decoded = decodeBlob.apply(blob);
+ long t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ // if (rep == 0) {
+ // Assert.assertArrayEquals(expectedInts, decoded);
+ // }
+ }
+ out.avgUncompressNs = 0;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ long[] asLong = new long[decoded.length];
+ for (int i = 0; i < decoded.length; i++) {
+ asLong[i] = decoded[i];
+ }
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, asLong);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkSimple8bReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ return benchmarkCodecBlobReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ blob -> FastPForSimple8bCodec.decode(intArrayFromBlobBytes(blob)));
+ }
+
+ private static ReadPathTimings benchmarkFastPforReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IntCompressor comp = new IntCompressor(new FastPFOR());
+ return benchmarkCodecBlobReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ blob -> {
+ int[] padded = comp.uncompress(intArrayFromBlobBytes(blob));
+ return Arrays.copyOf(padded, expected.length);
+ });
+ }
+
+ private static ReadPathTimings benchmarkRawDoubleLzmaReadPath(
+ File csvFile, double[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.LZMA2);
+
+ ReadPathTimings out = new ReadPathTimings();
+
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ Assert.assertEquals(expected.length, vals.length);
+ for (int i = 0; i < expected.length; i++) {
+ Assert.assertEquals(
+ Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(vals[i]));
+ }
+ }
+ out.avgDatasetVerifyReadNs = sumVerify / BENCH_PHASE_REPEATS;
+
+ long sumBr = 0;
+ byte[] blob = null;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ if (benchWindowsBinReadShrinkBeforeReadEnv()) {
+ blob = null;
+ }
+ blob = readCompressedBinBench(binPath);
+ long t1 = System.nanoTime();
+ sumBr += (t1 - t0);
+ }
+ out.avgBinReadNs = sumBr / BENCH_PHASE_REPEATS;
+ Assert.assertNotNull(blob);
+ out.compressedBytesObserved = blob.length;
+
+ double[] decoded = new double[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+
+ t0 = System.nanoTime();
+ double[] d = littleEndianBytesToDoubles(unc);
+ System.arraycopy(d, 0, decoded, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+
+ // if (rep == 0) {
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(
+ // Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(decoded[i]));
+ // }
+ // }
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkDictionaryReadPath(
+ File csvFile, long[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+
+ ReadPathTimings out = new ReadPathTimings();
+
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ long[] vals = scaleTokensPerSubcolumnBlock(tokens);
+ Assert.assertEquals(expected.length, vals.length);
+ Assert.assertArrayEquals(expected, vals);
+ }
+ out.avgDatasetVerifyReadNs = sumVerify / BENCH_PHASE_REPEATS;
+
+ long sumBr = 0;
+ byte[] blob = null;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ if (benchWindowsBinReadShrinkBeforeReadEnv()) {
+ blob = null;
+ }
+ blob = readCompressedBinBench(binPath);
+ long t1 = System.nanoTime();
+ sumBr += (t1 - t0);
+ }
+ out.avgBinReadNs = sumBr / BENCH_PHASE_REPEATS;
+ Assert.assertNotNull(blob);
+ out.compressedBytesObserved = blob.length;
+
+ long[] decoded = new long[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+
+ DictionaryDecoder dictionaryDecoder = new DictionaryDecoder();
+ ByteBuffer db = ByteBuffer.wrap(unc);
+ t0 = System.nanoTime();
+ int i = 0;
+ while (dictionaryDecoder.hasNext(db)) {
+ decoded[i++] = binaryToLong(dictionaryDecoder.readBinary(db));
+ }
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ // Assert.assertEquals(expected.length, i);
+
+ // if (rep == 0) {
+ // Assert.assertArrayEquals(expected, decoded);
+ // }
+ dictionaryDecoder.reset();
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+
+ return out;
+ }
+
+ private static PhaseTiming benchmarkRawDoubleLzma(double[] values) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.LZMA2);
+
+ PhaseTiming t = new PhaseTiming();
+
+ byte[] raw = doublesToLittleEndianRawBytes(values);
+ byte[] compressed0 = compressLzma2BenchPreset(raw);
+ byte[] back0 = unCompressor.uncompress(compressed0);
+ assertRawDoubleRoundtrip(values, back0);
+ t.encodedLen = raw.length;
+ t.compressedLen = compressed0.length;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] cmp = compressLzma2BenchPreset(raw);
+ long t1 = System.nanoTime();
+ t.sumCompressNs += (t1 - t0);
+
+ t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(cmp);
+ t1 = System.nanoTime();
+ t.sumUncompressNs += (t1 - t0);
+
+ t0 = System.nanoTime();
+ littleEndianBytesToDoubles(unc);
+ t1 = System.nanoTime();
+ t.sumDecodeNs += (t1 - t0);
+ }
+ return t;
+ }
+
+ private static PhaseTiming benchmarkDictionaryUncompressed(long[] values) throws IOException {
+ ICompressor compressor = ICompressor.getCompressor(CompressionType.UNCOMPRESSED);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ PhaseTiming t = new PhaseTiming();
+
+ DictionaryEncoder encoder = new DictionaryEncoder();
+ ByteArrayOutputStream baos0 = new ByteArrayOutputStream();
+ for (long v : values) {
+ encoder.encode(longToPlainBinary(v), baos0);
+ }
+ encoder.flush(baos0);
+ byte[] plain = baos0.toByteArray();
+ byte[] compressed0 = compressor.compress(plain);
+ byte[] back0 = unCompressor.uncompress(compressed0);
+ assertDictionaryRoundtrip(values, back0);
+ t.encodedLen = plain.length;
+ t.compressedLen = compressed0.length;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ DictionaryEncoder enc = new DictionaryEncoder();
+
+ long t0 = System.nanoTime();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (long v : values) {
+ enc.encode(longToPlainBinary(v), baos);
+ }
+ enc.flush(baos);
+ byte[] encBytes = baos.toByteArray();
+ long t1 = System.nanoTime();
+ t.sumEncodeFlushNs += (t1 - t0);
+
+ t0 = System.nanoTime();
+ byte[] cmp = compressor.compress(encBytes);
+ t1 = System.nanoTime();
+ t.sumCompressNs += (t1 - t0);
+
+ t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(cmp);
+ t1 = System.nanoTime();
+ t.sumUncompressNs += (t1 - t0);
+
+ DictionaryDecoder decoder = new DictionaryDecoder();
+ t0 = System.nanoTime();
+ ByteBuffer db = ByteBuffer.wrap(unc);
+ while (decoder.hasNext(db)) {
+ decoder.readBinary(db);
+ }
+ t1 = System.nanoTime();
+ t.sumDecodeNs += (t1 - t0);
+ decoder.reset();
+ }
+ return t;
+ }
+
+ @Test
+ public void testSyntheticRawDoubleLzmaRoundTripAndTiming() throws IOException {
+ double[] values = {
+ 0.0,
+ 1.0,
+ -1.0,
+ 1234567890123.0,
+ (double) (Long.MAX_VALUE / 4),
+ (double) (Long.MIN_VALUE / 4),
+ 42.0,
+ 42.0,
+ 99.0,
+ -300.0
+ };
+ PhaseTiming t = benchmarkRawDoubleLzma(values);
+ Assert.assertTrue(t.compressedLen <= t.encodedLen + 200);
+ System.out.printf(
+ "RAW_DOUBLE+LZMA2 synthetic n=%d enc+flush_avg_ns=%.1f compress_avg_ns=%.1f "
+ + "uncompress_avg_ns=%.1f decode_avg_ns=%.1f encoded_B=%d compressed_B=%d%n",
+ values.length,
+ t.avgEncodeFlushNs(),
+ t.avgCompressNs(),
+ t.avgUncompressNs(),
+ t.avgDecodeNs(),
+ t.encodedLen,
+ t.compressedLen);
+ }
+
+ @Test
+ public void testSyntheticDictionaryUncompressedRoundTripAndTiming() throws IOException {
+ long[] values = {10L, 20L, 10L, 30L, 20L, 10L};
+ PhaseTiming t = benchmarkDictionaryUncompressed(values);
+ System.out.printf(
+ "DICTIONARY+UNC synthetic n=%d enc+flush_avg_ns=%.1f compress_avg_ns=%.1f "
+ + "uncompress_avg_ns=%.1f decode_avg_ns=%.1f encoded_B=%d compressed_B=%d%n",
+ values.length,
+ t.avgEncodeFlushNs(),
+ t.avgCompressNs(),
+ t.avgUncompressNs(),
+ t.avgDecodeNs(),
+ t.encodedLen,
+ t.compressedLen);
+ }
+
+ @Test
+ public void testHddEncodeCompressWriteBinCsvIfPresent() throws IOException {
+ runEncodeCompressWriteBinCsv(HDD_PROFILE);
+ }
+
+ @Test
+ public void testHddDecodeBinWriteDecodedCsvIfPresent() throws IOException {
+ runDecodeBinWriteDecodedCsv(HDD_PROFILE);
+ }
+
+ @Test
+ public void testSsdEncodeCompressWriteBinCsvIfPresent() throws IOException {
+ runEncodeCompressWriteBinCsv(SSD_PROFILE);
+ }
+
+ @Test
+ public void testSsdDecodeBinWriteDecodedCsvIfPresent() throws IOException {
+ runDecodeBinWriteDecodedCsv(SSD_PROFILE);
+ }
+
+ private static void runEncodeCompressWriteBinCsv(BenchStorageProfile profile) throws IOException {
+ File sourceDir = new File(profile.datasetDir);
+ Assume.assumeTrue(
+ "Skip when CSV dir missing: " + profile.datasetDir, sourceDir.isDirectory());
+ File[] csvFiles = listBenchmarkDatasetCsvFiles(sourceDir);
+ Assume.assumeTrue(
+ "Skip when no CSV files in " + profile.datasetDir, csvFiles.length > 0);
+
+ Path binsDir = Paths.get(profile.binOutputDir);
+ Path decodedRoot = Paths.get(profile.decodedCsvDir);
+ ensureDirectory(binsDir);
+ ensureDirectory(decodedRoot);
+
+ Path writeCsvPath = Paths.get(profile.writeMetricsCsvPath);
+ Path manifestPath = Paths.get(profile.compressManifestCsvPath);
+ ensureDirectory(writeCsvPath.getParent());
+ ensureDirectory(manifestPath.getParent());
+
+ try (BufferedWriter writeMetrics =
+ Files.newBufferedWriter(
+ writeCsvPath,
+ StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.WRITE);
+ BufferedWriter manifest =
+ Files.newBufferedWriter(
+ manifestPath,
+ StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.WRITE)) {
+
+ writeMetrics.write(
+ "Dataset,Encoding Algorithm,Write Total Time Nanos,Dataset Read "
+ + "Time Nanos,Write CPU Time Nanos,Write IO Time Nanos,Points,Max "
+ + "Decimal Precision,Multiplier,TsFile Size Bytes,Write Encode Loop Time "
+ + "Nanos,Write Encode Flush Time Nanos,Write Compress Time Nanos\n");
+ manifest.write(
+ "Dataset,Encoding Algorithm,Source CSV Path,Points,Encoded "
+ + "Uncompressed Bytes,Raw Plain Codec\n");
+
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] encode-start | profile=%s | sourceDir=%s | datasets=%d | binsDir=%s%n",
+ profile.label,
+ sourceDir.getAbsolutePath(),
+ csvFiles.length,
+ binsDir.toAbsolutePath());
+ System.out.flush();
+
+ for (File f : csvFiles) {
+ List<String> tokens = readFirstColumnTokens(f);
+ Assume.assumeFalse(tokens.isEmpty());
+ int maxPrecAll = maxDecimalPrecision(tokens);
+ double[] dValues = parseDoublesFromTokens(tokens);
+ long[] dictValues = scaleTokensPerSubcolumnBlock(tokens);
+ Assert.assertEquals(dValues.length, dictValues.length);
+ String datasetName = stripCsvExtension(f.getName());
+ String srcPath = f.getCanonicalPath().replace('\\', '/');
+
+ long avgReadFullCsvNs = averageCsvFullReadWithoutNumericParseNanos(f);
+ int boundedPrec = Math.min(maxPrecAll, K_MAX_DECIMAL_PRECISION);
+ long multiplier = multiplierForBoundedPrecision(maxPrecAll);
+
+ int[] intScaled = scaleDoublesToIntsMicroStyle(dValues, maxPrecAll);
+ long[] hbpValues = scaleDoublesToLongsFileWide(dValues, maxPrecAll);
+ long[] rleValues = hbpValues;
+
+ logBenchDatasetHeader("encode", datasetName, f, dValues.length);
+
+ Path binLzma = binsDir.resolve(datasetName + "_" + SUFFIX_PLAIN_LZMA + ".bin");
+ logBenchWarmupDiscarded("EncodeCompressWriteBinCsv", datasetName, ALGO_LZMA_CSV);
+ benchmarkRawDoubleLzmaWritePath(dValues, binLzma);
+ logBenchProgress("encode", datasetName, ALGO_LZMA_CSV, dValues.length, binLzma);
+ WritePathTimings wLz = benchmarkRawDoubleLzmaWritePath(dValues, binLzma);
+ wLz.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics, datasetName, ALGO_LZMA_CSV, wLz, dValues.length, boundedPrec, multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_LZMA_CSV,
+ srcPath,
+ dValues.length,
+ wLz.encodedUncompressedBytes,
+ 1);
+
+ Path binDict = binsDir.resolve(datasetName + "_" + SUFFIX_DICTIONARY + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_DICTIONARY_CSV, dictValues.length, binDict);
+ WritePathTimings wDc = benchmarkDictionaryWritePath(dictValues, binDict);
+ wDc.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics,
+ datasetName,
+ ALGO_DICTIONARY_CSV,
+ wDc,
+ dictValues.length,
+ boundedPrec,
+ multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_DICTIONARY_CSV,
+ srcPath,
+ dictValues.length,
+ wDc.encodedUncompressedBytes,
+ 0);
+
+ Path binGorilla = binsDir.resolve(datasetName + "_" + SUFFIX_GORILLA + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_GORILLA_CSV, dValues.length, binGorilla);
+ WritePathTimings wGorilla = benchmarkGorillaWritePath(dValues, binGorilla);
+ wGorilla.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics, datasetName, ALGO_GORILLA_CSV, wGorilla, dValues.length, boundedPrec, multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
... 77248 lines suppressed ...