You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/05/19 17:42:43 UTC
(datafusion-comet) branch main updated: chore: remove dead native_iceberg_compat code path (#4363)
This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 0fd52b72d0 chore: remove dead native_iceberg_compat code path (#4363)
0fd52b72d0 is described below
commit 0fd52b72d08cdd0190a41bfc61cabae608e9a83e
Author: Andy Grove <ag...@apache.org>
AuthorDate: Tue May 19 11:42:38 2026 -0600
chore: remove dead native_iceberg_compat code path (#4363)
---
benchmarks/tpc/engines/comet-hashjoin.toml | 1 -
benchmarks/tpc/engines/comet.toml | 1 -
dev/diffs/3.4.3.diff | 91 +-
dev/diffs/3.5.8.diff | 119 +-
dev/diffs/4.0.2.diff | 99 +-
dev/diffs/4.1.1.diff | 99 +-
native/core/Cargo.toml | 12 -
native/core/benches/bit_util.rs | 216 --
native/core/benches/parquet_decode.rs | 56 -
native/core/benches/parquet_read.rs | 216 --
native/core/src/common/bit.rs | 1602 ---------
native/core/src/common/buffer.rs | 366 --
native/core/src/common/mod.rs | 21 -
native/core/src/lib.rs | 2 -
native/core/src/parquet/data_type.rs | 116 -
native/core/src/parquet/mod.rs | 305 +-
native/core/src/parquet/mutable_vector.rs | 243 --
native/core/src/parquet/read/column.rs | 839 -----
native/core/src/parquet/read/levels.rs | 229 --
native/core/src/parquet/read/mod.rs | 108 -
native/core/src/parquet/read/values.rs | 1131 ------
native/core/src/parquet/util/bit_packing.rs | 3658 --------------------
native/core/src/parquet/util/buffer.rs | 128 -
native/core/src/parquet/util/jni.rs | 228 --
native/core/src/parquet/util/memory.rs | 557 ---
native/core/src/parquet/util/mod.rs | 7 -
native/core/src/parquet/util/test_common/mod.rs | 23 -
.../core/src/parquet/util/test_common/page_util.rs | 309 --
.../core/src/parquet/util/test_common/rand_gen.rs | 57 -
.../arrow/c/AbstractCometSchemaImporter.java | 76 -
.../java/org/apache/comet/CometSchemaImporter.java | 32 -
.../src/main/java/org/apache/comet/IcebergApi.java | 44 -
.../apache/comet/parquet/AbstractColumnReader.java | 138 -
.../comet/parquet/ArrowConstantColumnReader.java | 255 --
.../comet/parquet/ArrowRowIndexColumnReader.java | 109 -
.../apache/comet/parquet/BloomFilterReader.java | 253 --
.../apache/comet/parquet/ColumnIndexReader.java | 230 --
.../org/apache/comet/parquet/ColumnPageReader.java | 252 --
.../org/apache/comet/parquet/ColumnReader.java | 314 --
.../org/apache/comet/parquet/CometInputFile.java | 157 -
.../apache/comet/parquet/DictionaryPageReader.java | 190 -
.../java/org/apache/comet/parquet/FileReader.java | 1308 -------
.../org/apache/comet/parquet/FooterReader.java | 56 -
.../parquet/IcebergCometNativeBatchReader.java | 85 -
.../java/org/apache/comet/parquet/IndexFilter.java | 145 -
.../org/apache/comet/parquet/LazyColumnReader.java | 188 -
.../main/java/org/apache/comet/parquet/Native.java | 266 --
.../apache/comet/parquet/NativeBatchReader.java | 1101 ------
.../apache/comet/parquet/NativeColumnReader.java | 159 -
.../apache/comet/parquet/ParquetColumnSpec.java | 112 -
.../comet/parquet/ParquetMetadataSerializer.java | 74 -
.../java/org/apache/comet/parquet/ReadOptions.java | 216 --
.../org/apache/comet/parquet/RowGroupFilter.java | 125 -
.../org/apache/comet/parquet/RowGroupReader.java | 88 -
.../java/org/apache/comet/parquet/TypeUtil.java | 339 --
.../main/java/org/apache/comet/parquet/Utils.java | 525 ---
.../org/apache/comet/parquet/WrappedInputFile.java | 71 -
.../comet/parquet/WrappedSeekableInputStream.java | 64 -
.../org/apache/comet/vector/CometLazyVector.java | 82 -
.../java/org/apache/comet/vector/CometVector.java | 9 -
.../main/scala/org/apache/comet/CometConf.scala | 19 -
.../comet/parquet/CometParquetFileFormat.scala | 260 --
.../comet/parquet/CometReaderThreadPool.scala | 60 -
.../org/apache/comet/parquet/ParquetFilters.scala | 1044 ------
.../apache/comet/parquet/SourceFilterSerde.scala | 179 -
.../org/apache/comet/rules/CometExecRule.scala | 7 +-
.../org/apache/comet/rules/CometScanRule.scala | 100 +-
.../spark/sql/comet/CometNativeScanExec.scala | 48 +-
.../org/apache/spark/sql/comet/CometScanExec.scala | 234 +-
.../apache/comet/parquet/TestCometInputFile.java | 44 -
.../org/apache/comet/parquet/TestFileReader.java | 835 -----
.../java/org/apache/comet/parquet/TestUtils.java | 149 -
.../apache/comet/CometArrayExpressionSuite.scala | 22 +-
.../scala/org/apache/comet/CometCastSuite.scala | 9 +-
.../org/apache/comet/CometCsvExpressionSuite.scala | 4 +-
.../org/apache/comet/CometExpressionSuite.scala | 15 -
.../apache/comet/CometHashExpressionSuite.scala | 12 -
.../scala/org/apache/comet/CometNativeSuite.scala | 13 -
.../org/apache/comet/CometSqlFileTestSuite.scala | 12 -
.../org/apache/comet/exec/CometExecSuite.scala | 11 +-
.../comet/exec/CometNativeColumnarToRowSuite.scala | 4 +-
.../apache/comet/exec/CometNativeReaderSuite.scala | 5 +-
.../comet/exec/CometNativeShuffleSuite.scala | 5 +-
.../apache/comet/exec/CometWindowExecSuite.scala | 3 +-
.../conditional/CometCaseWhenSuite.scala | 14 -
.../conditional/CometCoalesceSuite.scala | 14 -
.../expressions/conditional/CometIfSuite.scala | 14 -
.../comet/objectstore/NativeConfigSuite.scala | 68 +-
.../comet/parquet/CometParquetWriterSuite.scala | 29 +-
.../parquet/ParquetReadFromFakeHadoopFsSuite.scala | 10 +-
.../apache/comet/parquet/ParquetReadSuite.scala | 184 +-
.../parquet/ParquetTimestampLtzAsNtzSuite.scala | 42 +-
.../apache/comet/rules/CometScanRuleSuite.scala | 4 +-
.../benchmark/CometArrayExpressionBenchmark.scala | 14 +-
.../spark/sql/benchmark/CometBenchmarkBase.scala | 6 +-
.../spark/sql/comet/CometTaskMetricsSuite.scala | 17 +-
.../spark/sql/comet/ParquetEncryptionITCase.scala | 27 +-
97 files changed, 315 insertions(+), 21124 deletions(-)
diff --git a/benchmarks/tpc/engines/comet-hashjoin.toml b/benchmarks/tpc/engines/comet-hashjoin.toml
index 1aa4957241..202dcad914 100644
--- a/benchmarks/tpc/engines/comet-hashjoin.toml
+++ b/benchmarks/tpc/engines/comet-hashjoin.toml
@@ -30,6 +30,5 @@ driver_class_path = ["$COMET_JAR"]
"spark.executor.extraClassPath" = "$COMET_JAR"
"spark.plugins" = "org.apache.spark.CometPlugin"
"spark.shuffle.manager" = "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
-"spark.comet.scan.impl" = "native_datafusion"
"spark.comet.exec.replaceSortMergeJoin" = "true"
"spark.comet.expression.Cast.allowIncompatible" = "true"
diff --git a/benchmarks/tpc/engines/comet.toml b/benchmarks/tpc/engines/comet.toml
index 05b2cb22ba..30ffa20d22 100644
--- a/benchmarks/tpc/engines/comet.toml
+++ b/benchmarks/tpc/engines/comet.toml
@@ -30,5 +30,4 @@ driver_class_path = ["$COMET_JAR"]
"spark.executor.extraClassPath" = "$COMET_JAR"
"spark.plugins" = "org.apache.spark.CometPlugin"
"spark.shuffle.manager" = "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager"
-"spark.comet.scan.impl" = "native_datafusion"
"spark.comet.expression.Cast.allowIncompatible" = "true"
diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff
index 79a945add3..7f9ce51ed2 100644
--- a/dev/diffs/3.4.3.diff
+++ b/dev/diffs/3.4.3.diff
@@ -608,10 +608,10 @@ index 2796b1cf154..53dcfde932e 100644
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
new file mode 100644
-index 00000000000..5691536c114
+index 00000000000..4b31bea33de
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
-@@ -0,0 +1,45 @@
+@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
@@ -640,9 +640,6 @@ index 00000000000..5691536c114
+ * Tests with this tag will be ignored when Comet is enabled (e.g., via `ENABLE_COMET`).
+ */
+case class IgnoreComet(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeIcebergCompat(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeDataFusion(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeScan(reason: String) extends Tag("DisableComet")
+
+/**
+ * Helper trait that disables Comet for all tests regardless of default config values.
@@ -1040,7 +1037,7 @@ index 18123a4d6ec..0fe185baa33 100644
test("non-matching optional group") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
-index 75eabcb96f2..7a681f147e4 100644
+index 75eabcb96f2..f8141c28f60 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -21,10 +21,11 @@ import scala.collection.mutable.ArrayBuffer
@@ -1061,7 +1058,7 @@ index 75eabcb96f2..7a681f147e4 100644
_.asInstanceOf[FileScanRDD].filePartitions.forall(
_.files.forall(_.urlEncodedPath.contains("p=0"))))
+ case WholeStageCodegenExec(ColumnarToRowExec(InputAdapter(
-+ fs @ CometScanExec(_, _, _, _, partitionFilters, _, _, _, _, _, _)))) =>
++ fs @ CometScanExec(_, _, _, partitionFilters, _, _, _, _, _, _)))) =>
+ partitionFilters.exists(ExecSubqueryExpression.hasSubquery) &&
+ fs.inputRDDs().forall(
+ _.asInstanceOf[FileScanRDD].filePartitions.forall(
@@ -1969,14 +1966,14 @@ index 07e2849ce6f..3e73645b638 100644
ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 104b4e416cd..4adb273170a 100644
+index 104b4e416cd..835aaa18e39 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -38,6 +38,7 @@ import org.apache.parquet.schema.MessageType
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.sql._
-+import org.apache.spark.sql.{IgnoreCometNativeDataFusion, IgnoreCometNativeScan}
++import org.apache.spark.sql.IgnoreComet
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.optimizer.InferFiltersFromConstraints
@@ -1999,7 +1996,7 @@ index 104b4e416cd..4adb273170a 100644
- test("Filters should be pushed down for vectorized Parquet reader at row group level") {
+ test("Filters should be pushed down for vectorized Parquet reader at row group level",
-+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
++ IgnoreComet("Native scans do not support the tested accumulator")) {
import testImplicits._
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
@@ -2035,7 +2032,7 @@ index 104b4e416cd..4adb273170a 100644
- test("filter pushdown - StringPredicate") {
+ test("filter pushdown - StringPredicate",
-+ IgnoreCometNativeDataFusion("cannot be pushed down")) {
++ IgnoreComet("cannot be pushed down")) {
import testImplicits._
// keep() should take effect on StartsWith/EndsWith/Contains
Seq(
@@ -2045,7 +2042,7 @@ index 104b4e416cd..4adb273170a 100644
- test("SPARK-17091: Convert IN predicate to Parquet filter push-down") {
+ test("SPARK-17091: Convert IN predicate to Parquet filter push-down",
-+ IgnoreCometNativeScan("Comet has different push-down behavior")) {
++ IgnoreComet("Comet has different push-down behavior")) {
val schema = StructType(Seq(
StructField("a", IntegerType, nullable = false)
))
@@ -2090,7 +2087,7 @@ index 104b4e416cd..4adb273170a 100644
- test("SPARK-34562: Bloom filter push down") {
+ test("SPARK-34562: Bloom filter push down",
-+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
++ IgnoreComet("Native scans do not support the tested accumulator")) {
withTempPath { dir =>
val path = dir.getCanonicalPath
spark.range(100).selectExpr("id * 2 AS id")
@@ -2135,14 +2132,14 @@ index 8670d95c65e..b624c3811dd 100644
checkAnswer(
// "fruit" column in this file is encoded using DELTA_LENGTH_BYTE_ARRAY.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-index 29cb224c878..dcb8a0e9bef 100644
+index 29cb224c878..62e3ab96004 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
@@ -27,6 +27,7 @@ import org.apache.parquet.hadoop.ParquetOutputFormat
import org.apache.spark.{DebugFilesystem, SparkConf, SparkException}
import org.apache.spark.sql._
-+import org.apache.spark.sql.IgnoreCometNativeDataFusion
++import org.apache.spark.sql.IgnoreComet
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.expressions.SpecificInternalRow
import org.apache.spark.sql.catalyst.util.ArrayData
@@ -2152,7 +2149,7 @@ index 29cb224c878..dcb8a0e9bef 100644
- test("SPARK-36182: can't read TimestampLTZ as TimestampNTZ") {
+ test("SPARK-36182: can't read TimestampLTZ as TimestampNTZ",
-+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4219")) {
++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4219")) {
val data = (1 to 1000).map { i =>
val ts = new java.sql.Timestamp(i)
Row(ts)
@@ -2162,7 +2159,7 @@ index 29cb224c878..dcb8a0e9bef 100644
- test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") {
+ test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups",
-+ IgnoreCometNativeScan("Native scans had the filter pushed into DF operator, cannot strip")) {
++ IgnoreComet("Native scans had the filter pushed into DF operator, cannot strip")) {
withAllParquetReaders {
withTempPath { path =>
// Repeated values for dictionary encoding.
@@ -2172,7 +2169,7 @@ index 29cb224c878..dcb8a0e9bef 100644
- test("SPARK-34212 Parquet should read decimals correctly") {
+ test("SPARK-34212 Parquet should read decimals correctly",
-+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4354")) {
++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4354")) {
def readParquet(schema: String, path: File): DataFrame = {
spark.read.schema(schema).parquet(path.toString)
}
@@ -2202,7 +2199,7 @@ index 29cb224c878..dcb8a0e9bef 100644
- test("row group skipping doesn't overflow when reading into larger type") {
+ test("row group skipping doesn't overflow when reading into larger type",
-+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4354")) {
++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4354")) {
withTempPath { path =>
Seq(0).toDF("a").write.parquet(path.toString)
// The vectorized and non-vectorized readers will produce different exceptions, we don't need
@@ -2241,24 +2238,21 @@ index 240bb4e6dcb..8287ffa03ca 100644
import testImplicits._
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-index 351c6d698fc..cef6bb08b8c 100644
+index 351c6d698fc..d09a49f0482 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
@@ -26,6 +26,7 @@ import org.apache.parquet.hadoop.{ParquetFileReader, ParquetOutputFormat}
import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE
import org.apache.spark.sql.QueryTest
-+import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec, CometScanExec}
++import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec}
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.FileFormat
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
-@@ -230,6 +231,17 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
+@@ -230,6 +231,14 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
case f: FileSourceScanExec =>
numPartitions += f.inputRDD.partitions.length
numOutputRows += f.metrics("numOutputRows").value
-+ case b: CometScanExec =>
-+ numPartitions += b.inputRDD.partitions.length
-+ numOutputRows += b.metrics("numOutputRows").value
+ case b: CometBatchScanExec =>
+ numPartitions += b.inputRDD.partitions.length
+ numOutputRows += b.metrics("numOutputRows").value
@@ -2291,14 +2285,14 @@ index 5c0b7def039..151184bc98c 100644
assert(fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size,
s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
-index bf5c51b89bb..7e143a0e0f9 100644
+index bf5c51b89bb..f7402b7d883 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
@@ -27,6 +27,7 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
import org.apache.parquet.schema.Type._
import org.apache.spark.SparkException
-+import org.apache.spark.sql.{IgnoreComet, IgnoreCometNativeDataFusion}
++import org.apache.spark.sql.IgnoreComet
import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
import org.apache.spark.sql.functions.desc
@@ -2318,7 +2312,7 @@ index bf5c51b89bb..7e143a0e0f9 100644
- test("schema mismatch failure error message for parquet vectorized reader") {
+ test("schema mismatch failure error message for parquet vectorized reader",
-+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4316")) {
++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4316")) {
withTempPath { dir =>
val e = testSchemaMismatch(dir.getCanonicalPath, vectorizedReaderEnabled = true)
assert(e.getCause.isInstanceOf[SparkException])
@@ -2864,18 +2858,10 @@ index abe606ad9c1..2d930b64cca 100644
val tblTargetName = "tbl_target"
val tblSourceQualified = s"default.$tblSourceName"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-index dd55fcfe42c..cd18a23d4de 100644
+index dd55fcfe42c..d9a3f2df535 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-@@ -27,6 +27,7 @@ import scala.concurrent.duration._
- import scala.language.implicitConversions
- import scala.util.control.NonFatal
-
-+import org.apache.comet.CometConf
- import org.apache.hadoop.fs.Path
- import org.scalactic.source.Position
- import org.scalatest.{BeforeAndAfterAll, Suite, Tag}
-@@ -41,6 +42,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest
+@@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.PlanTestBase
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.util._
@@ -2883,7 +2869,7 @@ index dd55fcfe42c..cd18a23d4de 100644
import org.apache.spark.sql.execution.FilterExec
import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
import org.apache.spark.sql.execution.datasources.DataSourceUtils
-@@ -119,6 +121,34 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
+@@ -119,6 +120,11 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
(implicit pos: Position): Unit = {
@@ -2891,34 +2877,11 @@ index dd55fcfe42c..cd18a23d4de 100644
+ if (isCometEnabled && testTags.exists(_.isInstanceOf[IgnoreComet])) {
+ ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
+ return
-+ }
-+ if (isCometEnabled) {
-+ val cometScanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get(conf)
-+ val isNativeIcebergCompat = cometScanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT ||
-+ cometScanImpl == CometConf.SCAN_AUTO
-+ val isNativeDataFusion = cometScanImpl == CometConf.SCAN_NATIVE_DATAFUSION ||
-+ cometScanImpl == CometConf.SCAN_AUTO
-+ if (isNativeIcebergCompat &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeIcebergCompat])) {
-+ ignore(testName + " (disabled for NATIVE_ICEBERG_COMPAT)", testTags: _*)(testFun)
-+ return
-+ }
-+ if (isNativeDataFusion &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeDataFusion])) {
-+ ignore(testName + " (disabled for NATIVE_DATAFUSION)", testTags: _*)(testFun)
-+ return
-+ }
-+ if ((isNativeDataFusion || isNativeIcebergCompat) &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeScan])) {
-+ ignore(testName + " (disabled for NATIVE_DATAFUSION and NATIVE_ICEBERG_COMPAT)",
-+ testTags: _*)(testFun)
-+ return
-+ }
+ }
if (testTags.exists(_.isInstanceOf[DisableAdaptiveExecution])) {
super.test(testName, testTags: _*) {
withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-@@ -242,6 +272,11 @@ private[sql] trait SQLTestUtilsBase
+@@ -242,6 +248,11 @@ private[sql] trait SQLTestUtilsBase
protected override def _sqlContext: SQLContext = self.spark.sqlContext
}
@@ -2930,7 +2893,7 @@ index dd55fcfe42c..cd18a23d4de 100644
protected override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = {
SparkSession.setActiveSession(spark)
super.withSQLConf(pairs: _*)(f)
-@@ -434,6 +469,8 @@ private[sql] trait SQLTestUtilsBase
+@@ -434,6 +445,8 @@ private[sql] trait SQLTestUtilsBase
val schema = df.schema
val withoutFilters = df.queryExecution.executedPlan.transform {
case FilterExec(_, child) => child
diff --git a/dev/diffs/3.5.8.diff b/dev/diffs/3.5.8.diff
index a72e44fc4f..a86fa2a914 100644
--- a/dev/diffs/3.5.8.diff
+++ b/dev/diffs/3.5.8.diff
@@ -503,18 +503,10 @@ index a206e97c353..fea1149b67d 100644
test("SPARK-35884: Explain Formatted") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
-index 93275487f29..78150c9163e 100644
+index 93275487f29..a5208b8d54b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
-@@ -23,6 +23,7 @@ import java.nio.file.{Files, StandardOpenOption}
-
- import scala.collection.mutable
-
-+import org.apache.comet.CometConf
- import org.apache.hadoop.conf.Configuration
- import org.apache.hadoop.fs.{LocalFileSystem, Path}
-
-@@ -33,6 +34,7 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GreaterTha
+@@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GreaterTha
import org.apache.spark.sql.catalyst.expressions.IntegralLiteralTestUtils.{negativeInt, positiveInt}
import org.apache.spark.sql.catalyst.plans.logical.Filter
import org.apache.spark.sql.catalyst.types.DataTypeUtils
@@ -522,20 +514,16 @@ index 93275487f29..78150c9163e 100644
import org.apache.spark.sql.execution.{FileSourceScanLike, SimpleMode}
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
import org.apache.spark.sql.execution.datasources.FilePartition
-@@ -250,6 +252,12 @@ class FileBasedDataSourceSuite extends QueryTest
+@@ -250,6 +251,8 @@ class FileBasedDataSourceSuite extends QueryTest
case "" => "_LEGACY_ERROR_TEMP_2062"
case _ => "_LEGACY_ERROR_TEMP_2055"
}
-+ // native_datafusion Parquet scan cannot throw
-+ // a SparkFileNotFoundException
-+ assume(!Seq(
-+ CometConf.SCAN_NATIVE_DATAFUSION,
-+ CometConf.SCAN_AUTO
-+ ).contains(CometConf.COMET_NATIVE_SCAN_IMPL.get()))
++ // native_datafusion Parquet scan cannot throw a SparkFileNotFoundException
++ assume(false)
checkErrorMatchPVals(
exception = intercept[SparkException] {
testIgnoreMissingFiles(options)
-@@ -656,18 +664,25 @@ class FileBasedDataSourceSuite extends QueryTest
+@@ -656,18 +659,25 @@ class FileBasedDataSourceSuite extends QueryTest
checkAnswer(sql(s"select A from $tableName"), data.select("A"))
// RuntimeException is triggered at executor side, which is then wrapped as
@@ -568,7 +556,7 @@ index 93275487f29..78150c9163e 100644
errorClass = "_LEGACY_ERROR_TEMP_2093",
parameters = Map("requiredFieldName" -> "b", "matchedOrcFields" -> "[b, B]")
)
-@@ -955,6 +970,7 @@ class FileBasedDataSourceSuite extends QueryTest
+@@ -955,6 +965,7 @@ class FileBasedDataSourceSuite extends QueryTest
assert(bJoinExec.isEmpty)
val smJoinExec = collect(joinedDF.queryExecution.executedPlan) {
case smJoin: SortMergeJoinExec => smJoin
@@ -576,7 +564,7 @@ index 93275487f29..78150c9163e 100644
}
assert(smJoinExec.nonEmpty)
}
-@@ -1015,6 +1031,7 @@ class FileBasedDataSourceSuite extends QueryTest
+@@ -1015,6 +1026,7 @@ class FileBasedDataSourceSuite extends QueryTest
val fileScan = df.queryExecution.executedPlan collectFirst {
case BatchScanExec(_, f: FileScan, _, _, _, _) => f
@@ -584,7 +572,7 @@ index 93275487f29..78150c9163e 100644
}
assert(fileScan.nonEmpty)
assert(fileScan.get.partitionFilters.nonEmpty)
-@@ -1056,6 +1073,7 @@ class FileBasedDataSourceSuite extends QueryTest
+@@ -1056,6 +1068,7 @@ class FileBasedDataSourceSuite extends QueryTest
val fileScan = df.queryExecution.executedPlan collectFirst {
case BatchScanExec(_, f: FileScan, _, _, _, _) => f
@@ -592,7 +580,7 @@ index 93275487f29..78150c9163e 100644
}
assert(fileScan.nonEmpty)
assert(fileScan.get.partitionFilters.isEmpty)
-@@ -1240,6 +1258,9 @@ class FileBasedDataSourceSuite extends QueryTest
+@@ -1240,6 +1253,9 @@ class FileBasedDataSourceSuite extends QueryTest
val filters = df.queryExecution.executedPlan.collect {
case f: FileSourceScanLike => f.dataFilters
case b: BatchScanExec => b.scan.asInstanceOf[FileScan].dataFilters
@@ -604,10 +592,10 @@ index 93275487f29..78150c9163e 100644
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
new file mode 100644
-index 00000000000..1ee842b6f62
+index 00000000000..a42533c7c84
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
-@@ -0,0 +1,45 @@
+@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
@@ -636,9 +624,6 @@ index 00000000000..1ee842b6f62
+ * Tests with this tag will be ignored when Comet is enabled (e.g., via `ENABLE_COMET`).
+ */
+case class IgnoreComet(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeIcebergCompat(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeDataFusion(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeScan(reason: String) extends Tag("DisableComet")
+
+/**
+ * Helper trait that disables Comet for all tests regardless of default config values.
@@ -992,7 +977,7 @@ index 8b4ac474f87..3f79f20822f 100644
extensions.injectColumnar(session =>
MyColumnarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())) }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
-index 04702201f82..a4b5c6c93ce 100644
+index 04702201f82..c5ab5443ff7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -22,10 +22,11 @@ import scala.collection.mutable.ArrayBuffer
@@ -1013,7 +998,7 @@ index 04702201f82..a4b5c6c93ce 100644
_.asInstanceOf[FileScanRDD].filePartitions.forall(
_.files.forall(_.urlEncodedPath.contains("p=0"))))
+ case WholeStageCodegenExec(ColumnarToRowExec(InputAdapter(
-+ fs @ CometScanExec(_, _, _, _, partitionFilters, _, _, _, _, _, _)))) =>
++ fs @ CometScanExec(_, _, _, partitionFilters, _, _, _, _, _, _)))) =>
+ partitionFilters.exists(ExecSubqueryExpression.hasSubquery) &&
+ fs.inputRDDs().forall(
+ _.asInstanceOf[FileScanRDD].filePartitions.forall(
@@ -1958,7 +1943,7 @@ index 07e2849ce6f..3e73645b638 100644
ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 8e88049f51e..097c518a19a 100644
+index 8e88049f51e..f8e2194a8ac 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -1095,7 +1095,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
@@ -1980,7 +1965,7 @@ index 8e88049f51e..097c518a19a 100644
- test("Filters should be pushed down for vectorized Parquet reader at row group level") {
+ test("Filters should be pushed down for vectorized Parquet reader at row group level",
-+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
++ IgnoreComet("Native scans do not support the tested accumulator")) {
import testImplicits._
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
@@ -2015,7 +2000,7 @@ index 8e88049f51e..097c518a19a 100644
}
- test("filter pushdown - StringPredicate") {
-+ test("filter pushdown - StringPredicate", IgnoreCometNativeScan("cannot be pushed down")) {
++ test("filter pushdown - StringPredicate", IgnoreComet("cannot be pushed down")) {
import testImplicits._
// keep() should take effect on StartsWith/EndsWith/Contains
Seq(
@@ -2025,7 +2010,7 @@ index 8e88049f51e..097c518a19a 100644
- test("SPARK-17091: Convert IN predicate to Parquet filter push-down") {
+ test("SPARK-17091: Convert IN predicate to Parquet filter push-down",
-+ IgnoreCometNativeScan("Comet has different push-down behavior")) {
++ IgnoreComet("Comet has different push-down behavior")) {
val schema = StructType(Seq(
StructField("a", IntegerType, nullable = false)
))
@@ -2073,7 +2058,7 @@ index 8e88049f51e..097c518a19a 100644
- test("SPARK-34562: Bloom filter push down") {
+ test("SPARK-34562: Bloom filter push down",
-+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
++ IgnoreComet("Native scans do not support the tested accumulator")) {
withTempPath { dir =>
val path = dir.getCanonicalPath
spark.range(100).selectExpr("id * 2 AS id")
@@ -2118,7 +2103,7 @@ index 8ed9ef1630e..eed2a6f5ad5 100644
checkAnswer(
// "fruit" column in this file is encoded using DELTA_LENGTH_BYTE_ARRAY.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-index f6472ba3d9d..b62ff2975e6 100644
+index f6472ba3d9d..e8ccb8dec54 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
@@ -185,7 +185,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
@@ -2127,7 +2112,7 @@ index f6472ba3d9d..b62ff2975e6 100644
- test("SPARK-36182: can't read TimestampLTZ as TimestampNTZ") {
+ test("SPARK-36182: can't read TimestampLTZ as TimestampNTZ",
-+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4219")) {
++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4219")) {
val data = (1 to 1000).map { i =>
val ts = new java.sql.Timestamp(i)
Row(ts)
@@ -2137,7 +2122,7 @@ index f6472ba3d9d..b62ff2975e6 100644
- test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") {
+ test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups",
-+ IgnoreCometNativeScan("Native scans had the filter pushed into DF operator, cannot strip")) {
++ IgnoreComet("Native scans had the filter pushed into DF operator, cannot strip")) {
withAllParquetReaders {
withTempPath { path =>
// Repeated values for dictionary encoding.
@@ -2147,7 +2132,7 @@ index f6472ba3d9d..b62ff2975e6 100644
- test("SPARK-34212 Parquet should read decimals correctly") {
+ test("SPARK-34212 Parquet should read decimals correctly",
-+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4354")) {
++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4354")) {
def readParquet(schema: String, path: File): DataFrame = {
spark.read.schema(schema).parquet(path.toString)
}
@@ -2177,7 +2162,7 @@ index f6472ba3d9d..b62ff2975e6 100644
- test("row group skipping doesn't overflow when reading into larger type") {
+ test("row group skipping doesn't overflow when reading into larger type",
-+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4354")) {
++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4354")) {
withTempPath { path =>
Seq(0).toDF("a").write.parquet(path.toString)
// The vectorized and non-vectorized readers will produce different exceptions, we don't need
@@ -2216,24 +2201,21 @@ index 4f906411345..6cc69f7e915 100644
import testImplicits._
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-index 27c2a2148fd..808baf9e778 100644
+index 27c2a2148fd..b3d753b9824 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
@@ -26,6 +26,7 @@ import org.apache.parquet.hadoop.{ParquetFileReader, ParquetOutputFormat}
import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE
import org.apache.spark.sql.QueryTest
-+import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec, CometScanExec}
++import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec}
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.FileFormat
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
-@@ -243,6 +244,17 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
+@@ -243,6 +244,14 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
case f: FileSourceScanExec =>
numPartitions += f.inputRDD.partitions.length
numOutputRows += f.metrics("numOutputRows").value
-+ case b: CometScanExec =>
-+ numPartitions += b.inputRDD.partitions.length
-+ numOutputRows += b.metrics("numOutputRows").value
+ case b: CometBatchScanExec =>
+ numPartitions += b.inputRDD.partitions.length
+ numOutputRows += b.metrics("numOutputRows").value
@@ -2266,14 +2248,14 @@ index 5c0b7def039..151184bc98c 100644
assert(fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size,
s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
-index 3f47c5e506f..80b7ef6c46a 100644
+index 3f47c5e506f..8e8d2a1634d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
@@ -27,6 +27,7 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
import org.apache.parquet.schema.Type._
import org.apache.spark.SparkException
-+import org.apache.spark.sql.{IgnoreComet, IgnoreCometNativeDataFusion}
++import org.apache.spark.sql.IgnoreComet
import org.apache.spark.sql.catalyst.expressions.Cast.toSQLType
import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
import org.apache.spark.sql.functions.desc
@@ -2293,7 +2275,7 @@ index 3f47c5e506f..80b7ef6c46a 100644
- test("schema mismatch failure error message for parquet vectorized reader") {
+ test("schema mismatch failure error message for parquet vectorized reader",
-+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4316")) {
++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4316")) {
withTempPath { dir =>
val e = testSchemaMismatch(dir.getCanonicalPath, vectorizedReaderEnabled = true)
assert(e.getCause.isInstanceOf[SparkException])
@@ -2824,18 +2806,10 @@ index abe606ad9c1..2d930b64cca 100644
val tblTargetName = "tbl_target"
val tblSourceQualified = s"default.$tblSourceName"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-index e937173a590..3134078a122 100644
+index e937173a590..263934fbe7b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-@@ -27,6 +27,7 @@ import scala.concurrent.duration._
- import scala.language.implicitConversions
- import scala.util.control.NonFatal
-
-+import org.apache.comet.CometConf
- import org.apache.hadoop.fs.Path
- import org.scalactic.source.Position
- import org.scalatest.{BeforeAndAfterAll, Suite, Tag}
-@@ -41,6 +42,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest
+@@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.PlanTestBase
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.util._
@@ -2843,7 +2817,7 @@ index e937173a590..3134078a122 100644
import org.apache.spark.sql.execution.FilterExec
import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
import org.apache.spark.sql.execution.datasources.DataSourceUtils
-@@ -119,6 +121,34 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
+@@ -119,6 +120,11 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
(implicit pos: Position): Unit = {
@@ -2851,34 +2825,11 @@ index e937173a590..3134078a122 100644
+ if (isCometEnabled && testTags.exists(_.isInstanceOf[IgnoreComet])) {
+ ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
+ return
-+ }
-+ if (isCometEnabled) {
-+ val cometScanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get(conf)
-+ val isNativeIcebergCompat = cometScanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT ||
-+ cometScanImpl == CometConf.SCAN_AUTO
-+ val isNativeDataFusion = cometScanImpl == CometConf.SCAN_NATIVE_DATAFUSION ||
-+ cometScanImpl == CometConf.SCAN_AUTO
-+ if (isNativeIcebergCompat &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeIcebergCompat])) {
-+ ignore(testName + " (disabled for NATIVE_ICEBERG_COMPAT)", testTags: _*)(testFun)
-+ return
-+ }
-+ if (isNativeDataFusion &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeDataFusion])) {
-+ ignore(testName + " (disabled for NATIVE_DATAFUSION)", testTags: _*)(testFun)
-+ return
-+ }
-+ if ((isNativeDataFusion || isNativeIcebergCompat) &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeScan])) {
-+ ignore(testName + " (disabled for NATIVE_DATAFUSION and NATIVE_ICEBERG_COMPAT)",
-+ testTags: _*)(testFun)
-+ return
-+ }
+ }
if (testTags.exists(_.isInstanceOf[DisableAdaptiveExecution])) {
super.test(testName, testTags: _*) {
withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-@@ -242,6 +272,11 @@ private[sql] trait SQLTestUtilsBase
+@@ -242,6 +248,11 @@ private[sql] trait SQLTestUtilsBase
protected override def _sqlContext: SQLContext = self.spark.sqlContext
}
@@ -2890,7 +2841,7 @@ index e937173a590..3134078a122 100644
protected override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = {
SparkSession.setActiveSession(spark)
super.withSQLConf(pairs: _*)(f)
-@@ -435,6 +470,8 @@ private[sql] trait SQLTestUtilsBase
+@@ -435,6 +446,8 @@ private[sql] trait SQLTestUtilsBase
val schema = df.schema
val withoutFilters = df.queryExecution.executedPlan.transform {
case FilterExec(_, child) => child
diff --git a/dev/diffs/4.0.2.diff b/dev/diffs/4.0.2.diff
index 2614abc979..2d0d5cca35 100644
--- a/dev/diffs/4.0.2.diff
+++ b/dev/diffs/4.0.2.diff
@@ -739,10 +739,10 @@ index 9c529d14221..ab2850b5d68 100644
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
new file mode 100644
-index 00000000000..5691536c114
+index 00000000000..4b31bea33de
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
-@@ -0,0 +1,45 @@
+@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
@@ -771,9 +771,6 @@ index 00000000000..5691536c114
+ * Tests with this tag will be ignored when Comet is enabled (e.g., via `ENABLE_COMET`).
+ */
+case class IgnoreComet(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeIcebergCompat(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeDataFusion(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeScan(reason: String) extends Tag("DisableComet")
+
+/**
+ * Helper trait that disables Comet for all tests regardless of default config values.
@@ -1199,7 +1196,7 @@ index 0df7f806272..92390bd819f 100644
test("non-matching optional group") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
-index 2e33f6505ab..54f5081e10a 100644
+index 2e33f6505ab..fc1a2c8f964 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -23,10 +23,11 @@ import org.apache.spark.SparkRuntimeException
@@ -1220,7 +1217,7 @@ index 2e33f6505ab..54f5081e10a 100644
_.asInstanceOf[FileScanRDD].filePartitions.forall(
_.files.forall(_.urlEncodedPath.contains("p=0"))))
+ case WholeStageCodegenExec(ColumnarToRowExec(InputAdapter(
-+ fs @ CometScanExec(_, _, _, _, partitionFilters, _, _, _, _, _, _)))) =>
++ fs @ CometScanExec(_, _, _, partitionFilters, _, _, _, _, _, _)))) =>
+ partitionFilters.exists(ExecSubqueryExpression.hasSubquery) &&
+ fs.inputRDDs().forall(
+ _.asInstanceOf[FileScanRDD].filePartitions.forall(
@@ -2544,14 +2541,14 @@ index cd6f41b4ef4..4b6a17344bc 100644
ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 6080a5e8e4b..f5dadef89ae 100644
+index 6080a5e8e4b..23a451d5bcf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -38,6 +38,7 @@ import org.apache.parquet.schema.MessageType
import org.apache.spark.{SparkConf, SparkException, SparkRuntimeException}
import org.apache.spark.sql._
-+import org.apache.spark.sql.IgnoreCometNativeScan
++import org.apache.spark.sql.IgnoreComet
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.optimizer.InferFiltersFromConstraints
@@ -2574,7 +2571,7 @@ index 6080a5e8e4b..f5dadef89ae 100644
- test("Filters should be pushed down for vectorized Parquet reader at row group level") {
+ test("Filters should be pushed down for vectorized Parquet reader at row group level",
-+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
++ IgnoreComet("Native scans do not support the tested accumulator")) {
import testImplicits._
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
@@ -2609,7 +2606,7 @@ index 6080a5e8e4b..f5dadef89ae 100644
}
- test("filter pushdown - StringPredicate") {
-+ test("filter pushdown - StringPredicate", IgnoreCometNativeScan("cannot be pushed down")) {
++ test("filter pushdown - StringPredicate", IgnoreComet("cannot be pushed down")) {
import testImplicits._
// keep() should take effect on StartsWith/EndsWith/Contains
Seq(
@@ -2619,7 +2616,7 @@ index 6080a5e8e4b..f5dadef89ae 100644
- test("SPARK-17091: Convert IN predicate to Parquet filter push-down") {
+ test("SPARK-17091: Convert IN predicate to Parquet filter push-down",
-+ IgnoreCometNativeScan("Comet has different push-down behavior")) {
++ IgnoreComet("Comet has different push-down behavior")) {
val schema = StructType(Seq(
StructField("a", IntegerType, nullable = false)
))
@@ -2666,7 +2663,7 @@ index 6080a5e8e4b..f5dadef89ae 100644
- test("SPARK-34562: Bloom filter push down") {
+ test("SPARK-34562: Bloom filter push down",
-+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
++ IgnoreComet("Native scans do not support the tested accumulator")) {
withTempPath { dir =>
val path = dir.getCanonicalPath
spark.range(100).selectExpr("id * 2 AS id")
@@ -2773,32 +2770,21 @@ index 30503af0fab..1491f4bc2d5 100644
import testImplicits._
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-index 08fd8a9ecb5..27aee839b8c 100644
+index 08fd8a9ecb5..306958da489 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-@@ -20,6 +20,7 @@ import java.io.File
-
- import scala.jdk.CollectionConverters._
-
-+import org.apache.comet.CometConf
- import org.apache.hadoop.fs.Path
- import org.apache.parquet.column.ParquetProperties._
- import org.apache.parquet.hadoop.{ParquetFileReader, ParquetOutputFormat}
-@@ -27,6 +28,7 @@ import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE
+@@ -27,6 +27,7 @@ import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE
import org.apache.spark.SparkException
import org.apache.spark.sql.QueryTest
-+import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec, CometScanExec}
++import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec}
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.FileFormat
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
-@@ -245,6 +247,17 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
+@@ -245,6 +246,14 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
case f: FileSourceScanExec =>
numPartitions += f.inputRDD.partitions.length
numOutputRows += f.metrics("numOutputRows").value
-+ case b: CometScanExec =>
-+ numPartitions += b.inputRDD.partitions.length
-+ numOutputRows += b.metrics("numOutputRows").value
+ case b: CometBatchScanExec =>
+ numPartitions += b.inputRDD.partitions.length
+ numOutputRows += b.metrics("numOutputRows").value
@@ -2810,16 +2796,13 @@ index 08fd8a9ecb5..27aee839b8c 100644
case _ =>
}
assert(numPartitions > 0)
-@@ -303,6 +316,12 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
+@@ -303,6 +312,9 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
val conf = RowIndexTestConf(useDataSourceV2 = useDataSourceV2)
test(s"invalid row index column type - ${conf.desc}") {
+ // https://github.com/apache/datafusion-comet/issues/3886
+ // Comet throws RuntimeException instead of SparkException
-+ assume(!Seq(
-+ CometConf.SCAN_NATIVE_DATAFUSION,
-+ CometConf.SCAN_AUTO
-+ ).contains(CometConf.COMET_NATIVE_SCAN_IMPL.get()))
++ assume(false)
withSQLConf(conf.sqlConfs: _*) {
withTempPath{ path =>
val df = spark.range(0, 10, 1, 1).toDF("id")
@@ -2844,7 +2827,7 @@ index 5c0b7def039..151184bc98c 100644
assert(fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size,
s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
-index 0acb21f3e6f..e7c65429119 100644
+index 0acb21f3e6f..15bd866d8aa 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
@@ -27,7 +27,7 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
@@ -2852,7 +2835,7 @@ index 0acb21f3e6f..e7c65429119 100644
import org.apache.spark.SparkException
-import org.apache.spark.sql.{AnalysisException, Row}
-+import org.apache.spark.sql.{AnalysisException, IgnoreComet, IgnoreCometNativeDataFusion, Row}
++import org.apache.spark.sql.{AnalysisException, IgnoreComet, Row}
import org.apache.spark.sql.catalyst.expressions.Cast.toSQLType
import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
import org.apache.spark.sql.functions.desc
@@ -2872,7 +2855,7 @@ index 0acb21f3e6f..e7c65429119 100644
- test("schema mismatch failure error message for parquet vectorized reader") {
+ test("schema mismatch failure error message for parquet vectorized reader",
-+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4316")) {
++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4316")) {
withTempPath { dir =>
val e = testSchemaMismatch(dir.getCanonicalPath, vectorizedReaderEnabled = true)
assert(e.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException])
@@ -3460,26 +3443,19 @@ index 86c4e49f6f6..2e639e5f38d 100644
val tblTargetName = "tbl_target"
val tblSourceQualified = s"default.$tblSourceName"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-index f0f3f94b811..be5e113c3ed 100644
+index f0f3f94b811..b7d18771314 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-@@ -27,13 +27,14 @@ import scala.jdk.CollectionConverters._
- import scala.language.implicitConversions
- import scala.util.control.NonFatal
-
-+import org.apache.comet.CometConf
- import org.apache.hadoop.fs.Path
- import org.scalactic.source.Position
- import org.scalatest.{BeforeAndAfterAll, Suite, Tag}
+@@ -33,7 +33,7 @@ import org.scalatest.{BeforeAndAfterAll, Suite, Tag}
import org.scalatest.concurrent.Eventually
import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.{AnalysisException, Row}
-+import org.apache.spark.sql.{AnalysisException, IgnoreComet, IgnoreCometNativeDataFusion, IgnoreCometNativeIcebergCompat, IgnoreCometNativeScan, Row}
++import org.apache.spark.sql.{AnalysisException, IgnoreComet, Row}
import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.catalyst.catalog.SessionCatalog.DEFAULT_DATABASE
-@@ -42,6 +43,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTestBase
+@@ -42,6 +42,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTestBase
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.classic.{ClassicConversions, ColumnConversions, ColumnNodeToExpressionConverter, DataFrame, Dataset, SparkSession, SQLImplicits}
@@ -3487,7 +3463,7 @@ index f0f3f94b811..be5e113c3ed 100644
import org.apache.spark.sql.execution.FilterExec
import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
import org.apache.spark.sql.execution.datasources.DataSourceUtils
-@@ -121,6 +123,34 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
+@@ -121,6 +122,11 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
(implicit pos: Position): Unit = {
@@ -3495,34 +3471,11 @@ index f0f3f94b811..be5e113c3ed 100644
+ if (isCometEnabled && testTags.exists(_.isInstanceOf[IgnoreComet])) {
+ ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
+ return
-+ }
-+ if (isCometEnabled) {
-+ val cometScanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get(conf)
-+ val isNativeIcebergCompat = cometScanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT ||
-+ cometScanImpl == CometConf.SCAN_AUTO
-+ val isNativeDataFusion = cometScanImpl == CometConf.SCAN_NATIVE_DATAFUSION ||
-+ cometScanImpl == CometConf.SCAN_AUTO
-+ if (isNativeIcebergCompat &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeIcebergCompat])) {
-+ ignore(testName + " (disabled for NATIVE_ICEBERG_COMPAT)", testTags: _*)(testFun)
-+ return
-+ }
-+ if (isNativeDataFusion &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeDataFusion])) {
-+ ignore(testName + " (disabled for NATIVE_DATAFUSION)", testTags: _*)(testFun)
-+ return
-+ }
-+ if ((isNativeDataFusion || isNativeIcebergCompat) &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeScan])) {
-+ ignore(testName + " (disabled for NATIVE_DATAFUSION and NATIVE_ICEBERG_COMPAT)",
-+ testTags: _*)(testFun)
-+ return
-+ }
+ }
if (testTags.exists(_.isInstanceOf[DisableAdaptiveExecution])) {
super.test(testName, testTags: _*) {
withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-@@ -248,8 +278,15 @@ private[sql] trait SQLTestUtilsBase
+@@ -248,8 +254,15 @@ private[sql] trait SQLTestUtilsBase
override protected def converter: ColumnNodeToExpressionConverter = self.spark.converter
}
@@ -3538,7 +3491,7 @@ index f0f3f94b811..be5e113c3ed 100644
super.withSQLConf(pairs: _*)(f)
}
-@@ -451,6 +488,8 @@ private[sql] trait SQLTestUtilsBase
+@@ -451,6 +464,8 @@ private[sql] trait SQLTestUtilsBase
val schema = df.schema
val withoutFilters = df.queryExecution.executedPlan.transform {
case FilterExec(_, child) => child
diff --git a/dev/diffs/4.1.1.diff b/dev/diffs/4.1.1.diff
index 2ed8a5a32f..960e421da9 100644
--- a/dev/diffs/4.1.1.diff
+++ b/dev/diffs/4.1.1.diff
@@ -800,10 +800,10 @@ index 95e86fe4311..fb2b6363af6 100644
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
new file mode 100644
-index 00000000000..5691536c114
+index 00000000000..4b31bea33de
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/IgnoreComet.scala
-@@ -0,0 +1,45 @@
+@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
@@ -832,9 +832,6 @@ index 00000000000..5691536c114
+ * Tests with this tag will be ignored when Comet is enabled (e.g., via `ENABLE_COMET`).
+ */
+case class IgnoreComet(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeIcebergCompat(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeDataFusion(reason: String) extends Tag("DisableComet")
-+case class IgnoreCometNativeScan(reason: String) extends Tag("DisableComet")
+
+/**
+ * Helper trait that disables Comet for all tests regardless of default config values.
@@ -1292,7 +1289,7 @@ index ff0ee19ae97..01958e0c45b 100644
test("non-matching optional group") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
-index 3ba48da0e32..401a2851b66 100644
+index 3ba48da0e32..a33e65d4420 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -23,10 +23,11 @@ import org.apache.spark.SparkRuntimeException
@@ -1313,7 +1310,7 @@ index 3ba48da0e32..401a2851b66 100644
_.asInstanceOf[FileScanRDD].filePartitions.forall(
_.files.forall(_.urlEncodedPath.contains("p=0"))))
+ case WholeStageCodegenExec(ColumnarToRowExec(InputAdapter(
-+ fs @ CometScanExec(_, _, _, _, partitionFilters, _, _, _, _, _, _)))) =>
++ fs @ CometScanExec(_, _, _, partitionFilters, _, _, _, _, _, _)))) =>
+ partitionFilters.exists(ExecSubqueryExpression.hasSubquery) &&
+ fs.inputRDDs().forall(
+ _.asInstanceOf[FileScanRDD].filePartitions.forall(
@@ -2685,14 +2682,14 @@ index cd6f41b4ef4..4b6a17344bc 100644
ParquetOutputFormat.WRITER_VERSION -> ParquetProperties.WriterVersion.PARQUET_2_0.toString
)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 6b73cc8618d..81a58f43784 100644
+index 6b73cc8618d..e67aaeff9df 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -38,6 +38,7 @@ import org.apache.parquet.schema.MessageType
import org.apache.spark.{SparkConf, SparkException, SparkRuntimeException}
import org.apache.spark.sql._
-+import org.apache.spark.sql.IgnoreCometNativeScan
++import org.apache.spark.sql.IgnoreComet
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.optimizer.InferFiltersFromConstraints
@@ -2715,7 +2712,7 @@ index 6b73cc8618d..81a58f43784 100644
- test("Filters should be pushed down for vectorized Parquet reader at row group level") {
+ test("Filters should be pushed down for vectorized Parquet reader at row group level",
-+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
++ IgnoreComet("Native scans do not support the tested accumulator")) {
import testImplicits._
withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
@@ -2750,7 +2747,7 @@ index 6b73cc8618d..81a58f43784 100644
}
- test("filter pushdown - StringPredicate") {
-+ test("filter pushdown - StringPredicate", IgnoreCometNativeScan("cannot be pushed down")) {
++ test("filter pushdown - StringPredicate", IgnoreComet("cannot be pushed down")) {
import testImplicits._
// keep() should take effect on StartsWith/EndsWith/Contains
Seq(
@@ -2760,7 +2757,7 @@ index 6b73cc8618d..81a58f43784 100644
- test("SPARK-17091: Convert IN predicate to Parquet filter push-down") {
+ test("SPARK-17091: Convert IN predicate to Parquet filter push-down",
-+ IgnoreCometNativeScan("Comet has different push-down behavior")) {
++ IgnoreComet("Comet has different push-down behavior")) {
val schema = StructType(Seq(
StructField("a", IntegerType, nullable = false)
))
@@ -2807,7 +2804,7 @@ index 6b73cc8618d..81a58f43784 100644
- test("SPARK-34562: Bloom filter push down") {
+ test("SPARK-34562: Bloom filter push down",
-+ IgnoreCometNativeScan("Native scans do not support the tested accumulator")) {
++ IgnoreComet("Native scans do not support the tested accumulator")) {
withTempPath { dir =>
val path = dir.getCanonicalPath
spark.range(100).selectExpr("id * 2 AS id")
@@ -2932,32 +2929,21 @@ index 30503af0fab..1491f4bc2d5 100644
import testImplicits._
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-index 08fd8a9ecb5..27aee839b8c 100644
+index 08fd8a9ecb5..306958da489 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowIndexSuite.scala
-@@ -20,6 +20,7 @@ import java.io.File
-
- import scala.jdk.CollectionConverters._
-
-+import org.apache.comet.CometConf
- import org.apache.hadoop.fs.Path
- import org.apache.parquet.column.ParquetProperties._
- import org.apache.parquet.hadoop.{ParquetFileReader, ParquetOutputFormat}
-@@ -27,6 +28,7 @@ import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE
+@@ -27,6 +27,7 @@ import org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE
import org.apache.spark.SparkException
import org.apache.spark.sql.QueryTest
-+import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec, CometScanExec}
++import org.apache.spark.sql.comet.{CometBatchScanExec, CometNativeScanExec}
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.FileFormat
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
-@@ -245,6 +247,17 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
+@@ -245,6 +246,14 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
case f: FileSourceScanExec =>
numPartitions += f.inputRDD.partitions.length
numOutputRows += f.metrics("numOutputRows").value
-+ case b: CometScanExec =>
-+ numPartitions += b.inputRDD.partitions.length
-+ numOutputRows += b.metrics("numOutputRows").value
+ case b: CometBatchScanExec =>
+ numPartitions += b.inputRDD.partitions.length
+ numOutputRows += b.metrics("numOutputRows").value
@@ -2969,16 +2955,13 @@ index 08fd8a9ecb5..27aee839b8c 100644
case _ =>
}
assert(numPartitions > 0)
-@@ -303,6 +316,12 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
+@@ -303,6 +312,9 @@ class ParquetRowIndexSuite extends QueryTest with SharedSparkSession {
val conf = RowIndexTestConf(useDataSourceV2 = useDataSourceV2)
test(s"invalid row index column type - ${conf.desc}") {
+ // https://github.com/apache/datafusion-comet/issues/3886
+ // Comet throws RuntimeException instead of SparkException
-+ assume(!Seq(
-+ CometConf.SCAN_NATIVE_DATAFUSION,
-+ CometConf.SCAN_AUTO
-+ ).contains(CometConf.COMET_NATIVE_SCAN_IMPL.get()))
++ assume(false)
withSQLConf(conf.sqlConfs: _*) {
withTempPath{ path =>
val df = spark.range(0, 10, 1, 1).toDF("id")
@@ -3003,7 +2986,7 @@ index 5c0b7def039..151184bc98c 100644
assert(fileSourceScanSchemata.size === expectedSchemaCatalogStrings.size,
s"Found ${fileSourceScanSchemata.size} file sources in dataframe, " +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
-index 56076175d60..5872d9962cc 100644
+index 56076175d60..a3d47b24634 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala
@@ -27,7 +27,7 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
@@ -3011,7 +2994,7 @@ index 56076175d60..5872d9962cc 100644
import org.apache.spark.SparkException
-import org.apache.spark.sql.{AnalysisException, Row}
-+import org.apache.spark.sql.{AnalysisException, IgnoreComet, IgnoreCometNativeDataFusion, Row}
++import org.apache.spark.sql.{AnalysisException, IgnoreComet, Row}
import org.apache.spark.sql.catalyst.expressions.Cast.toSQLType
import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException
import org.apache.spark.sql.functions.desc
@@ -3031,7 +3014,7 @@ index 56076175d60..5872d9962cc 100644
- test("schema mismatch failure error message for parquet vectorized reader") {
+ test("schema mismatch failure error message for parquet vectorized reader",
-+ IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4316")) {
++ IgnoreComet("https://github.com/apache/datafusion-comet/issues/4316")) {
withTempPath { dir =>
val e = testSchemaMismatch(dir.getCanonicalPath, vectorizedReaderEnabled = true)
assert(e.getCause.isInstanceOf[SchemaColumnConvertNotSupportedException])
@@ -3724,26 +3707,19 @@ index 89f65562295..166a11a9079 100644
val tblTargetName = "tbl_target"
val tblSourceQualified = s"default.$tblSourceName"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-index f0f3f94b811..be5e113c3ed 100644
+index f0f3f94b811..b7d18771314 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-@@ -27,13 +27,14 @@ import scala.jdk.CollectionConverters._
- import scala.language.implicitConversions
- import scala.util.control.NonFatal
-
-+import org.apache.comet.CometConf
- import org.apache.hadoop.fs.Path
- import org.scalactic.source.Position
- import org.scalatest.{BeforeAndAfterAll, Suite, Tag}
+@@ -33,7 +33,7 @@ import org.scalatest.{BeforeAndAfterAll, Suite, Tag}
import org.scalatest.concurrent.Eventually
import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.{AnalysisException, Row}
-+import org.apache.spark.sql.{AnalysisException, IgnoreComet, IgnoreCometNativeDataFusion, IgnoreCometNativeIcebergCompat, IgnoreCometNativeScan, Row}
++import org.apache.spark.sql.{AnalysisException, IgnoreComet, Row}
import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
import org.apache.spark.sql.catalyst.catalog.SessionCatalog.DEFAULT_DATABASE
-@@ -42,6 +43,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTestBase
+@@ -42,6 +42,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTestBase
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.classic.{ClassicConversions, ColumnConversions, ColumnNodeToExpressionConverter, DataFrame, Dataset, SparkSession, SQLImplicits}
@@ -3751,7 +3727,7 @@ index f0f3f94b811..be5e113c3ed 100644
import org.apache.spark.sql.execution.FilterExec
import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
import org.apache.spark.sql.execution.datasources.DataSourceUtils
-@@ -121,6 +123,34 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
+@@ -121,6 +122,11 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
(implicit pos: Position): Unit = {
@@ -3759,34 +3735,11 @@ index f0f3f94b811..be5e113c3ed 100644
+ if (isCometEnabled && testTags.exists(_.isInstanceOf[IgnoreComet])) {
+ ignore(testName + " (disabled when Comet is on)", testTags: _*)(testFun)
+ return
-+ }
-+ if (isCometEnabled) {
-+ val cometScanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get(conf)
-+ val isNativeIcebergCompat = cometScanImpl == CometConf.SCAN_NATIVE_ICEBERG_COMPAT ||
-+ cometScanImpl == CometConf.SCAN_AUTO
-+ val isNativeDataFusion = cometScanImpl == CometConf.SCAN_NATIVE_DATAFUSION ||
-+ cometScanImpl == CometConf.SCAN_AUTO
-+ if (isNativeIcebergCompat &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeIcebergCompat])) {
-+ ignore(testName + " (disabled for NATIVE_ICEBERG_COMPAT)", testTags: _*)(testFun)
-+ return
-+ }
-+ if (isNativeDataFusion &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeDataFusion])) {
-+ ignore(testName + " (disabled for NATIVE_DATAFUSION)", testTags: _*)(testFun)
-+ return
-+ }
-+ if ((isNativeDataFusion || isNativeIcebergCompat) &&
-+ testTags.exists(_.isInstanceOf[IgnoreCometNativeScan])) {
-+ ignore(testName + " (disabled for NATIVE_DATAFUSION and NATIVE_ICEBERG_COMPAT)",
-+ testTags: _*)(testFun)
-+ return
-+ }
+ }
if (testTags.exists(_.isInstanceOf[DisableAdaptiveExecution])) {
super.test(testName, testTags: _*) {
withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-@@ -248,8 +278,15 @@ private[sql] trait SQLTestUtilsBase
+@@ -248,8 +254,15 @@ private[sql] trait SQLTestUtilsBase
override protected def converter: ColumnNodeToExpressionConverter = self.spark.converter
}
@@ -3802,7 +3755,7 @@ index f0f3f94b811..be5e113c3ed 100644
super.withSQLConf(pairs: _*)(f)
}
-@@ -451,6 +488,8 @@ private[sql] trait SQLTestUtilsBase
+@@ -451,6 +464,8 @@ private[sql] trait SQLTestUtilsBase
val schema = df.schema
val withoutFilters = df.queryExecution.executedPlan.transform {
case FilterExec(_, child) => child
diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml
index 4fb3ed4c5d..c58d446917 100644
--- a/native/core/Cargo.toml
+++ b/native/core/Cargo.toml
@@ -109,18 +109,6 @@ name = "comet"
# "rlib" is for benchmarking with criterion.
crate-type = ["cdylib", "rlib"]
-[[bench]]
-name = "parquet_read"
-harness = false
-
-[[bench]]
-name = "bit_util"
-harness = false
-
-[[bench]]
-name = "parquet_decode"
-harness = false
-
[[bench]]
name = "array_element_append"
harness = false
diff --git a/native/core/benches/bit_util.rs b/native/core/benches/bit_util.rs
deleted file mode 100644
index 92ae497a56..0000000000
--- a/native/core/benches/bit_util.rs
+++ /dev/null
@@ -1,216 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{mem::size_of, time::Duration};
-
-use rand::{rng, RngExt};
-
-use arrow::buffer::Buffer;
-use comet::common::bit::{
- log2, read_num_bytes_u32, read_num_bytes_u64, read_u32, read_u64, set_bits, trailing_bits,
- BitReader, BitWriter,
-};
-use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
-use std::hint::black_box;
-
-/// Benchmark to measure bit_util performance.
-/// To run this benchmark:
-/// `cd core && cargo bench --bench bit_util`
-/// Results will be written to "core/target/criterion/bit_util/"
-fn criterion_benchmark(c: &mut Criterion) {
- let mut group = c.benchmark_group("bit_util");
-
- const N: usize = 1024 * 1024;
- let mut writer: BitWriter = BitWriter::new(N * 10);
- for _ in 0..N {
- if !writer.put_vlq_int(rng().random::<u64>()) {
- break;
- }
- }
- let buffer = writer.consume();
- let buffer = Buffer::from(buffer.as_slice());
-
- // log2
- for bits in (0..64).step_by(3) {
- let x = 1u64 << bits;
- group.bench_with_input(BenchmarkId::new("log2", bits), &x, |b, &x| {
- b.iter(|| log2(black_box(x)));
- });
- }
-
- // set_bits
- for offset in (0..16).step_by(3) {
- for length in (0..16).step_by(3) {
- let x = (offset, length);
- group.bench_with_input(
- BenchmarkId::new("set_bits", format!("offset_{}_length_{}", x.0, x.1)),
- &x,
- |b, &x| {
- b.iter(|| set_bits(&mut [0u8; 4], black_box(x.0), black_box(x.1)));
- },
- );
- }
- }
-
- // get_vlq_int
- group.bench_function("get_vlq_int", |b| {
- b.iter(|| {
- let mut reader: BitReader = BitReader::new_all(buffer.slice(0));
- bench_get_vlq_int(&mut reader)
- })
- });
-
- // get_bits
- for offset in (0..32).step_by(17) {
- for num_bits in (1..5).step_by(1) {
- let x = (offset, num_bits);
- group.bench_with_input(
- BenchmarkId::new("get_bits", format!("offset_{}_num_bits_{}", x.0, x.1)),
- &x,
- |b, &x| {
- let mut reader: BitReader = BitReader::new_all(buffer.slice(0));
- b.iter(|| reader.get_bits(&mut [0u8; 4], black_box(x.0), black_box(x.1)));
- },
- );
- }
- }
-
- // get_aligned
- for num_bytes in (1..=size_of::<u8>()).step_by(3) {
- let x = num_bytes;
- group.bench_with_input(
- BenchmarkId::new("get_aligned", format!("u8_num_bytes_{x}")),
- &x,
- |b, &x| {
- let mut reader: BitReader = BitReader::new_all(buffer.slice(0));
- b.iter(|| reader.get_aligned::<u8>(black_box(x)));
- },
- );
- }
- for num_bytes in (1..=size_of::<u32>()).step_by(3) {
- let x = num_bytes;
- group.bench_with_input(
- BenchmarkId::new("get_aligned", format!("u32_num_bytes_{x}")),
- &x,
- |b, &x| {
- let mut reader: BitReader = BitReader::new_all(buffer.slice(0));
- b.iter(|| reader.get_aligned::<u32>(black_box(x)));
- },
- );
- }
- for num_bytes in (1..=size_of::<i32>()).step_by(3) {
- let x = num_bytes;
- group.bench_with_input(
- BenchmarkId::new("get_aligned", format!("i32_num_bytes_{x}")),
- &x,
- |b, &x| {
- let mut reader: BitReader = BitReader::new_all(buffer.slice(0));
- b.iter(|| reader.get_aligned::<i32>(black_box(x)));
- },
- );
- }
-
- // get_value
- for num_bytes in (1..=size_of::<i32>()).step_by(3) {
- let x = num_bytes * 8;
- group.bench_with_input(
- BenchmarkId::new("get_value", format!("i32_num_bits_{x}")),
- &x,
- |b, &x| {
- let mut reader: BitReader = BitReader::new_all(buffer.slice(0));
- b.iter(|| reader.get_value::<i32>(black_box(x)));
- },
- );
- }
-
- // read_num_bytes_u64
- for num_bytes in (1..=8).step_by(7) {
- let x = num_bytes;
- group.bench_with_input(
- BenchmarkId::new("read_num_bytes_u64", format!("num_bytes_{x}")),
- &x,
- |b, &x| {
- b.iter(|| read_num_bytes_u64(black_box(x), black_box(buffer.as_slice())));
- },
- );
- }
-
- // read_num_bytes_u32
- for num_bytes in (1..=4).step_by(3) {
- let x = num_bytes;
- group.bench_with_input(
- BenchmarkId::new("read_num_bytes_u32", format!("num_bytes_{x}")),
- &x,
- |b, &x| {
- b.iter(|| read_num_bytes_u32(black_box(x), black_box(buffer.as_slice())));
- },
- );
- }
-
- // trailing_bits
- for length in (0..=64).step_by(32) {
- let x = length;
- group.bench_with_input(
- BenchmarkId::new("trailing_bits", format!("num_bits_{x}")),
- &x,
- |b, &x| {
- b.iter(|| trailing_bits(black_box(1234567890), black_box(x)));
- },
- );
- }
-
- // read_u64
- group.bench_function("read_u64", |b| {
- b.iter(|| read_u64(black_box(&[0u8; 8])));
- });
-
- // read_u32
- group.bench_function("read_u32", |b| {
- b.iter(|| read_u32(black_box(&[0u8; 4])));
- });
-
- // get_u32_value
- group.bench_function("get_u32_value", |b| {
- b.iter(|| {
- let mut reader: BitReader = BitReader::new_all(buffer.slice(0));
- for _ in 0..(buffer.len() * 8 / 31) {
- black_box(reader.get_u32_value(black_box(31)));
- }
- })
- });
-
- group.finish();
-}
-
-fn bench_get_vlq_int(reader: &mut BitReader) {
- while let Some(v) = reader.get_vlq_int() {
- black_box(v);
- }
-}
-
-fn config() -> Criterion {
- Criterion::default()
- .measurement_time(Duration::from_millis(500))
- .warm_up_time(Duration::from_millis(500))
-}
-
-criterion_group! {
- name = benches;
- config = config();
- targets = criterion_benchmark
-}
-criterion_main!(benches);
diff --git a/native/core/benches/parquet_decode.rs b/native/core/benches/parquet_decode.rs
deleted file mode 100644
index d3976aefc3..0000000000
--- a/native/core/benches/parquet_decode.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use arrow::datatypes::ToByteSlice;
-use comet::parquet::read::values::{copy_i32_to_i16, copy_i32_to_u16, copy_i64_to_i64};
-use criterion::{criterion_group, criterion_main, Criterion};
-
-fn criterion_benchmark(c: &mut Criterion) {
- let num = 1000;
- let source = vec![78_i8; num * 8];
- let mut group = c.benchmark_group("parquet_decode");
- group.bench_function("decode_i32_to_i16", |b| {
- let mut dest: Vec<u8> = vec![b' '; num * 2];
- b.iter(|| {
- copy_i32_to_i16(source.to_byte_slice(), dest.as_mut_slice(), num);
- });
- });
- group.bench_function("decode_i32_to_u16", |b| {
- let mut dest: Vec<u8> = vec![b' '; num * 4];
- b.iter(|| {
- copy_i32_to_u16(source.to_byte_slice(), dest.as_mut_slice(), num);
- });
- });
- group.bench_function("decode_i64_to_i64", |b| {
- let mut dest: Vec<u8> = vec![b' '; num * 8];
- b.iter(|| {
- copy_i64_to_i64(source.to_byte_slice(), dest.as_mut_slice(), num);
- });
- });
-}
-
-// Create UTF8 batch with strings representing ints, floats, nulls
-fn config() -> Criterion {
- Criterion::default()
-}
-
-criterion_group! {
- name = benches;
- config = config();
- targets = criterion_benchmark
-}
-criterion_main!(benches);
diff --git a/native/core/benches/parquet_read.rs b/native/core/benches/parquet_read.rs
deleted file mode 100644
index 52ab96eb1b..0000000000
--- a/native/core/benches/parquet_read.rs
+++ /dev/null
@@ -1,216 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-mod perf;
-
-use std::sync::Arc;
-
-use arrow::{array::ArrayData, buffer::Buffer};
-use comet::parquet::{read::ColumnReader, util::jni::TypePromotionInfo};
-use criterion::{criterion_group, criterion_main, Criterion};
-use parquet::{
- basic::{Encoding, Type as PhysicalType},
- column::page::{PageIterator, PageReader},
- data_type::Int32Type,
- schema::types::{
- ColumnDescPtr, ColumnDescriptor, ColumnPath, PrimitiveTypeBuilder, SchemaDescPtr, TypePtr,
- },
-};
-
-use comet::parquet::util::test_common::page_util::{
- DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator,
-};
-
-use perf::FlamegraphProfiler;
-use rand::{prelude::StdRng, RngExt, SeedableRng};
-
-fn bench(c: &mut Criterion) {
- let expected_num_values: usize = NUM_PAGES * VALUES_PER_PAGE;
- let mut group = c.benchmark_group("comet_parquet_read");
- let schema = build_test_schema();
-
- let pages = build_plain_int32_pages(schema.column(0), 0.0);
- group.bench_function("INT/PLAIN/NOT_NULL", |b| {
- let t = TypePtr::new(
- PrimitiveTypeBuilder::new("f", PhysicalType::INT32)
- .with_length(4)
- .build()
- .unwrap(),
- );
- b.iter(|| {
- let cd = ColumnDescriptor::new(t.clone(), 0, 0, ColumnPath::from(Vec::new()));
- let promotion_info = TypePromotionInfo::new(PhysicalType::INT32, -1, -1, 32);
- let mut column_reader = TestColumnReader::new(
- cd,
- promotion_info,
- BATCH_SIZE,
- pages.clone(),
- expected_num_values,
- );
-
- let mut total = 0;
- for batch in column_reader.by_ref() {
- total += batch.len();
- ::std::mem::forget(batch);
- }
- assert_eq!(total, expected_num_values);
- });
- });
-}
-
-fn profiled() -> Criterion {
- Criterion::default().with_profiler(FlamegraphProfiler::new(100))
-}
-
-criterion_group! {
- name = benches;
- config = profiled();
- targets = bench
-}
-criterion_main!(benches);
-
-fn build_test_schema() -> SchemaDescPtr {
- use parquet::schema::{parser::parse_message_type, types::SchemaDescriptor};
- let message_type = "
- message test_schema {
- REQUIRED INT32 c1;
- OPTIONAL INT32 c2;
- }
- ";
- parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap()
-}
-
-fn seedable_rng() -> StdRng {
- StdRng::seed_from_u64(42)
-}
-
-// test data params
-const NUM_PAGES: usize = 1000;
-const VALUES_PER_PAGE: usize = 10_000;
-const BATCH_SIZE: usize = 4096;
-
-fn build_plain_int32_pages(
- column_desc: ColumnDescPtr,
- null_density: f32,
-) -> impl PageIterator + Clone {
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
- let rep_levels = vec![0; VALUES_PER_PAGE];
- let mut rng = seedable_rng();
- let mut pages: Vec<parquet::column::page::Page> = Vec::new();
- let mut int32_value = 0;
- for _ in 0..NUM_PAGES {
- // generate page
- let mut values = Vec::with_capacity(VALUES_PER_PAGE);
- let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE);
- for _ in 0..VALUES_PER_PAGE {
- let def_level = if rng.random::<f32>() < null_density {
- max_def_level - 1
- } else {
- max_def_level
- };
- if def_level == max_def_level {
- int32_value += 1;
- values.push(int32_value);
- }
- def_levels.push(def_level);
- }
- let mut page_builder =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
- page_builder.add_rep_levels(max_rep_level, &rep_levels);
- page_builder.add_def_levels(max_def_level, &def_levels);
- page_builder.add_values::<Int32Type>(Encoding::PLAIN, &values);
- pages.push(page_builder.consume());
- }
-
- // Since `InMemoryPageReader` is not exposed from parquet crate, here we use
- // `InMemoryPageIterator` instead which is a Iter<Iter<Page>>.
- InMemoryPageIterator::new(vec![pages])
-}
-
-struct TestColumnReader {
- inner: ColumnReader,
- pages: Box<dyn PageReader>,
- batch_size: usize,
- total_num_values: usize,
- total_num_values_read: usize,
- first_page_loaded: bool,
-}
-
-impl TestColumnReader {
- pub fn new(
- cd: ColumnDescriptor,
- promotion_info: TypePromotionInfo,
- batch_size: usize,
- mut page_iter: impl PageIterator + 'static,
- total_num_values: usize,
- ) -> Self {
- let reader = ColumnReader::get(cd, promotion_info, batch_size, false, false);
- let first = page_iter.next().unwrap().unwrap();
- Self {
- inner: reader,
- pages: first,
- batch_size,
- total_num_values,
- total_num_values_read: 0,
- first_page_loaded: false,
- }
- }
-
- fn load_page(&mut self) {
- if let Some(page) = self.pages.get_next_page().unwrap() {
- let num_values = page.num_values() as usize;
- let buffer = Buffer::from_slice_ref(page.buffer());
- self.inner.set_page_v1(num_values, buffer, page.encoding());
- }
- }
-}
-
-impl Iterator for TestColumnReader {
- type Item = ArrayData;
-
- fn next(&mut self) -> Option<Self::Item> {
- if self.total_num_values_read >= self.total_num_values {
- return None;
- }
-
- if !self.first_page_loaded {
- self.load_page();
- self.first_page_loaded = true;
- }
-
- self.inner.reset_batch();
- let total = ::std::cmp::min(
- self.batch_size,
- self.total_num_values - self.total_num_values_read,
- );
-
- let mut left = total;
- while left > 0 {
- let (num_read, _) = self.inner.read_batch(left, 0);
- if num_read < left {
- self.load_page();
- }
- left -= num_read;
- }
- self.total_num_values_read += total;
-
- Some(self.inner.current_batch().unwrap())
- }
-}
diff --git a/native/core/src/common/bit.rs b/native/core/src/common/bit.rs
deleted file mode 100644
index 459c38a52b..0000000000
--- a/native/core/src/common/bit.rs
+++ /dev/null
@@ -1,1602 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{cmp::min, mem::size_of};
-
-use crate::{
- errors::CometResult as Result,
- parquet::{data_type::AsBytes, util::bit_packing::unpack32},
-};
-use arrow::buffer::Buffer;
-use datafusion_comet_spark_expr::utils::{likely, unlikely};
-
-#[inline]
-pub fn from_ne_slice<T: FromBytes>(bs: &[u8]) -> T {
- let mut b = T::Buffer::default();
- {
- let b = b.as_mut();
- let bs = &bs[..b.len()];
- b.copy_from_slice(bs);
- }
- T::from_ne_bytes(b)
-}
-
-pub trait FromBytes: Sized {
- type Buffer: AsMut<[u8]> + Default;
- fn from_le_bytes(bs: Self::Buffer) -> Self;
- fn from_be_bytes(bs: Self::Buffer) -> Self;
- fn from_ne_bytes(bs: Self::Buffer) -> Self;
- fn from(v: u64) -> Self;
-}
-
-macro_rules! from_le_bytes {
- ($($ty: ty),*) => {
- $(
- impl FromBytes for $ty {
- type Buffer = [u8; size_of::<Self>()];
- fn from_le_bytes(bs: Self::Buffer) -> Self {
- <$ty>::from_le_bytes(bs)
- }
- fn from_be_bytes(bs: Self::Buffer) -> Self {
- <$ty>::from_be_bytes(bs)
- }
- fn from_ne_bytes(bs: Self::Buffer) -> Self {
- <$ty>::from_ne_bytes(bs)
- }
- fn from(v: u64) -> Self {
- v as $ty
- }
- }
- )*
- };
-}
-
-impl FromBytes for bool {
- type Buffer = [u8; 1];
- fn from_le_bytes(bs: Self::Buffer) -> Self {
- Self::from_ne_bytes(bs)
- }
- fn from_be_bytes(bs: Self::Buffer) -> Self {
- Self::from_ne_bytes(bs)
- }
- fn from_ne_bytes(bs: Self::Buffer) -> Self {
- match bs[0] {
- 0 => false,
- 1 => true,
- _ => panic!("Invalid byte when reading bool"),
- }
- }
- fn from(v: u64) -> Self {
- (v & 1) == 1
- }
-}
-
-// TODO: support f32 and f64 in the future, but there is no use case right now
-// f32/f64::from(v: u64) will be like `from_ne_slice(v.as_bytes()))` and that is
-// expensive as it involves copying buffers
-from_le_bytes! { u8, u16, u32, u64, i8, i16, i32, i64 }
-
-/// Reads `$size` of bytes from `$src`, and reinterprets them as type `$ty`, in
-/// little-endian order. `$ty` must implement the `Default` trait. Otherwise this won't
-/// compile.
-/// This is copied and modified from byteorder crate.
-macro_rules! read_num_bytes {
- ($ty:ty, $size:expr, $src:expr) => {{
- debug_assert!($size <= $src.len());
- let mut buffer = <$ty as $crate::common::bit::FromBytes>::Buffer::default();
- buffer.as_mut()[..$size].copy_from_slice(&$src[..$size]);
- <$ty>::from_ne_bytes(buffer)
- }};
-}
-
-/// u64 specific version of read_num_bytes!
-/// This is faster than read_num_bytes! because this method avoids buffer copies.
-#[inline]
-pub fn read_num_bytes_u64(size: usize, src: &[u8]) -> u64 {
- debug_assert!(size <= src.len());
- if unlikely(src.len() < 8) {
- return read_num_bytes!(u64, size, src);
- }
- let in_ptr = src as *const [u8] as *const u8 as *const u64;
- let v = unsafe { in_ptr.read_unaligned() };
- trailing_bits(v, size * 8)
-}
-
-/// u32 specific version of read_num_bytes!
-/// This is faster than read_num_bytes! because this method avoids buffer copies.
-#[inline]
-pub fn read_num_bytes_u32(size: usize, src: &[u8]) -> u32 {
- debug_assert!(size <= src.len());
- if unlikely(src.len() < 4) {
- return read_num_bytes!(u32, size, src);
- }
- let in_ptr = src as *const [u8] as *const u8 as *const u32;
- let v = unsafe { in_ptr.read_unaligned() };
- trailing_bits(v as u64, size * 8) as u32
-}
-
-#[inline]
-pub fn read_u64(src: &[u8]) -> u64 {
- let in_ptr = src.as_ptr() as *const u64;
- unsafe { in_ptr.read_unaligned() }
-}
-
-#[inline]
-pub fn read_u32(src: &[u8]) -> u32 {
- let in_ptr = src.as_ptr() as *const u32;
- unsafe { in_ptr.read_unaligned() }
-}
-
-#[inline]
-pub fn memcpy(source: &[u8], target: &mut [u8]) {
- debug_assert!(target.len() >= source.len(), "Copying from source to target is not possible. Source has {} bytes but target has {} bytes", source.len(), target.len());
- // Originally `target[..source.len()].copy_from_slice(source)`
- // We use the unsafe copy method to avoid some expensive bounds checking/
- unsafe { std::ptr::copy_nonoverlapping(source.as_ptr(), target.as_mut_ptr(), source.len()) }
-}
-
-#[inline]
-pub fn memcpy_value<T>(source: &T, num_bytes: usize, target: &mut [u8])
-where
- T: ?Sized + AsBytes,
-{
- debug_assert!(
- target.len() >= num_bytes,
- "Not enough space. Only had {} bytes but need to put {} bytes",
- target.len(),
- num_bytes
- );
- memcpy(&source.as_bytes()[..num_bytes], target)
-}
-
-/// Returns ceil(log2(x))
-#[inline]
-pub fn log2(mut x: u64) -> u32 {
- if x == 1 {
- return 0;
- }
- x -= 1;
- 64u32 - x.leading_zeros()
-}
-
-/// Returns the `num_bits` least-significant bits of `v`
-#[inline]
-pub fn trailing_bits(v: u64, num_bits: usize) -> u64 {
- if unlikely(num_bits == 0) {
- return 0;
- }
- if unlikely(num_bits >= 64) {
- return v;
- }
- v & ((1 << num_bits) - 1)
-}
-
-#[inline]
-pub fn set_bit(bits: &mut [u8], i: usize) {
- bits[i / 8] |= 1 << (i % 8);
-}
-
-/// Set the bit value at index `i`, for buffer `bits`.
-///
-/// # Safety
-/// This doesn't check bounds, the caller must ensure that `i` is in (0, bits.len() * 8)
-#[inline]
-pub unsafe fn set_bit_raw(bits: *mut u8, i: usize) {
- *bits.add(i / 8) |= 1 << (i % 8);
-}
-
-#[inline]
-pub fn unset_bit(bits: &mut [u8], i: usize) {
- bits[i / 8] &= !(1 << (i % 8));
-}
-
-#[inline]
-pub fn set_bits(bits: &mut [u8], offset: usize, length: usize) {
- let mut byte_i = offset / 8;
- let offset_r = offset % 8;
- let end = offset + length;
- let end_byte_i = end / 8;
- let end_r = end % 8;
-
- // if the offset starts in the middle of a byte, update the byte first
- if offset_r != 0 {
- let num_bits = min(length, 7);
- bits[byte_i] |= ((1u8 << num_bits) - 1) << offset_r;
- byte_i += 1;
- }
-
- // See if there is an opportunity to do a bulk byte write
- if byte_i < end_byte_i {
- unsafe {
- bits.as_mut_ptr()
- .add(byte_i)
- .write_bytes(255, end_byte_i - byte_i);
- }
- byte_i = end_byte_i;
- }
-
- // take care of the last byte
- if end_r > 0 && (byte_i == end_byte_i) {
- bits[byte_i] |= (1u8 << end_r) - 1;
- }
-}
-
-#[inline(always)]
-pub fn mix_hash(lower: u64, upper: u64) -> u64 {
- let hash = (17 * 37u64).wrapping_add(lower);
- hash.wrapping_mul(37).wrapping_add(upper)
-}
-
-static BIT_MASK: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128];
-
-/// Returns whether bit at position `i` in `data` is set or not
-#[inline]
-pub fn get_bit(data: &[u8], i: usize) -> bool {
- (data[i >> 3] & BIT_MASK[i & 7]) != 0
-}
-
-/// Returns the boolean value at index `i`.
-///
-/// # Safety
-/// This doesn't check bounds, the caller must ensure that index < self.len()
-#[inline]
-pub unsafe fn get_bit_raw(ptr: *const u8, i: usize) -> bool {
- (*ptr.add(i >> 3) & BIT_MASK[i & 7]) != 0
-}
-
-/// Utility class for writing bit/byte streams. This class can write data in either
-/// bit packed or byte aligned fashion.
-pub struct BitWriter {
- buffer: Vec<u8>,
- max_bytes: usize,
- buffered_values: u64,
- byte_offset: usize,
- bit_offset: usize,
- start: usize,
-}
-
-impl BitWriter {
- pub fn new(max_bytes: usize) -> Self {
- Self {
- buffer: vec![0; max_bytes],
- max_bytes,
- buffered_values: 0,
- byte_offset: 0,
- bit_offset: 0,
- start: 0,
- }
- }
-
- /// Initializes the writer from the existing buffer `buffer` and starting
- /// offset `start`.
- pub fn new_from_buf(buffer: Vec<u8>, start: usize) -> Self {
- debug_assert!(start < buffer.len());
- let len = buffer.len();
- Self {
- buffer,
- max_bytes: len,
- buffered_values: 0,
- byte_offset: start,
- bit_offset: 0,
- start,
- }
- }
-
- /// Extend buffer size by `increment` bytes
- #[inline]
- pub fn extend(&mut self, increment: usize) {
- self.max_bytes += increment;
- let extra = vec![0; increment];
- self.buffer.extend(extra);
- }
-
- /// Report buffer size, in bytes
- #[inline]
- pub fn capacity(&mut self) -> usize {
- self.max_bytes
- }
-
- /// Consumes and returns the current buffer.
- #[inline]
- pub fn consume(mut self) -> Vec<u8> {
- self.flush();
- self.buffer.truncate(self.byte_offset);
- self.buffer
- }
-
- /// Flushes the internal buffered bits and returns the buffer's content.
- /// This is a borrow equivalent of `consume` method.
- #[inline]
- pub fn flush_buffer(&mut self) -> &[u8] {
- self.flush();
- &self.buffer()[0..self.byte_offset]
- }
-
- /// Clears the internal state so the buffer can be reused.
- #[inline]
- pub fn clear(&mut self) {
- self.buffered_values = 0;
- self.byte_offset = self.start;
- self.bit_offset = 0;
- }
-
- /// Flushes the internal buffered bits and the align the buffer to the next byte.
- #[inline]
- pub fn flush(&mut self) {
- let num_bytes = self.bit_offset.div_ceil(8);
- debug_assert!(self.byte_offset + num_bytes <= self.max_bytes);
- memcpy_value(
- &self.buffered_values,
- num_bytes,
- &mut self.buffer[self.byte_offset..],
- );
- self.buffered_values = 0;
- self.bit_offset = 0;
- self.byte_offset += num_bytes;
- }
-
- /// Advances the current offset by skipping `num_bytes`, flushing the internal bit
- /// buffer first.
- /// This is useful when you want to jump over `num_bytes` bytes and come back later
- /// to fill these bytes.
- ///
- /// Returns error if `num_bytes` is beyond the boundary of the internal buffer.
- /// Otherwise, returns the old offset.
- #[inline]
- pub fn skip(&mut self, num_bytes: usize) -> Result<usize> {
- self.flush();
- debug_assert!(self.byte_offset <= self.max_bytes);
- if unlikely(self.byte_offset + num_bytes > self.max_bytes) {
- return Err(general_err!(
- "Not enough bytes left in BitWriter. Need {} but only have {}",
- self.byte_offset + num_bytes,
- self.max_bytes
- ));
- }
- let result = self.byte_offset;
- self.byte_offset += num_bytes;
- Ok(result)
- }
-
- /// Returns a slice containing the next `num_bytes` bytes starting from the current
- /// offset, and advances the underlying buffer by `num_bytes`.
- /// This is useful when you want to jump over `num_bytes` bytes and come back later
- /// to fill these bytes.
- #[inline]
- pub fn get_next_byte_ptr(&mut self, num_bytes: usize) -> Result<&mut [u8]> {
- let offset = self.skip(num_bytes)?;
- Ok(&mut self.buffer[offset..offset + num_bytes])
- }
-
- #[inline]
- pub fn bytes_written(&self) -> usize {
- self.byte_offset - self.start + self.bit_offset.div_ceil(8)
- }
-
- #[inline]
- pub fn buffer(&self) -> &[u8] {
- &self.buffer[self.start..]
- }
-
- #[inline]
- pub fn byte_offset(&self) -> usize {
- self.byte_offset
- }
-
- /// Returns the internal buffer length. This is the maximum number of bytes that this
- /// writer can write. User needs to call `consume` to consume the current buffer
- /// before more data can be written.
- #[inline]
- pub fn buffer_len(&self) -> usize {
- self.max_bytes
- }
-
- /// Writes the entire byte `value` at the byte `offset`
- pub fn write_at(&mut self, offset: usize, value: u8) {
- self.buffer[offset] = value;
- }
-
- /// Writes the `num_bits` LSB of value `v` to the internal buffer of this writer.
- /// The `num_bits` must not be greater than 64. This is bit packed.
- ///
- /// Returns false if there's not enough room left. True otherwise.
- #[inline]
- #[allow(clippy::unnecessary_cast)]
- pub fn put_value(&mut self, v: u64, num_bits: usize) -> bool {
- debug_assert!(num_bits <= 64);
- debug_assert_eq!(v.checked_shr(num_bits as u32).unwrap_or(0), 0); // covers case v >> 64
-
- let num_bytes = self.byte_offset * 8 + self.bit_offset + num_bits;
- if unlikely(num_bytes > self.max_bytes as usize * 8) {
- return false;
- }
-
- self.buffered_values |= v << self.bit_offset;
- self.bit_offset += num_bits;
- if self.bit_offset >= 64 {
- memcpy_value(
- &self.buffered_values,
- 8,
- &mut self.buffer[self.byte_offset..],
- );
- self.byte_offset += 8;
- self.bit_offset -= 64;
- self.buffered_values = 0;
- // Perform checked right shift: v >> offset, where offset < 64, otherwise we
- // shift all bits
- self.buffered_values = v
- .checked_shr((num_bits - self.bit_offset) as u32)
- .unwrap_or(0);
- }
- debug_assert!(self.bit_offset < 64);
- true
- }
-
- /// Writes `val` of `num_bytes` bytes to the next aligned byte. If size of `T` is
- /// larger than `num_bytes`, extra higher ordered bytes will be ignored.
- ///
- /// Returns false if there's not enough room left. True otherwise.
- #[inline]
- pub fn put_aligned<T: AsBytes>(&mut self, val: T, num_bytes: usize) -> bool {
- let result = self.get_next_byte_ptr(num_bytes);
- if unlikely(result.is_err()) {
- // TODO: should we return `Result` for this func?
- return false;
- }
- let ptr = result.unwrap();
- memcpy_value(&val, num_bytes, ptr);
- true
- }
-
- /// Writes `val` of `num_bytes` bytes at the designated `offset`. The `offset` is the
- /// offset starting from the beginning of the internal buffer that this writer
- /// maintains. Note that this will overwrite any existing data between `offset` and
- /// `offset + num_bytes`. Also that if size of `T` is larger than `num_bytes`, extra
- /// higher ordered bytes will be ignored.
- ///
- /// Returns false if there's not enough room left, or the `pos` is not valid.
- /// True otherwise.
- #[inline]
- pub fn put_aligned_offset<T: AsBytes>(
- &mut self,
- val: T,
- num_bytes: usize,
- offset: usize,
- ) -> bool {
- if unlikely(num_bytes + offset > self.max_bytes) {
- return false;
- }
- memcpy_value(
- &val,
- num_bytes,
- &mut self.buffer[offset..offset + num_bytes],
- );
- true
- }
-
- /// Writes a VLQ encoded integer `v` to this buffer. The value is byte aligned.
- ///
- /// Returns false if there's not enough room left. True otherwise.
- #[inline]
- pub fn put_vlq_int(&mut self, mut v: u64) -> bool {
- let mut result = true;
- while v & 0xFFFFFFFFFFFFFF80 != 0 {
- result &= self.put_aligned::<u8>(((v & 0x7F) | 0x80) as u8, 1);
- v >>= 7;
- }
- result &= self.put_aligned::<u8>((v & 0x7F) as u8, 1);
- result
- }
-
- /// Writes a zigzag-VLQ encoded (in little endian order) int `v` to this buffer.
- /// Zigzag-VLQ is a variant of VLQ encoding where negative and positive
- /// numbers are encoded in a zigzag fashion.
- /// See: https://developers.google.com/protocol-buffers/docs/encoding
- ///
- /// Returns false if there's not enough room left. True otherwise.
- #[inline]
- pub fn put_zigzag_vlq_int(&mut self, v: i64) -> bool {
- let u: u64 = ((v << 1) ^ (v >> 63)) as u64;
- self.put_vlq_int(u)
- }
-}
-
-/// Maximum byte length for a VLQ encoded integer
-/// MAX_VLQ_BYTE_LEN = 5 for i32, and MAX_VLQ_BYTE_LEN = 10 for i64
-pub const MAX_VLQ_BYTE_LEN: usize = 10;
-
-pub struct BitReader {
- /// The byte buffer to read from, passed in by client
- buffer: Buffer, // TODO: generalize this
-
- /// Bytes are memcpy'd from `buffer` and values are read from this variable.
- /// This is faster than reading values byte by byte directly from `buffer`
- buffered_values: u64,
-
- ///
- /// End Start
- /// |............|B|B|B|B|B|B|B|B|..............|
- /// ^ ^
- /// bit_offset byte_offset
- ///
- /// Current byte offset in `buffer`
- byte_offset: usize,
-
- /// Current bit offset in `buffered_values`
- bit_offset: usize,
-
- /// Total number of bytes in `buffer`
- total_bytes: usize,
-}
-
-/// Utility class to read bit/byte stream. This class can read bits or bytes that are
-/// either byte aligned or not.
-impl BitReader {
- pub fn new(buf: Buffer, len: usize) -> Self {
- let buffered_values = if size_of::<u64>() > len {
- read_num_bytes_u64(len, buf.as_slice())
- } else {
- read_u64(buf.as_slice())
- };
- BitReader {
- buffer: buf,
- buffered_values,
- byte_offset: 0,
- bit_offset: 0,
- total_bytes: len,
- }
- }
-
- pub fn new_all(buf: Buffer) -> Self {
- let len = buf.len();
- Self::new(buf, len)
- }
-
- pub fn reset(&mut self, buf: Buffer) {
- self.buffer = buf;
- self.total_bytes = self.buffer.len();
- self.buffered_values = if size_of::<u64>() > self.total_bytes {
- read_num_bytes_u64(self.total_bytes, self.buffer.as_slice())
- } else {
- read_u64(self.buffer.as_slice())
- };
- self.byte_offset = 0;
- self.bit_offset = 0;
- }
-
- /// Gets the current byte offset
- #[inline]
- pub fn get_byte_offset(&self) -> usize {
- self.byte_offset + self.bit_offset.div_ceil(8)
- }
-
- /// Reads a value of type `T` and of size `num_bits`.
- ///
- /// Returns `None` if there's not enough data available. `Some` otherwise.
- pub fn get_value<T: FromBytes>(&mut self, num_bits: usize) -> Option<T> {
- debug_assert!(num_bits <= 64);
- debug_assert!(num_bits <= size_of::<T>() * 8);
-
- if unlikely(self.byte_offset * 8 + self.bit_offset + num_bits > self.total_bytes * 8) {
- return None;
- }
-
- let v = self.get_u64_value(num_bits);
- Some(T::from(v))
- }
-
- /// Reads a `u32` value encoded using `num_bits` of bits.
- ///
- /// # Safety
- ///
- /// This method asusumes the following:
- ///
- /// - the `num_bits` is <= 64
- /// - the remaining number of bits to read in this reader is >= `num_bits`.
- ///
- /// Undefined behavior will happen if any of the above assumptions is violated.
- #[inline]
- pub fn get_u32_value(&mut self, num_bits: usize) -> u32 {
- self.get_u64_value(num_bits) as u32
- }
-
- #[inline(always)]
- fn get_u64_value(&mut self, num_bits: usize) -> u64 {
- if unlikely(num_bits == 0) {
- 0
- } else {
- let v = self.buffered_values >> self.bit_offset;
- let mask = u64::MAX >> (64 - num_bits);
- self.bit_offset += num_bits;
- if self.bit_offset < 64 {
- v & mask
- } else {
- self.byte_offset += 8;
- self.bit_offset -= 64;
- self.reload_buffer_values();
- ((self.buffered_values << (num_bits - self.bit_offset)) | v) & mask
- }
- }
- }
-
- /// Gets at most `num` bits from this reader, and append them to the `dst` byte slice, starting
- /// at bit offset `offset`.
- ///
- /// Returns the actual number of bits appended. In case either the `dst` slice doesn't have
- /// enough space or the current reader doesn't have enough bits to consume, the returned value
- /// will be less than the input `num_bits`.
- ///
- /// # Preconditions
- /// * `offset` MUST < dst.len() * 8
- pub fn get_bits(&mut self, dst: &mut [u8], offset: usize, num_bits: usize) -> usize {
- debug_assert!(offset < dst.len() * 8);
-
- let remaining_bits = (self.total_bytes - self.byte_offset) * 8 - self.bit_offset;
- let num_bits_to_read = min(remaining_bits, min(num_bits, dst.len() * 8 - offset));
- let mut i = 0;
-
- // First consume all the remaining bits from the `buffered_values` if there're any.
- if likely(self.bit_offset != 0) {
- i += self.get_bits_buffered(dst, offset, num_bits_to_read);
- }
-
- debug_assert!(self.bit_offset == 0 || i == num_bits_to_read);
-
- // Check if there's opportunity to directly copy bytes using `memcpy`.
- if (offset + i).is_multiple_of(8) && i < num_bits_to_read {
- let num_bytes = (num_bits_to_read - i) / 8;
- let dst_byte_offset = (offset + i) / 8;
- if num_bytes > 0 {
- memcpy(
- &self.buffer[self.byte_offset..self.byte_offset + num_bytes],
- &mut dst[dst_byte_offset..],
- );
- i += num_bytes * 8;
- self.byte_offset += num_bytes;
- self.reload_buffer_values();
- }
- }
-
- debug_assert!(!(offset + i).is_multiple_of(8) || num_bits_to_read - i < 8);
-
- // Now copy the remaining bits if there's any.
- while i < num_bits_to_read {
- i += self.get_bits_buffered(dst, offset + i, num_bits_to_read - i);
- }
-
- num_bits_to_read
- }
-
- /// Consume at most `n` bits from `buffered_values`. Returns the actual number of bits consumed.
- ///
- /// # Postcondition
- /// - either bits from `buffered_values` are completely drained (i.e., `bit_offset` == 0)
- /// - OR the `num_bits` is < the number of remaining bits in `buffered_values` and thus the
- /// returned value is < `num_bits`.
- ///
- /// Either way, the returned value is in range [0, 64].
- #[inline]
- fn get_bits_buffered(&mut self, dst: &mut [u8], offset: usize, num_bits: usize) -> usize {
- if unlikely(num_bits == 0) {
- return 0;
- }
-
- let n = min(num_bits, 64 - self.bit_offset);
- let offset_i = offset / 8;
- let offset_r = offset % 8;
-
- // Extract the value to read out of the buffer
- let mut v = trailing_bits(self.buffered_values >> self.bit_offset, n);
-
- // Read the first byte always because n > 0
- dst[offset_i] |= (v << offset_r) as u8;
- v >>= 8 - offset_r;
-
- // Read the rest of the bytes
- ((offset_i + 1)..(offset_i + usize::div_ceil(n + offset_r, 8))).for_each(|i| {
- dst[i] |= v as u8;
- v >>= 8;
- });
-
- self.bit_offset += n;
- if self.bit_offset == 64 {
- self.byte_offset += 8;
- self.bit_offset -= 64;
- self.reload_buffer_values();
- }
-
- n
- }
-
- /// Skips at most `num` bits from this reader.
- ///
- /// Returns the actual number of bits skipped.
- pub fn skip_bits(&mut self, num_bits: usize) -> usize {
- let remaining_bits = (self.total_bytes - self.byte_offset) * 8 - self.bit_offset;
- let num_bits_to_read = min(remaining_bits, num_bits);
- let mut i = 0;
-
- // First skip all the remaining bits by updating the offsets of `buffered_values`.
- if likely(self.bit_offset != 0) {
- let n = 64 - self.bit_offset;
- if num_bits_to_read < n {
- self.bit_offset += num_bits_to_read;
- i = num_bits_to_read;
- } else {
- self.byte_offset += 8;
- self.bit_offset = 0;
- i = n;
- }
- }
-
- // Check if there's opportunity to skip by byte
- if i + 7 < num_bits_to_read {
- let num_bytes = (num_bits_to_read - i) / 8;
- i += num_bytes * 8;
- self.byte_offset += num_bytes;
- }
-
- if self.bit_offset == 0 {
- self.reload_buffer_values();
- }
-
- // Now skip the remaining bits if there's any.
- if i < num_bits_to_read {
- self.bit_offset += num_bits_to_read - i;
- }
-
- num_bits_to_read
- }
-
- /// Reads a batch of `u32` values encoded using `num_bits` of bits, into `dst`.
- ///
- /// # Safety
- ///
- /// This method asusumes the following:
- ///
- /// - the `num_bits` is <= 64
- /// - the remaining number of bits to read in this reader is >= `total * num_bits`.
- ///
- /// Undefined behavior will happen if any of the above assumptions is violated.
- ///
- /// Unlike `[get_batch]`, this method removes a few checks such as checking the remaining number
- /// of bits as well as checking the bit width for the element type in `dst`. Therefore, it is
- /// more efficient.
- pub unsafe fn get_u32_batch(&mut self, mut dst: *mut u32, total: usize, num_bits: usize) {
- let mut i = 0;
-
- // First align bit offset to byte offset
- if likely(self.bit_offset != 0) {
- while i < total && self.bit_offset != 0 {
- *dst = self.get_u32_value(num_bits);
- dst = dst.offset(1);
- i += 1;
- }
- }
-
- let in_buf = &self.buffer.as_slice()[self.byte_offset..];
- let mut in_ptr = in_buf as *const [u8] as *const u8 as *const u32;
- while total - i >= 32 {
- in_ptr = unpack32(in_ptr, dst, num_bits);
- self.byte_offset += 4 * num_bits;
- dst = dst.offset(32);
- i += 32;
- }
-
- self.reload_buffer_values();
- while i < total {
- *dst = self.get_u32_value(num_bits);
- dst = dst.offset(1);
- i += 1;
- }
- }
-
- pub fn get_batch<T: FromBytes>(&mut self, batch: &mut [T], num_bits: usize) -> usize {
- debug_assert!(num_bits <= 32);
- debug_assert!(num_bits <= size_of::<T>() * 8);
-
- let mut values_to_read = batch.len();
- let needed_bits = num_bits * values_to_read;
- let remaining_bits = (self.total_bytes - self.byte_offset) * 8 - self.bit_offset;
- if remaining_bits < needed_bits {
- values_to_read = remaining_bits / num_bits;
- }
-
- let mut i = 0;
-
- // First align bit offset to byte offset
- if likely(self.bit_offset != 0) {
- while i < values_to_read && self.bit_offset != 0 {
- batch[i] = self
- .get_value(num_bits)
- .expect("expected to have more data");
- i += 1;
- }
- }
-
- unsafe {
- let in_buf = &self.buffer.as_slice()[self.byte_offset..];
- let mut in_ptr = in_buf as *const [u8] as *const u8 as *const u32;
- // FIXME assert!(memory::is_ptr_aligned(in_ptr));
- if size_of::<T>() == 4 {
- while values_to_read - i >= 32 {
- let out_ptr = &mut batch[i..] as *mut [T] as *mut T as *mut u32;
- in_ptr = unpack32(in_ptr, out_ptr, num_bits);
- self.byte_offset += 4 * num_bits;
- i += 32;
- }
- } else {
- let mut out_buf = [0u32; 32];
- let out_ptr = &mut out_buf as &mut [u32] as *mut [u32] as *mut u32;
- while values_to_read - i >= 32 {
- in_ptr = unpack32(in_ptr, out_ptr, num_bits);
- self.byte_offset += 4 * num_bits;
- for n in 0..32 {
- // We need to copy from smaller size to bigger size to avoid
- // overwriting other memory regions.
- if size_of::<T>() > size_of::<u32>() {
- std::ptr::copy_nonoverlapping(
- out_buf[n..].as_ptr(),
- &mut batch[i] as *mut T as *mut u32,
- 1,
- );
- } else {
- std::ptr::copy_nonoverlapping(
- out_buf[n..].as_ptr() as *const T,
- &mut batch[i] as *mut T,
- 1,
- );
- }
- i += 1;
- }
- }
- }
- }
-
- debug_assert!(values_to_read - i < 32);
-
- self.reload_buffer_values();
- while i < values_to_read {
- batch[i] = self
- .get_value(num_bits)
- .expect("expected to have more data");
- i += 1;
- }
-
- values_to_read
- }
-
- /// Reads a `num_bytes`-sized value from this buffer and return it.
- /// `T` needs to be a little-endian native type. The value is assumed to be byte
- /// aligned so the bit reader will be advanced to the start of the next byte before
- /// reading the value.
- /// Returns `Some` if there's enough bytes left to form a value of `T`.
- /// Otherwise `None`.
- pub fn get_aligned<T: FromBytes>(&mut self, num_bytes: usize) -> Option<T> {
- debug_assert!(8 >= size_of::<T>());
- debug_assert!(num_bytes <= size_of::<T>());
-
- let bytes_read = self.bit_offset.div_ceil(8);
- if unlikely(self.byte_offset + bytes_read + num_bytes > self.total_bytes) {
- return None;
- }
-
- if bytes_read + num_bytes > 8 {
- // There may be still unread bytes in buffered_values; however, just reloading seems to
- // be faster than stitching the buffer with the next buffer based on micro benchmarks
- // because reloading logic can be simpler
-
- // Advance byte_offset to next unread byte
- self.byte_offset += bytes_read;
- // Reset buffered_values
- self.reload_buffer_values();
- self.bit_offset = 0
- } else {
- // Advance bit_offset to next unread byte
- self.bit_offset = bytes_read * 8;
- }
-
- let v = T::from(trailing_bits(
- self.buffered_values >> self.bit_offset,
- num_bytes * 8,
- ));
- self.bit_offset += num_bytes * 8;
-
- if self.bit_offset == 64 {
- self.byte_offset += 8;
- self.bit_offset -= 64;
- self.reload_buffer_values();
- }
-
- Some(v)
- }
-
- /// Reads a VLQ encoded (in little endian order) int from the stream.
- /// The encoded int must start at the beginning of a byte.
- ///
- /// Returns `None` if there's not enough bytes in the stream. `Some` otherwise.
- pub fn get_vlq_int(&mut self) -> Option<i64> {
- let mut shift = 0;
- let mut v: i64 = 0;
- while let Some(byte) = self.get_aligned::<u8>(1) {
- v |= ((byte & 0x7F) as i64) << shift;
- shift += 7;
- debug_assert!(
- shift <= MAX_VLQ_BYTE_LEN * 7,
- "Num of bytes exceed MAX_VLQ_BYTE_LEN ({MAX_VLQ_BYTE_LEN})"
- );
- if likely(byte & 0x80 == 0) {
- return Some(v);
- }
- }
- None
- }
-
- /// Reads a zigzag-VLQ encoded (in little endian order) int from the stream
- /// Zigzag-VLQ is a variant of VLQ encoding where negative and positive numbers are
- /// encoded in a zigzag fashion.
- /// See: https://developers.google.com/protocol-buffers/docs/encoding
- ///
- /// Note: the encoded int must start at the beginning of a byte.
- ///
- /// Returns `None` if the number of bytes there's not enough bytes in the stream.
- /// `Some` otherwise.
- #[inline]
- pub fn get_zigzag_vlq_int(&mut self) -> Option<i64> {
- self.get_vlq_int().map(|v| {
- let u = v as u64;
- (u >> 1) as i64 ^ -((u & 1) as i64)
- })
- }
-
- fn reload_buffer_values(&mut self) {
- let bytes_to_read = self.total_bytes - self.byte_offset;
- self.buffered_values = if 8 > bytes_to_read {
- read_num_bytes_u64(bytes_to_read, &self.buffer.as_slice()[self.byte_offset..])
- } else {
- read_u64(&self.buffer.as_slice()[self.byte_offset..])
- };
- }
-}
-
-impl From<Vec<u8>> for BitReader {
- #[inline]
- fn from(vec: Vec<u8>) -> Self {
- let len = vec.len();
- BitReader::new(Buffer::from(vec.as_slice()), len)
- }
-}
-
-/// Returns the nearest multiple of `factor` that is `>=` than `num`. Here `factor` must
-/// be a power of 2.
-///
-/// Copied from the arrow crate to make arrow optional
-pub fn round_upto_power_of_2(num: usize, factor: usize) -> usize {
- debug_assert!(factor > 0 && (factor & (factor - 1)) == 0);
- (num + (factor - 1)) & !(factor - 1)
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::parquet::util::test_common::*;
-
- use rand::{
- distr::{Distribution, StandardUniform},
- RngExt,
- };
- use std::fmt::Debug;
-
- #[test]
- fn test_read_num_bytes_u64() {
- let buffer: Vec<u8> = vec![0, 1, 2, 3, 4, 5, 6, 7];
- for size in 0..buffer.len() {
- assert_eq!(
- read_num_bytes_u64(size, &buffer),
- read_num_bytes!(u64, size, &buffer),
- );
- }
- }
-
- #[test]
- fn test_read_u64() {
- let buffer: Vec<u8> = vec![0, 1, 2, 3, 4, 5, 6, 7];
- assert_eq!(read_u64(&buffer), read_num_bytes!(u64, 8, &buffer),);
- }
-
- #[test]
- fn test_read_num_bytes_u32() {
- let buffer: Vec<u8> = vec![0, 1, 2, 3];
- for size in 0..buffer.len() {
- assert_eq!(
- read_num_bytes_u32(size, &buffer),
- read_num_bytes!(u32, size, &buffer),
- );
- }
- }
-
- #[test]
- fn test_read_u32() {
- let buffer: Vec<u8> = vec![0, 1, 2, 3];
- assert_eq!(read_u32(&buffer), read_num_bytes!(u32, 4, &buffer),);
- }
-
- #[test]
- fn test_bit_reader_get_byte_offset() {
- let buffer = vec![255; 10];
- let mut bit_reader = BitReader::from(buffer);
- assert_eq!(bit_reader.get_byte_offset(), 0); // offset (0 bytes, 0 bits)
- bit_reader.get_value::<i32>(6);
- assert_eq!(bit_reader.get_byte_offset(), 1); // offset (0 bytes, 6 bits)
- bit_reader.get_value::<i32>(10);
- assert_eq!(bit_reader.get_byte_offset(), 2); // offset (0 bytes, 16 bits)
- bit_reader.get_value::<i32>(20);
- assert_eq!(bit_reader.get_byte_offset(), 5); // offset (0 bytes, 36 bits)
- bit_reader.get_value::<i32>(30);
- assert_eq!(bit_reader.get_byte_offset(), 9); // offset (8 bytes, 2 bits)
- }
-
- #[test]
- fn test_bit_reader_get_value() {
- let buffer = vec![255, 0];
- let mut bit_reader = BitReader::from(buffer);
- assert_eq!(bit_reader.get_value::<i32>(1), Some(1));
- assert_eq!(bit_reader.get_value::<i32>(2), Some(3));
- assert_eq!(bit_reader.get_value::<i32>(3), Some(7));
- assert_eq!(bit_reader.get_value::<i32>(4), Some(3));
- }
-
- #[test]
- fn test_bit_reader_get_value_boundary() {
- let buffer = vec![10, 0, 0, 0, 20, 0, 30, 0, 0, 0, 40, 0];
- let mut bit_reader = BitReader::from(buffer);
- assert_eq!(bit_reader.get_value::<i64>(32), Some(10));
- assert_eq!(bit_reader.get_value::<i64>(16), Some(20));
- assert_eq!(bit_reader.get_value::<i64>(32), Some(30));
- assert_eq!(bit_reader.get_value::<i64>(16), Some(40));
- }
-
- #[test]
- fn test_bit_reader_get_aligned() {
- // 01110101 11001011
- let buffer = Buffer::from(&[0x75, 0xCB]);
- let mut bit_reader = BitReader::new_all(buffer.clone());
- assert_eq!(bit_reader.get_value::<i32>(3), Some(5));
- assert_eq!(bit_reader.get_aligned::<i32>(1), Some(203));
- assert_eq!(bit_reader.get_value::<i32>(1), None);
- bit_reader.reset(buffer);
- assert_eq!(bit_reader.get_aligned::<i32>(3), None);
- }
-
- #[test]
- fn test_bit_reader_get_vlq_int() {
- // 10001001 00000001 11110010 10110101 00000110
- let buffer: Vec<u8> = vec![0x89, 0x01, 0xF2, 0xB5, 0x06];
- let mut bit_reader = BitReader::from(buffer);
- assert_eq!(bit_reader.get_vlq_int(), Some(137));
- assert_eq!(bit_reader.get_vlq_int(), Some(105202));
- }
-
- #[test]
- fn test_bit_reader_get_zigzag_vlq_int() {
- let buffer: Vec<u8> = vec![0, 1, 2, 3];
- let mut bit_reader = BitReader::from(buffer);
- assert_eq!(bit_reader.get_zigzag_vlq_int(), Some(0));
- assert_eq!(bit_reader.get_zigzag_vlq_int(), Some(-1));
- assert_eq!(bit_reader.get_zigzag_vlq_int(), Some(1));
- assert_eq!(bit_reader.get_zigzag_vlq_int(), Some(-2));
- }
-
- #[test]
- fn test_set_bit() {
- let mut buffer = vec![0, 0, 0];
- set_bit(&mut buffer[..], 1);
- assert_eq!(buffer, vec![2, 0, 0]);
- set_bit(&mut buffer[..], 4);
- assert_eq!(buffer, vec![18, 0, 0]);
- unset_bit(&mut buffer[..], 1);
- assert_eq!(buffer, vec![16, 0, 0]);
- set_bit(&mut buffer[..], 10);
- assert_eq!(buffer, vec![16, 4, 0]);
- set_bit(&mut buffer[..], 10);
- assert_eq!(buffer, vec![16, 4, 0]);
- set_bit(&mut buffer[..], 11);
- assert_eq!(buffer, vec![16, 12, 0]);
- unset_bit(&mut buffer[..], 10);
- assert_eq!(buffer, vec![16, 8, 0]);
- }
-
- #[test]
- fn test_set_bits() {
- for offset in 0..=16 {
- for length in 0..=16 {
- let mut actual = vec![0, 0, 0, 0];
- set_bits(&mut actual[..], offset, length);
- let mut expected = vec![0, 0, 0, 0];
- for i in 0..length {
- set_bit(&mut expected, offset + i);
- }
- assert_eq!(actual, expected);
- }
- }
- }
-
- #[test]
- fn test_get_bit() {
- // 00001101
- assert!(get_bit(&[0b00001101], 0));
- assert!(!get_bit(&[0b00001101], 1));
- assert!(get_bit(&[0b00001101], 2));
- assert!(get_bit(&[0b00001101], 3));
-
- // 01001001 01010010
- assert!(get_bit(&[0b01001001, 0b01010010], 0));
- assert!(!get_bit(&[0b01001001, 0b01010010], 1));
- assert!(!get_bit(&[0b01001001, 0b01010010], 2));
- assert!(get_bit(&[0b01001001, 0b01010010], 3));
- assert!(!get_bit(&[0b01001001, 0b01010010], 4));
- assert!(!get_bit(&[0b01001001, 0b01010010], 5));
- assert!(get_bit(&[0b01001001, 0b01010010], 6));
- assert!(!get_bit(&[0b01001001, 0b01010010], 7));
- assert!(!get_bit(&[0b01001001, 0b01010010], 8));
- assert!(get_bit(&[0b01001001, 0b01010010], 9));
- assert!(!get_bit(&[0b01001001, 0b01010010], 10));
- assert!(!get_bit(&[0b01001001, 0b01010010], 11));
- assert!(get_bit(&[0b01001001, 0b01010010], 12));
- assert!(!get_bit(&[0b01001001, 0b01010010], 13));
- assert!(get_bit(&[0b01001001, 0b01010010], 14));
- assert!(!get_bit(&[0b01001001, 0b01010010], 15));
- }
-
- #[test]
- fn test_log2() {
- assert_eq!(log2(1), 0);
- assert_eq!(log2(2), 1);
- assert_eq!(log2(3), 2);
- assert_eq!(log2(4), 2);
- assert_eq!(log2(5), 3);
- assert_eq!(log2(5), 3);
- assert_eq!(log2(6), 3);
- assert_eq!(log2(7), 3);
- assert_eq!(log2(8), 3);
- assert_eq!(log2(9), 4);
- }
-
- #[test]
- fn test_skip() {
- let mut writer = BitWriter::new(5);
- let old_offset = writer.skip(1).expect("skip() should return OK");
- writer.put_aligned(42, 4);
- writer.put_aligned_offset(0x10, 1, old_offset);
- let result = writer.consume();
- assert_eq!(result.as_ref(), [0x10, 42, 0, 0, 0]);
-
- writer = BitWriter::new(4);
- let result = writer.skip(5);
- assert!(result.is_err());
- }
-
- #[test]
- fn test_get_next_byte_ptr() {
- let mut writer = BitWriter::new(5);
- {
- let first_byte = writer
- .get_next_byte_ptr(1)
- .expect("get_next_byte_ptr() should return OK");
- first_byte[0] = 0x10;
- }
- writer.put_aligned(42, 4);
- let result = writer.consume();
- assert_eq!(result.as_ref(), [0x10, 42, 0, 0, 0]);
- }
-
- #[test]
- fn test_consume_flush_buffer() {
- let mut writer1 = BitWriter::new(3);
- let mut writer2 = BitWriter::new(3);
- for i in 1..10 {
- writer1.put_value(i, 4);
- writer2.put_value(i, 4);
- }
- let res1 = writer1.flush_buffer();
- let res2 = writer2.consume();
- assert_eq!(res1, &res2[..]);
- }
-
- #[test]
- fn test_put_get_bool() {
- let len = 8;
- let mut writer = BitWriter::new(len);
-
- for i in 0..8 {
- let result = writer.put_value(i % 2, 1);
- assert!(result);
- }
-
- writer.flush();
- {
- let buffer = writer.buffer();
- assert_eq!(buffer[0], 0b10101010);
- }
-
- // Write 00110011
- for i in 0..8 {
- let result = match i {
- 0 | 1 | 4 | 5 => writer.put_value(false as u64, 1),
- _ => writer.put_value(true as u64, 1),
- };
- assert!(result);
- }
- writer.flush();
- {
- let buffer = writer.buffer();
- assert_eq!(buffer[0], 0b10101010);
- assert_eq!(buffer[1], 0b11001100);
- }
-
- let mut reader = BitReader::from(writer.consume());
-
- for i in 0..8 {
- let val = reader
- .get_value::<u8>(1)
- .expect("get_value() should return OK");
- assert_eq!(val, i % 2);
- }
-
- for i in 0..8 {
- let val = reader
- .get_value::<bool>(1)
- .expect("get_value() should return OK");
- match i {
- 0 | 1 | 4 | 5 => assert!(!val),
- _ => assert!(val),
- }
- }
- }
-
- #[test]
- fn test_put_value_roundtrip() {
- test_put_value_rand_numbers(32, 2);
- test_put_value_rand_numbers(32, 3);
- test_put_value_rand_numbers(32, 4);
- test_put_value_rand_numbers(32, 5);
- test_put_value_rand_numbers(32, 6);
- test_put_value_rand_numbers(32, 7);
- test_put_value_rand_numbers(32, 8);
- test_put_value_rand_numbers(64, 16);
- test_put_value_rand_numbers(64, 24);
- test_put_value_rand_numbers(64, 32);
- }
-
- fn test_put_value_rand_numbers(total: usize, num_bits: usize) {
- assert!(num_bits < 64);
- let num_bytes = num_bits.div_ceil(8);
- let mut writer = BitWriter::new(num_bytes * total);
- let values: Vec<u64> = random_numbers::<u64>(total)
- .iter()
- .map(|v| v & ((1 << num_bits) - 1))
- .collect();
- (0..total).for_each(|i| {
- assert!(
- writer.put_value(values[i], num_bits),
- "[{i}]: put_value() failed"
- );
- });
-
- let mut reader = BitReader::from(writer.consume());
- (0..total).for_each(|i| {
- let v = reader
- .get_value::<u64>(num_bits)
- .expect("get_value() should return OK");
- assert_eq!(
- v, values[i],
- "[{}]: expected {} but got {}",
- i, values[i], v
- );
- });
- }
-
- #[test]
- fn test_get_bits() {
- const NUM_BYTES: usize = 100;
-
- let mut vec = vec![0; NUM_BYTES];
- let total_num_bits = NUM_BYTES * 8;
- let v = random_bools(total_num_bits);
- (0..total_num_bits).for_each(|i| {
- if v[i] {
- set_bit(&mut vec, i);
- } else {
- unset_bit(&mut vec, i);
- }
- });
-
- let expected = vec.clone();
-
- // test reading the first time from a buffer
- for &(offset, num_bits) in [(0, 10), (2, 10), (8, 16), (25, 40), (7, 64)].iter() {
- let mut reader = BitReader::from(vec.clone());
- let mut buffer = vec![0; NUM_BYTES];
-
- let actual_bits_read = reader.get_bits(&mut buffer, offset, num_bits);
- let expected_bits_read = ::std::cmp::min(buffer.len() * 8 - offset, num_bits);
- assert_eq!(expected_bits_read, actual_bits_read);
-
- for i in 0..actual_bits_read {
- assert_eq!(get_bit(&expected, i), get_bit(&buffer, offset + i));
- }
- }
-
- // test reading consecutively from a buffer
- let mut reader = BitReader::from(vec);
- let mut buffer = vec![0; NUM_BYTES];
- let mut rng = rand::rng();
- let mut bits_read = 0;
-
- loop {
- if bits_read >= total_num_bits {
- break;
- }
- let n: u64 = rng.random();
- let num_bits = n % 20;
- bits_read += reader.get_bits(&mut buffer, bits_read, num_bits as usize);
- }
-
- assert_eq!(total_num_bits, bits_read);
- assert_eq!(&expected, &buffer);
- }
-
- #[test]
- fn test_skip_bits() {
- const NUM_BYTES: usize = 100;
-
- let mut vec = vec![0; NUM_BYTES];
- let total_num_bits = NUM_BYTES * 8;
- let v = random_bools(total_num_bits);
- (0..total_num_bits).for_each(|i| {
- if v[i] {
- set_bit(&mut vec, i);
- } else {
- unset_bit(&mut vec, i);
- }
- });
-
- let expected = vec.clone();
-
- // test skipping and check the next value
- let mut reader = BitReader::from(vec);
- let mut bits_read = 0;
- for &num_bits in [10, 60, 8].iter() {
- let actual_bits_read = reader.skip_bits(num_bits);
- assert_eq!(num_bits, actual_bits_read);
-
- bits_read += num_bits;
- assert_eq!(Some(get_bit(&expected, bits_read)), reader.get_value(1));
- bits_read += 1;
- }
-
- // test skipping consecutively
- let mut rng = rand::rng();
- loop {
- if bits_read >= total_num_bits {
- break;
- }
- let n: u64 = rng.random();
- let num_bits = n % 20;
- bits_read += reader.skip_bits(num_bits as usize);
- }
-
- assert_eq!(total_num_bits, bits_read);
- }
-
- #[test]
- fn test_get_batch() {
- const SIZE: &[usize] = &[1, 31, 32, 33, 128, 129];
- for s in SIZE {
- for i in 0..33 {
- match i {
- 0..=8 => test_get_batch_helper::<u8>(*s, i),
- 9..=16 => test_get_batch_helper::<u16>(*s, i),
- _ => test_get_batch_helper::<u32>(*s, i),
- }
- }
- }
- }
-
- fn test_get_batch_helper<T>(total: usize, num_bits: usize)
- where
- T: FromBytes + Default + Clone + Debug + Eq,
- {
- assert!(num_bits <= 32);
- let num_bytes = num_bits.div_ceil(8);
- let mut writer = BitWriter::new(num_bytes * total);
-
- let values: Vec<u32> = random_numbers::<u32>(total)
- .iter()
- .map(|v| v & ((1u64 << num_bits) - 1) as u32)
- .collect();
-
- // Generic values used to check against actual values read from `get_batch`.
- let expected_values: Vec<T> = values.iter().map(|v| from_ne_slice(v.as_bytes())).collect();
-
- (0..total).for_each(|i| {
- assert!(writer.put_value(values[i] as u64, num_bits));
- });
-
- let buf = writer.consume();
- let mut reader = BitReader::from(buf);
- let mut batch = vec![T::default(); values.len()];
- let values_read = reader.get_batch::<T>(&mut batch, num_bits);
- assert_eq!(values_read, values.len());
- for i in 0..batch.len() {
- assert_eq!(
- batch[i], expected_values[i],
- "num_bits = {num_bits}, index = {i}"
- );
- }
- }
-
- #[test]
- fn test_get_u32_batch() {
- const SIZE: &[usize] = &[1, 31, 32, 33, 128, 129];
- for total in SIZE {
- for num_bits in 1..33 {
- let num_bytes = usize::div_ceil(num_bits, 8);
- let mut writer = BitWriter::new(num_bytes * total);
-
- let values: Vec<u32> = random_numbers::<u32>(*total)
- .iter()
- .map(|v| v & ((1u64 << num_bits) - 1) as u32)
- .collect();
-
- (0..*total).for_each(|i| {
- assert!(writer.put_value(values[i] as u64, num_bits));
- });
-
- let buf = writer.consume();
- let mut reader = BitReader::from(buf);
- let mut batch = vec![0u32; values.len()];
- unsafe {
- reader.get_u32_batch(batch.as_mut_ptr(), *total, num_bits);
- }
- for i in 0..batch.len() {
- assert_eq!(batch[i], values[i], "num_bits = {num_bits}, index = {i}");
- }
- }
- }
- }
-
- #[test]
- fn test_put_aligned_roundtrip() {
- test_put_aligned_rand_numbers::<u8>(4, 3);
- test_put_aligned_rand_numbers::<u8>(16, 5);
- test_put_aligned_rand_numbers::<i16>(32, 7);
- test_put_aligned_rand_numbers::<i16>(32, 9);
- test_put_aligned_rand_numbers::<i32>(32, 11);
- test_put_aligned_rand_numbers::<i32>(32, 13);
- test_put_aligned_rand_numbers::<i64>(32, 17);
- test_put_aligned_rand_numbers::<i64>(32, 23);
- }
-
- fn test_put_aligned_rand_numbers<T>(total: usize, num_bits: usize)
- where
- T: Copy + FromBytes + AsBytes + Debug + PartialEq,
- StandardUniform: Distribution<T>,
- {
- assert!(num_bits <= 32);
- assert_eq!(total % 2, 0);
-
- let aligned_value_byte_width = std::mem::size_of::<T>();
- let value_byte_width = num_bits.div_ceil(8);
- let mut writer =
- BitWriter::new((total / 2) * (aligned_value_byte_width + value_byte_width));
- let values: Vec<u32> = random_numbers::<u32>(total / 2)
- .iter()
- .map(|v| v & ((1 << num_bits) - 1))
- .collect();
- let aligned_values = random_numbers::<T>(total / 2);
-
- for i in 0..total {
- let j = i / 2;
- if i % 2 == 0 {
- assert!(
- writer.put_value(values[j] as u64, num_bits),
- "[{i}]: put_value() failed"
- );
- } else {
- assert!(
- writer.put_aligned::<T>(aligned_values[j], aligned_value_byte_width),
- "[{i}]: put_aligned() failed"
- );
- }
- }
-
- let mut reader = BitReader::from(writer.consume());
- for i in 0..total {
- let j = i / 2;
- if i % 2 == 0 {
- let v = reader
- .get_value::<u64>(num_bits)
- .expect("get_value() should return OK");
- assert_eq!(
- v, values[j] as u64,
- "[{}]: expected {} but got {}",
- i, values[j], v
- );
- } else {
- let v = reader
- .get_aligned::<T>(aligned_value_byte_width)
- .expect("get_aligned() should return OK");
- assert_eq!(
- v, aligned_values[j],
- "[{}]: expected {:?} but got {:?}",
- i, aligned_values[j], v
- );
- }
- }
- }
-
- #[test]
- fn test_put_vlq_int() {
- let total = 64;
- let mut writer = BitWriter::new(total * 32);
- let values = random_numbers::<u32>(total);
- (0..total).for_each(|i| {
- assert!(
- writer.put_vlq_int(values[i] as u64),
- "[{i}]; put_vlq_int() failed"
- );
- });
-
- let mut reader = BitReader::from(writer.consume());
- (0..total).for_each(|i| {
- let v = reader
- .get_vlq_int()
- .expect("get_vlq_int() should return OK");
- assert_eq!(
- v as u32, values[i],
- "[{}]: expected {} but got {}",
- i, values[i], v
- );
- });
- }
-
- #[test]
- fn test_put_zigzag_vlq_int() {
- let total = 64;
- let mut writer = BitWriter::new(total * 32);
- let values = random_numbers::<i32>(total);
- (0..total).for_each(|i| {
- assert!(
- writer.put_zigzag_vlq_int(values[i] as i64),
- "[{i}]; put_zigzag_vlq_int() failed"
- );
- });
-
- let mut reader = BitReader::from(writer.consume());
- (0..total).for_each(|i| {
- let v = reader
- .get_zigzag_vlq_int()
- .expect("get_zigzag_vlq_int() should return OK");
- assert_eq!(
- v as i32, values[i],
- "[{}]: expected {} but got {}",
- i, values[i], v
- );
- });
- }
-}
diff --git a/native/core/src/common/buffer.rs b/native/core/src/common/buffer.rs
deleted file mode 100644
index 8b437a9f18..0000000000
--- a/native/core/src/common/buffer.rs
+++ /dev/null
@@ -1,366 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use crate::common::bit;
-use crate::execution::operators::ExecutionError;
-use arrow::buffer::Buffer as ArrowBuffer;
-use std::{
- alloc::{handle_alloc_error, Layout},
- ptr::NonNull,
- sync::Arc,
-};
-
-/// A buffer implementation. This is very similar to Arrow's [`MutableBuffer`] implementation,
-/// except that there are two modes depending on whether `owned` is true or false.
-///
-/// If `owned` is true, this behaves the same way as a Arrow [`MutableBuffer`], and the struct is
-/// the unique owner for the memory it wraps. The holder of this buffer can read or write the
-/// buffer, and the buffer itself will be released when it goes out of scope.
-///
-/// Also note that, in `owned` mode, the buffer is always filled with 0s, and its length is always
-/// equal to its capacity. It's up to the caller to decide which part of the buffer contains valid
-/// data.
-///
-/// If `owned` is false, this buffer is an alias to another buffer. The buffer itself becomes
-/// immutable and can only be read.
-#[derive(Debug)]
-pub struct CometBuffer {
- data: NonNull<u8>,
- len: usize,
- capacity: usize,
- /// Whether this buffer owns the data it points to.
- owned: bool,
- /// The allocation instance for this buffer.
- allocation: Arc<CometBufferAllocation>,
-}
-
-unsafe impl Sync for CometBuffer {}
-unsafe impl Send for CometBuffer {}
-
-/// All buffers are aligned to 64 bytes.
-const ALIGNMENT: usize = 64;
-
-impl CometBuffer {
- /// Initializes a owned buffer filled with 0.
- pub fn new(capacity: usize) -> Self {
- let aligned_capacity = bit::round_upto_power_of_2(capacity, ALIGNMENT);
- unsafe {
- let layout = Layout::from_size_align_unchecked(aligned_capacity, ALIGNMENT);
- let ptr = std::alloc::alloc_zeroed(layout);
- Self {
- data: NonNull::new(ptr).unwrap_or_else(|| handle_alloc_error(layout)),
- len: aligned_capacity,
- capacity: aligned_capacity,
- owned: true,
- allocation: Arc::new(CometBufferAllocation::new()),
- }
- }
- }
-
- /// Returns the capacity of this buffer.
- pub fn capacity(&self) -> usize {
- self.capacity
- }
-
- /// Returns the length (i.e., number of bytes) in this buffer.
- pub fn len(&self) -> usize {
- self.len
- }
-
- /// Whether this buffer is empty.
- pub fn is_empty(&self) -> bool {
- self.len == 0
- }
-
- /// Returns the data stored in this buffer as a slice.
- pub fn as_slice(&self) -> &[u8] {
- self
- }
-
- /// Returns the data stored in this buffer as a mutable slice.
- pub fn as_slice_mut(&mut self) -> &mut [u8] {
- debug_assert!(self.owned, "cannot modify un-owned buffer");
- self
- }
-
- /// Returns a raw pointer to this buffer's internal memory
- /// This pointer is guaranteed to be aligned along cache-lines.
- #[inline]
- pub const fn as_ptr(&self) -> *const u8 {
- self.data.as_ptr()
- }
-
- /// Returns a mutable raw pointer to this buffer's internal memory
- /// This pointer is guaranteed to be aligned along cache-lines.
- #[inline]
- pub fn as_mut_ptr(&mut self) -> *mut u8 {
- debug_assert!(self.owned, "cannot modify un-owned buffer");
- self.data.as_ptr()
- }
-
- /// Returns an immutable Arrow buffer on the content of this buffer.
- ///
- /// # Safety
- ///
- /// This function is highly unsafe since it leaks the raw pointer to the memory region that the
- /// originally this buffer is tracking. Because of this, the caller of this function is
- /// expected to make sure the returned immutable [`ArrowBuffer`] will never live longer than the
- /// this buffer. Otherwise it will result to dangling pointers.
- ///
- /// In the particular case of the columnar reader, we'll guarantee the above since the reader
- /// itself is closed at the very end, after the Spark task is completed (either successfully or
- /// unsuccessfully) through task completion listener.
- ///
- /// When re-using [`MutableVector`] in Comet native operators, across multiple input batches,
- /// because of the iterator-style pattern, the content of the original mutable buffer will only
- /// be updated once upstream operators fully consumed the previous output batch. For breaking
- /// operators, they are responsible for copying content out of the buffers.
- pub unsafe fn to_arrow(&self) -> Result<ArrowBuffer, ExecutionError> {
- let ptr = NonNull::new_unchecked(self.data.as_ptr());
- self.check_reference()?;
- Ok(ArrowBuffer::from_custom_allocation(
- ptr,
- self.len,
- Arc::<CometBufferAllocation>::clone(&self.allocation),
- ))
- }
-
- /// Checks if this buffer is exclusively owned by Comet. If not, an error is returned.
- /// We run this check when we want to update the buffer. If the buffer is also shared by
- /// other components, e.g. one DataFusion operator stores the buffer, Comet cannot safely
- /// modify the buffer.
- pub fn check_reference(&self) -> Result<(), ExecutionError> {
- if Arc::strong_count(&self.allocation) > 1 {
- Err(ExecutionError::GeneralError(
- "Error on modifying a buffer which is not exclusively owned by Comet".to_string(),
- ))
- } else {
- Ok(())
- }
- }
-
- /// Resets this buffer by filling all bytes with zeros.
- pub fn reset(&mut self) {
- debug_assert!(self.owned, "cannot modify un-owned buffer");
- unsafe {
- std::ptr::write_bytes(self.as_mut_ptr(), 0, self.len);
- }
- }
-
- /// Resize this buffer to the `new_capacity`. For additional bytes allocated, they are filled
- /// with 0. if `new_capacity` is less than the current capacity of this buffer, this is a no-op.
- #[inline(always)]
- pub fn resize(&mut self, new_capacity: usize) {
- assert!(self.owned, "cannot modify un-owned buffer");
- if new_capacity > self.len {
- let (ptr, new_capacity) =
- unsafe { Self::reallocate(self.data, self.capacity, new_capacity) };
- let diff = new_capacity - self.len;
- self.data = ptr;
- self.capacity = new_capacity;
- // write the value
- unsafe { self.data.as_ptr().add(self.len).write_bytes(0, diff) };
- self.len = new_capacity;
- }
- }
-
- unsafe fn reallocate(
- ptr: NonNull<u8>,
- old_capacity: usize,
- new_capacity: usize,
- ) -> (NonNull<u8>, usize) {
- let new_capacity = bit::round_upto_power_of_2(new_capacity, ALIGNMENT);
- let new_capacity = std::cmp::max(new_capacity, old_capacity * 2);
- let raw_ptr = std::alloc::realloc(
- ptr.as_ptr(),
- Layout::from_size_align_unchecked(old_capacity, ALIGNMENT),
- new_capacity,
- );
- let ptr = NonNull::new(raw_ptr).unwrap_or_else(|| {
- handle_alloc_error(Layout::from_size_align_unchecked(new_capacity, ALIGNMENT))
- });
- (ptr, new_capacity)
- }
-}
-
-impl Drop for CometBuffer {
- fn drop(&mut self) {
- if self.owned {
- unsafe {
- std::alloc::dealloc(
- self.data.as_ptr(),
- Layout::from_size_align_unchecked(self.capacity, ALIGNMENT),
- )
- }
- }
- }
-}
-
-impl PartialEq for CometBuffer {
- fn eq(&self, other: &CometBuffer) -> bool {
- if self.data.as_ptr() == other.data.as_ptr() {
- return true;
- }
- if self.len != other.len {
- return false;
- }
- if self.capacity != other.capacity {
- return false;
- }
- self.as_slice() == other.as_slice()
- }
-}
-
-impl std::ops::Deref for CometBuffer {
- type Target = [u8];
-
- fn deref(&self) -> &[u8] {
- unsafe { std::slice::from_raw_parts(self.as_ptr(), self.len) }
- }
-}
-
-impl std::ops::DerefMut for CometBuffer {
- fn deref_mut(&mut self) -> &mut [u8] {
- assert!(self.owned, "cannot modify un-owned buffer");
- unsafe { std::slice::from_raw_parts_mut(self.as_mut_ptr(), self.capacity) }
- }
-}
-
-#[derive(Debug)]
-struct CometBufferAllocation {}
-
-impl CometBufferAllocation {
- fn new() -> Self {
- Self {}
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use arrow::buffer::Buffer as ArrowBuffer;
-
- impl CometBuffer {
- pub fn from_ptr(ptr: *const u8, len: usize, capacity: usize) -> Self {
- assert_eq!(
- capacity % ALIGNMENT,
- 0,
- "input buffer is not aligned to {ALIGNMENT} bytes"
- );
- Self {
- data: NonNull::new(ptr as *mut u8).unwrap_or_else(|| {
- panic!(
- "cannot create CometBuffer from (ptr: {ptr:?}, len: {len}, capacity: {capacity}"
- )
- }),
- len,
- capacity,
- owned: false,
- allocation: Arc::new(CometBufferAllocation::new()),
- }
- }
-
- /// Extends this buffer (must be an owned buffer) by appending bytes from `src`,
- /// starting from `offset`.
- pub fn extend_from_slice(&mut self, offset: usize, src: &[u8]) {
- assert!(self.owned, "cannot modify un-owned buffer");
- assert!(
- offset + src.len() <= self.capacity(),
- "buffer overflow, offset = {}, src.len = {}, capacity = {}",
- offset,
- src.len(),
- self.capacity()
- );
-
- unsafe {
- let dst = self.data.as_ptr().add(offset);
- std::ptr::copy_nonoverlapping(src.as_ptr(), dst, src.len())
- }
- }
- }
-
- #[test]
- fn test_buffer_new() {
- let buf = CometBuffer::new(63);
- assert_eq!(64, buf.capacity());
- assert_eq!(64, buf.len());
- assert!(!buf.is_empty());
- }
-
- #[test]
- fn test_resize() {
- let mut buf = CometBuffer::new(1);
- assert_eq!(64, buf.capacity());
- assert_eq!(64, buf.len());
-
- buf.resize(100);
- assert_eq!(128, buf.capacity());
- assert_eq!(128, buf.len());
-
- // resize with less capacity is no-op
- buf.resize(20);
- assert_eq!(128, buf.capacity());
- assert_eq!(128, buf.len());
- }
-
- #[test]
- fn test_extend_from_slice() {
- let mut buf = CometBuffer::new(100);
- buf.extend_from_slice(0, b"hello");
- assert_eq!(b"hello", &buf.as_slice()[0..5]);
-
- buf.extend_from_slice(5, b" world");
- assert_eq!(b"hello world", &buf.as_slice()[0..11]);
-
- buf.reset();
- buf.extend_from_slice(0, b"hello arrow");
- assert_eq!(b"hello arrow", &buf.as_slice()[0..11]);
- }
-
- #[test]
- fn test_to_arrow() {
- let mut buf = CometBuffer::new(1);
-
- let str = b"aaaa bbbb cccc dddd";
- buf.extend_from_slice(0, str.as_slice());
-
- assert_eq!(64, buf.len());
- assert_eq!(64, buf.capacity());
- assert_eq!(b"aaaa bbbb cccc dddd", &buf.as_slice()[0..str.len()]);
-
- unsafe {
- let immutable_buf: ArrowBuffer = buf.to_arrow().unwrap();
- assert_eq!(64, immutable_buf.len());
- assert_eq!(str, &immutable_buf.as_slice()[0..str.len()]);
- }
- }
-
- #[test]
- fn test_unowned() {
- let arrow_buf = ArrowBuffer::from(b"hello comet");
- let buf = CometBuffer::from_ptr(arrow_buf.as_ptr(), arrow_buf.len(), arrow_buf.capacity());
-
- assert_eq!(11, buf.len());
- assert_eq!(64, buf.capacity());
- assert_eq!(b"hello comet", &buf.as_slice()[0..11]);
-
- unsafe {
- let arrow_buf2 = buf.to_arrow().unwrap();
- assert_eq!(arrow_buf, arrow_buf2);
- }
- }
-}
diff --git a/native/core/src/common/mod.rs b/native/core/src/common/mod.rs
deleted file mode 100644
index dc539879fa..0000000000
--- a/native/core/src/common/mod.rs
+++ /dev/null
@@ -1,21 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-pub mod bit;
-mod buffer;
-pub use buffer::*;
diff --git a/native/core/src/lib.rs b/native/core/src/lib.rs
index 7d0b6a5454..f4ae0b8834 100644
--- a/native/core/src/lib.rs
+++ b/native/core/src/lib.rs
@@ -65,8 +65,6 @@ pub mod jvm_bridge {
use errors::{try_unwrap_or_throw, CometError, CometResult};
-#[macro_use]
-pub mod common;
pub mod execution;
pub mod parquet;
// this module is for non release only. Intended for debugging/profiling purposes
diff --git a/native/core/src/parquet/data_type.rs b/native/core/src/parquet/data_type.rs
deleted file mode 100644
index 9a584d2359..0000000000
--- a/native/core/src/parquet/data_type.rs
+++ /dev/null
@@ -1,116 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use super::read::{PlainDecoding, PlainDictDecoding};
-
-pub trait DataType: PlainDecoding + PlainDictDecoding + 'static {}
-
-macro_rules! make_type {
- ($name:ident) => {
- pub struct $name {}
- impl DataType for $name {}
- };
-}
-
-make_type!(BoolType);
-make_type!(Int8Type);
-make_type!(UInt8Type);
-make_type!(Int16Type);
-make_type!(Int16ToDoubleType);
-make_type!(UInt16Type);
-make_type!(Int32Type);
-make_type!(Int32To64Type);
-make_type!(Int32ToDecimal64Type);
-make_type!(Int32ToDoubleType);
-make_type!(UInt32Type);
-make_type!(Int64Type);
-make_type!(Int64ToDecimal64Type);
-make_type!(UInt64Type);
-make_type!(FloatType);
-make_type!(DoubleType);
-make_type!(FloatToDoubleType);
-make_type!(ByteArrayType);
-make_type!(StringType);
-make_type!(Int32DecimalType);
-make_type!(Int64DecimalType);
-make_type!(FLBADecimalType);
-make_type!(FLBADecimal32Type);
-make_type!(FLBADecimal64Type);
-make_type!(FLBAType);
-make_type!(Int32DateType);
-make_type!(Int32TimestampMicrosType);
-make_type!(Int64TimestampMillisType);
-make_type!(Int64TimestampMicrosType);
-make_type!(Int96TimestampMicrosType);
-
-pub trait AsBytes {
- /// Returns the slice of bytes for an instance of this data type.
- fn as_bytes(&self) -> &[u8];
-}
-
-impl AsBytes for Vec<u8> {
- fn as_bytes(&self) -> &[u8] {
- self.as_slice()
- }
-}
-
-impl AsBytes for &str {
- fn as_bytes(&self) -> &[u8] {
- (self as &str).as_bytes()
- }
-}
-
-impl AsBytes for [u8] {
- fn as_bytes(&self) -> &[u8] {
- self
- }
-}
-
-impl AsBytes for str {
- fn as_bytes(&self) -> &[u8] {
- (self as &str).as_bytes()
- }
-}
-
-macro_rules! make_as_bytes {
- ($source_ty:ident) => {
- impl AsBytes for $source_ty {
- #[allow(clippy::size_of_in_element_count)]
- fn as_bytes(&self) -> &[u8] {
- unsafe {
- ::std::slice::from_raw_parts(
- self as *const $source_ty as *const u8,
- ::std::mem::size_of::<$source_ty>(),
- )
- }
- }
- }
- };
-}
-
-make_as_bytes!(bool);
-make_as_bytes!(i8);
-make_as_bytes!(u8);
-make_as_bytes!(i16);
-make_as_bytes!(u16);
-make_as_bytes!(i32);
-make_as_bytes!(u32);
-make_as_bytes!(i64);
-make_as_bytes!(u64);
-make_as_bytes!(f32);
-make_as_bytes!(f64);
-make_as_bytes!(i128);
diff --git a/native/core/src/parquet/mod.rs b/native/core/src/parquet/mod.rs
index c522b3a14e..806800cd59 100644
--- a/native/core/src/parquet/mod.rs
+++ b/native/core/src/parquet/mod.rs
@@ -15,18 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-pub mod data_type;
pub mod encryption_support;
-pub mod mutable_vector;
-pub use mutable_vector::*;
-#[macro_use]
-pub mod util;
pub mod parquet_exec;
pub mod parquet_read_cached_factory;
pub mod parquet_support;
-pub mod read;
pub mod schema_adapter;
+pub mod util;
mod cast_column;
mod objectstore;
@@ -44,7 +39,6 @@ use jni::{
Env, EnvUnowned,
};
-use self::util::jni::TypePromotionInfo;
use crate::execution::jni_api::get_runtime;
use crate::execution::metrics::utils::update_comet_metric;
use crate::execution::operators::ExecutionError;
@@ -53,281 +47,19 @@ use crate::execution::serde;
use crate::execution::spark_plan::SparkPlan;
use crate::execution::utils::SparkArrowConvert;
use crate::jvm_bridge::JVMClasses;
-use crate::parquet::data_type::AsBytes;
use crate::parquet::encryption_support::{CometEncryptionFactory, ENCRYPTION_FACTORY_ID};
use crate::parquet::parquet_exec::init_datasource_exec;
use crate::parquet::parquet_support::prepare_object_store_with_configs;
use arrow::array::{Array, RecordBatch};
-use arrow::buffer::MutableBuffer;
use datafusion::datasource::listing::PartitionedFile;
use datafusion::execution::SendableRecordBatchStream;
use datafusion::physical_plan::ExecutionPlan;
use datafusion::prelude::{SessionConfig, SessionContext};
use futures::{poll, StreamExt};
-use jni::objects::{JByteArray, JLongArray, JMap, JObject, JObjectArray, JString, ReleaseMode};
-use jni::sys::{jintArray, JNI_FALSE};
+use jni::objects::{JByteArray, JLongArray, JMap, JObject, JString, ReleaseMode};
+use jni::sys::JNI_FALSE;
use object_store::path::Path;
-use read::ColumnReader;
-use util::jni::{convert_column_descriptor, convert_encoding, deserialize_schema};
-
-/// Parquet read context maintained across multiple JNI calls.
-struct Context {
- pub column_reader: ColumnReader,
-}
-
-#[no_mangle]
-pub extern "system" fn Java_org_apache_comet_parquet_Native_initColumnReader(
- e: EnvUnowned,
- _jclass: JClass,
- primitive_type: jint,
- logical_type: jint,
- read_primitive_type: jint,
- jni_path: JObjectArray,
- max_dl: jint,
- max_rl: jint,
- bit_width: jint,
- read_bit_width: jint,
- is_signed: jboolean,
- type_length: jint,
- precision: jint,
- read_precision: jint,
- scale: jint,
- read_scale: jint,
- time_unit: jint,
- is_adjusted_utc: jboolean,
- batch_size: jint,
- use_decimal_128: jboolean,
- use_legacy_date_timestamp: jboolean,
-) -> jlong {
- try_unwrap_or_throw(&e, |env| {
- let desc = convert_column_descriptor(
- env,
- primitive_type,
- logical_type,
- max_dl,
- max_rl,
- bit_width,
- is_signed,
- type_length,
- precision,
- scale,
- time_unit,
- is_adjusted_utc,
- jni_path,
- )?;
- let promotion_info = TypePromotionInfo::new_from_jni(
- read_primitive_type,
- read_precision,
- read_scale,
- read_bit_width,
- );
- let ctx = Context {
- column_reader: ColumnReader::get(
- desc,
- promotion_info,
- batch_size as usize,
- use_decimal_128,
- use_legacy_date_timestamp,
- ),
- };
- let res = Box::new(ctx);
- Ok(Box::into_raw(res) as i64)
- })
-}
-
-/// # Safety
-/// This function is inheritly unsafe since it deals with raw pointers passed from JNI.
-#[no_mangle]
-pub unsafe extern "system" fn Java_org_apache_comet_parquet_Native_setDictionaryPage(
- e: EnvUnowned,
- _jclass: JClass,
- handle: jlong,
- page_value_count: jint,
- page_data: JByteArray,
- encoding: jint,
-) {
- try_unwrap_or_throw(&e, |env| {
- let reader = get_reader(handle)?;
-
- // convert value encoding ordinal to the native encoding definition
- let encoding = convert_encoding(encoding);
-
- // copy the input on-heap buffer to native
- let page_len = page_data.len(env)?;
- let mut buffer = MutableBuffer::from_len_zeroed(page_len);
- page_data.get_region(env, 0, from_u8_slice(buffer.as_slice_mut()))?;
-
- reader.set_dictionary_page(page_value_count as usize, buffer.into(), encoding);
- Ok(())
- })
-}
-
-/// # Safety
-/// This function is inheritly unsafe since it deals with raw pointers passed from JNI.
-#[no_mangle]
-pub unsafe extern "system" fn Java_org_apache_comet_parquet_Native_setPageV1(
- e: EnvUnowned,
- _jclass: JClass,
- handle: jlong,
- page_value_count: jint,
- page_data: JByteArray,
- value_encoding: jint,
-) {
- try_unwrap_or_throw(&e, |env| {
- let reader = get_reader(handle)?;
-
- // convert value encoding ordinal to the native encoding definition
- let encoding = convert_encoding(value_encoding);
-
- // copy the input on-heap buffer to native
- let page_len = page_data.len(env)?;
- let mut buffer = MutableBuffer::from_len_zeroed(page_len);
- page_data.get_region(env, 0, from_u8_slice(buffer.as_slice_mut()))?;
-
- reader.set_page_v1(page_value_count as usize, buffer.into(), encoding);
- Ok(())
- })
-}
-
-/// # Safety
-/// This function is inheritly unsafe since it deals with raw pointers passed from JNI.
-#[no_mangle]
-pub unsafe extern "system" fn Java_org_apache_comet_parquet_Native_setPageV2(
- e: EnvUnowned,
- _jclass: JClass,
- handle: jlong,
- page_value_count: jint,
- def_level_data: JByteArray,
- rep_level_data: JByteArray,
- value_data: JByteArray,
- value_encoding: jint,
-) {
- try_unwrap_or_throw(&e, |env| {
- let reader = get_reader(handle)?;
-
- // convert value encoding ordinal to the native encoding definition
- let encoding = convert_encoding(value_encoding);
-
- // copy the input on-heap buffer to native
- let dl_len = def_level_data.len(env)?;
- let mut dl_buffer = MutableBuffer::from_len_zeroed(dl_len);
- def_level_data.get_region(env, 0, from_u8_slice(dl_buffer.as_slice_mut()))?;
-
- let rl_len = rep_level_data.len(env)?;
- let mut rl_buffer = MutableBuffer::from_len_zeroed(rl_len);
- rep_level_data.get_region(env, 0, from_u8_slice(rl_buffer.as_slice_mut()))?;
-
- let v_len = value_data.len(env)?;
- let mut v_buffer = MutableBuffer::from_len_zeroed(v_len);
- value_data.get_region(env, 0, from_u8_slice(v_buffer.as_slice_mut()))?;
-
- reader.set_page_v2(
- page_value_count as usize,
- dl_buffer.into(),
- rl_buffer.into(),
- v_buffer.into(),
- encoding,
- );
- Ok(())
- })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_org_apache_comet_parquet_Native_resetBatch(
- env: EnvUnowned,
- _jclass: JClass,
- handle: jlong,
-) {
- try_unwrap_or_throw(&env, |_| {
- let reader = get_reader(handle)?;
- reader.reset_batch();
- Ok(())
- })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_org_apache_comet_parquet_Native_readBatch(
- e: EnvUnowned,
- _jclass: JClass,
- handle: jlong,
- batch_size: jint,
- null_pad_size: jint,
-) -> jintArray {
- try_unwrap_or_throw(&e, |env| {
- let reader = get_reader(handle)?;
- let (num_values, num_nulls) =
- reader.read_batch(batch_size as usize, null_pad_size as usize);
- let res = env.new_int_array(2)?;
- let buf: [i32; 2] = [num_values as i32, num_nulls as i32];
- res.set_region(env, 0, &buf)?;
- Ok(res.into_raw())
- })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_org_apache_comet_parquet_Native_skipBatch(
- env: EnvUnowned,
- _jclass: JClass,
- handle: jlong,
- batch_size: jint,
- discard: jboolean,
-) -> jint {
- try_unwrap_or_throw(&env, |_| {
- let reader = get_reader(handle)?;
- Ok(reader.skip_batch(batch_size as usize, discard) as jint)
- })
-}
-
-#[no_mangle]
-pub extern "system" fn Java_org_apache_comet_parquet_Native_currentBatch(
- e: EnvUnowned,
- _jclass: JClass,
- handle: jlong,
- array_addr: jlong,
- schema_addr: jlong,
-) {
- try_unwrap_or_throw(&e, |_env| {
- let ctx = get_context(handle)?;
- let reader = &mut ctx.column_reader;
- let data = reader.current_batch()?;
- data.move_to_spark(array_addr, schema_addr)
- .map_err(|e| e.into())
- })
-}
-
-#[inline]
-fn get_context<'a>(handle: jlong) -> Result<&'a mut Context, CometError> {
- unsafe {
- (handle as *mut Context)
- .as_mut()
- .ok_or_else(|| CometError::NullPointer("null context handle".to_string()))
- }
-}
-
-#[inline]
-fn get_reader<'a>(handle: jlong) -> Result<&'a mut ColumnReader, CometError> {
- Ok(&mut get_context(handle)?.column_reader)
-}
-
-#[no_mangle]
-pub extern "system" fn Java_org_apache_comet_parquet_Native_closeColumnReader(
- env: EnvUnowned,
- _jclass: JClass,
- handle: jlong,
-) {
- try_unwrap_or_throw(&env, |_| {
- unsafe {
- let ctx = get_context(handle)?;
- let _ = Box::from_raw(ctx);
- };
- Ok(())
- })
-}
-
-fn from_u8_slice(src: &mut [u8]) -> &mut [i8] {
- let raw_ptr = src.as_mut_ptr() as *mut i8;
- unsafe { std::slice::from_raw_parts_mut(raw_ptr, src.len()) }
-}
+use util::jni::deserialize_schema;
// TODO: (ARROW NATIVE) remove this if not needed.
enum ParquetReaderState {
@@ -397,31 +129,6 @@ pub fn get_object_store_options(
Ok(collected_map)
}
-/// # Safety
-/// This function is inherently unsafe since it deals with raw pointers passed from JNI.
-#[no_mangle]
-pub unsafe extern "system" fn Java_org_apache_comet_parquet_Native_validateObjectStoreConfig(
- e: EnvUnowned,
- _jclass: JClass,
- file_path: JString,
- object_store_options: JObject,
-) {
- try_unwrap_or_throw(&e, |env| {
- let session_config = SessionConfig::new();
- let planner =
- PhysicalPlanner::new(Arc::new(SessionContext::new_with_config(session_config)), 0);
- let session_ctx = planner.session_ctx();
- let path: String = file_path.try_to_string(env).unwrap();
- let object_store_config = get_object_store_options(env, object_store_options)?;
- let (_, _) = prepare_object_store_with_configs(
- session_ctx.runtime_env(),
- path.clone(),
- &object_store_config,
- )?;
- Ok(())
- })
-}
-
/// # Safety
/// This function is inherently unsafe since it deals with raw pointers passed from JNI.
#[no_mangle]
@@ -460,10 +167,10 @@ pub unsafe extern "system" fn Java_org_apache_comet_parquet_Native_initRecordBat
)?;
let required_schema_buffer = env.convert_byte_array(&required_schema)?;
- let required_schema = Arc::new(deserialize_schema(required_schema_buffer.as_bytes())?);
+ let required_schema = Arc::new(deserialize_schema(&required_schema_buffer)?);
let data_schema_buffer = env.convert_byte_array(&data_schema)?;
- let data_schema = Arc::new(deserialize_schema(data_schema_buffer.as_bytes())?);
+ let data_schema = Arc::new(deserialize_schema(&data_schema_buffer)?);
let data_filters = if !filter.is_null() {
let filter_buffer = env.convert_byte_array(&filter)?;
diff --git a/native/core/src/parquet/mutable_vector.rs b/native/core/src/parquet/mutable_vector.rs
deleted file mode 100644
index 5fe6aa1f3b..0000000000
--- a/native/core/src/parquet/mutable_vector.rs
+++ /dev/null
@@ -1,243 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use arrow::{array::ArrayData, datatypes::DataType as ArrowDataType};
-
-use crate::common::{bit, CometBuffer};
-use crate::execution::operators::ExecutionError;
-
-const DEFAULT_ARRAY_LEN: usize = 4;
-
-/// A mutable vector that can be re-used across batches, for Parquet read.
-///
-/// Note this class is similar to [`MutableVector`](crate::common::MutableVector). However, the
-/// latter has functionalities such as `ValueGetter`, `ValueSetter`. In addition, it represents
-/// String and Binary data using [`StringView`](crate::data_type::StringView), while this struct
-/// uses Arrow format to represent them.
-///
-/// TODO: unify the two structs in future
-#[derive(Debug)]
-pub struct ParquetMutableVector {
- /// The Arrow type for the elements of this vector.
- pub(crate) arrow_type: ArrowDataType,
-
- /// The number of total elements in this vector.
- pub(crate) num_values: usize,
-
- /// The number of null elements in this vector, must <= `num_values`.
- pub(crate) num_nulls: usize,
-
- /// The validity buffer of this Arrow vector. A bit set at position `i` indicates the `i`th
- /// element is not null. Otherwise, an unset bit at position `i` indicates the `i`th element is
- /// null.
- pub(crate) validity_buffer: CometBuffer,
-
- /// The value buffer of this Arrow vector. This could store either offsets if the vector
- /// is of list or struct type, or actual values themselves otherwise.
- pub(crate) value_buffer: CometBuffer,
-
- /// Child vectors for non-primitive types (e.g., list, struct).
- pub(crate) children: Vec<ParquetMutableVector>,
-
- /// Dictionary (i.e., values) associated with this vector. Only set if using dictionary
- /// encoding.
- pub(crate) dictionary: Option<Box<ParquetMutableVector>>,
-}
-
-impl ParquetMutableVector {
- pub fn new(capacity: usize, arrow_type: &ArrowDataType) -> Self {
- let bit_width = Self::bit_width(arrow_type);
- Self::new_with_bit_width(capacity, arrow_type.clone(), bit_width)
- }
-
- pub fn new_with_bit_width(
- capacity: usize,
- arrow_type: ArrowDataType,
- bit_width: usize,
- ) -> Self {
- let validity_len = capacity.div_ceil(8);
- let validity_buffer = CometBuffer::new(validity_len);
-
- let mut value_capacity = capacity;
- if Self::is_binary_type(&arrow_type) {
- // Arrow offset array needs to have one extra slot
- value_capacity += 1;
- }
- // Make sure the capacity is positive
- let len = usize::div_ceil(value_capacity * bit_width, 8);
- let mut value_buffer = CometBuffer::new(len);
-
- let mut children = Vec::new();
-
- match arrow_type {
- ArrowDataType::Binary | ArrowDataType::Utf8 => {
- children.push(ParquetMutableVector::new_with_bit_width(
- capacity,
- ArrowDataType::Int8,
- DEFAULT_ARRAY_LEN * 8,
- ));
- }
- _ => {}
- }
-
- if Self::is_binary_type(&arrow_type) {
- // Setup the first offset which is always 0.
- let zero: u32 = 0;
- bit::memcpy_value(&zero, 4, &mut value_buffer);
- }
-
- Self {
- arrow_type,
- num_values: 0,
- num_nulls: 0,
- validity_buffer,
- value_buffer,
- children,
- dictionary: None,
- }
- }
-
- /// Whether the given value at `idx` of this vector is null.
- #[inline]
- pub fn is_null(&self, idx: usize) -> bool {
- unsafe { !bit::get_bit_raw(self.validity_buffer.as_ptr(), idx) }
- }
-
- /// Resets this vector to the initial state.
- #[inline]
- pub fn reset(&mut self) {
- self.num_values = 0;
- self.num_nulls = 0;
- self.validity_buffer.reset();
- if Self::is_binary_type(&self.arrow_type) {
- // Reset the first offset to 0
- let zero: u32 = 0;
- bit::memcpy_value(&zero, 4, &mut self.value_buffer);
- // Also reset the child value vector
- let child = &mut self.children[0];
- child.reset();
- } else if Self::should_reset_value_buffer(&self.arrow_type) {
- self.value_buffer.reset();
- }
- }
-
- /// Appends a new null value to the end of this vector.
- #[inline]
- pub fn put_null(&mut self) {
- self.put_nulls(1)
- }
-
- /// Appends `n` null values to the end of this vector.
- #[inline]
- pub fn put_nulls(&mut self, n: usize) {
- // We need to update offset buffer for binary.
- if Self::is_binary_type(&self.arrow_type) {
- let mut offset = self.num_values * 4;
- let prev_offset_value = bit::read_u32(&self.value_buffer[offset..]);
- offset += 4;
- (0..n).for_each(|_| {
- bit::memcpy_value(&prev_offset_value, 4, &mut self.value_buffer[offset..]);
- offset += 4;
- });
- }
-
- self.num_nulls += n;
- self.num_values += n;
- }
-
- /// Returns the number of total values (including both null and non-null) of this vector.
- #[inline]
- pub fn num_values(&self) -> usize {
- self.num_values
- }
-
- /// Returns the number of null values of this vector.
- #[inline]
- pub fn num_nulls(&self) -> usize {
- self.num_nulls
- }
-
- /// Sets the dictionary of this to be `dict`.
- pub fn set_dictionary(&mut self, dict: ParquetMutableVector) {
- self.dictionary = Some(Box::new(dict))
- }
-
- /// Clones this into an Arrow [`ArrayData`](arrow::array::ArrayData). Note that the caller of
- /// this method MUST make sure the returned `ArrayData` won't live longer than this vector
- /// itself. Otherwise, dangling pointer may happen.
- ///
- /// # Safety
- ///
- /// This method is highly unsafe since it calls `CometBuffer::to_arrow` which leaks raw
- /// pointer to the memory region that are tracked by `CometBuffer`. Please see comments on
- /// `to_arrow` buffer to understand the motivation.
- pub fn get_array_data(&mut self) -> Result<ArrayData, ExecutionError> {
- unsafe {
- let data_type = if let Some(d) = &self.dictionary {
- ArrowDataType::Dictionary(
- Box::new(ArrowDataType::Int32),
- Box::new(d.arrow_type.clone()),
- )
- } else {
- self.arrow_type.clone()
- };
- let mut builder = ArrayData::builder(data_type)
- .len(self.num_values)
- .add_buffer(self.value_buffer.to_arrow()?)
- .null_bit_buffer(Some(self.validity_buffer.to_arrow()?))
- .null_count(self.num_nulls);
-
- if Self::is_binary_type(&self.arrow_type) && self.dictionary.is_none() {
- let child = &mut self.children[0];
- builder = builder.add_buffer(child.value_buffer.to_arrow()?);
- }
-
- if let Some(d) = &mut self.dictionary {
- builder = builder.add_child_data(d.get_array_data()?);
- }
- Ok(builder.build_unchecked())
- }
- }
-
- /// Returns the number of bits it takes to store one element of `arrow_type` in the value buffer
- /// of this vector.
- pub fn bit_width(arrow_type: &ArrowDataType) -> usize {
- match arrow_type {
- ArrowDataType::Boolean => 1,
- ArrowDataType::Int8 => 8,
- ArrowDataType::Int16 => 16,
- ArrowDataType::Int32 | ArrowDataType::Float32 | ArrowDataType::Date32 => 32,
- ArrowDataType::Int64 | ArrowDataType::Float64 | ArrowDataType::Timestamp(_, _) => 64,
- ArrowDataType::FixedSizeBinary(type_length) => *type_length as usize * 8,
- ArrowDataType::Decimal128(..) => 128, // Arrow stores decimal with 16 bytes
- ArrowDataType::Binary | ArrowDataType::Utf8 => 32, // Only count offset size
- dt => panic!("Unsupported Arrow data type: {dt:?}"),
- }
- }
-
- #[inline]
- fn is_binary_type(dt: &ArrowDataType) -> bool {
- matches!(dt, ArrowDataType::Binary | ArrowDataType::Utf8)
- }
-
- #[inline]
- fn should_reset_value_buffer(dt: &ArrowDataType) -> bool {
- // - Boolean type expects have a zeroed value buffer
- // - Decimal may pad buffer with 0xff so we need to clear them before a new batch
- matches!(dt, ArrowDataType::Boolean | ArrowDataType::Decimal128(_, _))
- }
-}
diff --git a/native/core/src/parquet/read/column.rs b/native/core/src/parquet/read/column.rs
deleted file mode 100644
index cdbb76bbf0..0000000000
--- a/native/core/src/parquet/read/column.rs
+++ /dev/null
@@ -1,839 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{marker::PhantomData, sync::Arc};
-
-use arrow::{
- array::ArrayData,
- buffer::Buffer,
- datatypes::{DataType as ArrowDataType, TimeUnit},
-};
-
-use parquet::{
- basic::{Encoding, LogicalType, TimeUnit as ParquetTimeUnit, Type as PhysicalType},
- schema::types::{ColumnDescPtr, ColumnDescriptor},
-};
-
-use crate::parquet::{data_type::*, util::jni::TypePromotionInfo, ParquetMutableVector};
-
-use super::{
- levels::LevelDecoder,
- values::{get_decoder, Decoder},
- ReadOptions,
-};
-
-use crate::common::bit::log2;
-use crate::execution::operators::ExecutionError;
-
-/// Maximum number of decimal digits an i32 can represent
-const DECIMAL_MAX_INT_DIGITS: i32 = 9;
-
-/// Maximum number of decimal digits an i64 can represent
-const DECIMAL_MAX_LONG_DIGITS: i32 = 18;
-
-pub enum ColumnReader {
- BoolColumnReader(TypedColumnReader<BoolType>),
- Int8ColumnReader(TypedColumnReader<Int8Type>),
- UInt8ColumnReader(TypedColumnReader<UInt8Type>),
- Int16ColumnReader(TypedColumnReader<Int16Type>),
- Int16ToDoubleColumnReader(TypedColumnReader<Int16ToDoubleType>),
- UInt16ColumnReader(TypedColumnReader<UInt16Type>),
- Int32ColumnReader(TypedColumnReader<Int32Type>),
- Int32To64ColumnReader(TypedColumnReader<Int32To64Type>),
- Int32ToDecimal64ColumnReader(TypedColumnReader<Int32ToDecimal64Type>),
- Int32ToDoubleColumnReader(TypedColumnReader<Int32ToDoubleType>),
- UInt32ColumnReader(TypedColumnReader<UInt32Type>),
- Int32DecimalColumnReader(TypedColumnReader<Int32DecimalType>),
- Int32DateColumnReader(TypedColumnReader<Int32DateType>),
- Int32TimestampMicrosColumnReader(TypedColumnReader<Int32TimestampMicrosType>),
- Int64ColumnReader(TypedColumnReader<Int64Type>),
- Int64ToDecimal64ColumnReader(TypedColumnReader<Int64ToDecimal64Type>),
- UInt64DecimalColumnReader(TypedColumnReader<UInt64Type>),
- Int64DecimalColumnReader(TypedColumnReader<Int64DecimalType>),
- Int64TimestampMillisColumnReader(TypedColumnReader<Int64TimestampMillisType>),
- Int64TimestampMicrosColumnReader(TypedColumnReader<Int64TimestampMicrosType>),
- Int64TimestampNanosColumnReader(TypedColumnReader<Int64Type>),
- Int96ColumnReader(TypedColumnReader<Int96TimestampMicrosType>),
- FloatColumnReader(TypedColumnReader<FloatType>),
- FloatToDoubleColumnReader(TypedColumnReader<FloatToDoubleType>),
- DoubleColumnReader(TypedColumnReader<DoubleType>),
- ByteArrayColumnReader(TypedColumnReader<ByteArrayType>),
- StringColumnReader(TypedColumnReader<StringType>),
- FLBADecimalColumnReader(TypedColumnReader<FLBADecimalType>),
- FLBADecimal32ColumnReader(TypedColumnReader<FLBADecimal32Type>),
- FLBADecimal64ColumnReader(TypedColumnReader<FLBADecimal64Type>),
- FLBAColumnReader(TypedColumnReader<FLBAType>),
-}
-
-impl ColumnReader {
- /// Creates a new column reader according to the input `desc`.
- ///
- /// - `desc`: The actual descriptor for the underlying Parquet files
- /// - `promotion_info`: Extra information about type promotion. This is passed in to support
- /// schema evolution, e.g., int -> long, where Parquet type is int but Spark type is long.
- /// - `use_decimal_128`: Whether to read small precision decimals as `i128` instead of as `i32`
- /// or `i64` as Spark does
- /// - `use_legacy_date_timestamp_or_ntz`: Whether to read dates/timestamps that were written
- /// using the legacy Julian/Gregorian hybrid calendar as it is. If false, exceptions will be
- /// thrown. If the spark type is TimestampNTZ, this should be true.
- pub fn get(
- desc: ColumnDescriptor,
- promotion_info: TypePromotionInfo,
- capacity: usize,
- use_decimal_128: bool,
- use_legacy_date_timestamp_or_ntz: bool,
- ) -> Self {
- let read_options = ReadOptions {
- use_legacy_date_timestamp_or_ntz,
- };
- macro_rules! typed_reader {
- ($reader_ty:ident, $arrow_ty:ident) => {
- Self::$reader_ty(TypedColumnReader::new(
- desc,
- capacity,
- ArrowDataType::$arrow_ty,
- read_options,
- ))
- };
- ($reader_ty:ident, $arrow_ty:expr) => {
- Self::$reader_ty(TypedColumnReader::new(
- desc,
- capacity,
- $arrow_ty,
- read_options,
- ))
- };
- }
-
- match desc.physical_type() {
- PhysicalType::BOOLEAN => typed_reader!(BoolColumnReader, Boolean),
- PhysicalType::INT32 => {
- if let Some(ref logical_type) = desc.logical_type_ref() {
- match logical_type {
- lt @ LogicalType::Integer {
- bit_width,
- is_signed,
- } => match (bit_width, is_signed) {
- (8, true) => match promotion_info.physical_type {
- PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- if promotion_info.precision <= DECIMAL_MAX_INT_DIGITS
- && promotion_info.scale < 1
- {
- typed_reader!(Int32ColumnReader, Int32)
- } else if promotion_info.precision <= DECIMAL_MAX_LONG_DIGITS {
- typed_reader!(
- Int32ToDecimal64ColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- } else {
- typed_reader!(
- Int32DecimalColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- }
- }
- // promote byte to short
- PhysicalType::INT32 if promotion_info.bit_width == 16 => {
- typed_reader!(Int16ColumnReader, Int16)
- }
- // promote byte to int
- PhysicalType::INT32 if promotion_info.bit_width == 32 => {
- typed_reader!(Int32ColumnReader, Int32)
- }
- // promote byte to long
- PhysicalType::INT64 => typed_reader!(Int32To64ColumnReader, Int64),
- _ => typed_reader!(Int8ColumnReader, Int8),
- },
- (8, false) => typed_reader!(UInt8ColumnReader, Int16),
- (16, true) => match promotion_info.physical_type {
- PhysicalType::DOUBLE => {
- typed_reader!(Int16ToDoubleColumnReader, Float64)
- }
- // promote short to long
- PhysicalType::INT64 => {
- typed_reader!(Int32To64ColumnReader, Int64)
- }
- PhysicalType::INT32 if promotion_info.bit_width == 32 => {
- typed_reader!(Int32ColumnReader, Int32)
- }
- PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- if promotion_info.precision <= DECIMAL_MAX_INT_DIGITS
- && promotion_info.scale < 1
- {
- typed_reader!(Int32ColumnReader, Int32)
- } else if promotion_info.precision <= DECIMAL_MAX_LONG_DIGITS {
- typed_reader!(
- Int32ToDecimal64ColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- } else {
- typed_reader!(
- Int32DecimalColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- }
- }
- _ => typed_reader!(Int16ColumnReader, Int16),
- },
- (16, false) => typed_reader!(UInt16ColumnReader, Int32),
- (32, true) => match promotion_info.physical_type {
- PhysicalType::INT32 if promotion_info.bit_width == 16 => {
- typed_reader!(Int16ColumnReader, Int16)
- }
- _ => typed_reader!(Int32ColumnReader, Int32),
- },
- (32, false) => typed_reader!(UInt32ColumnReader, Int64),
- _ => unimplemented!("Unsupported INT32 annotation: {:?}", lt),
- },
- LogicalType::Decimal {
- scale: _,
- precision: _,
- } => {
- if use_decimal_128 || promotion_info.precision > DECIMAL_MAX_LONG_DIGITS
- {
- typed_reader!(
- Int32DecimalColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- } else {
- typed_reader!(
- Int32ToDecimal64ColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- }
- }
- LogicalType::Date => match promotion_info.physical_type {
- PhysicalType::INT64 => typed_reader!(
- Int32TimestampMicrosColumnReader,
- ArrowDataType::Timestamp(TimeUnit::Microsecond, None)
- ),
- _ => typed_reader!(Int32DateColumnReader, Date32),
- },
- lt => unimplemented!("Unsupported logical type for INT32: {:?}", lt),
- }
- } else {
- // We support type promotion from int to long
- match promotion_info.physical_type {
- PhysicalType::INT32 if promotion_info.bit_width == 16 => {
- typed_reader!(Int16ColumnReader, Int16)
- }
- PhysicalType::INT32 => typed_reader!(Int32ColumnReader, Int32),
- PhysicalType::INT64 => typed_reader!(Int32To64ColumnReader, Int64),
- PhysicalType::DOUBLE => typed_reader!(Int32ToDoubleColumnReader, Float64),
- PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- if promotion_info.precision <= DECIMAL_MAX_INT_DIGITS
- && promotion_info.scale < 1
- {
- typed_reader!(Int32ColumnReader, Int32)
- } else if promotion_info.precision <= DECIMAL_MAX_LONG_DIGITS {
- typed_reader!(
- Int32ToDecimal64ColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- } else {
- typed_reader!(
- Int32DecimalColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- }
- }
- t => unimplemented!("Unsupported read physical type for INT32: {}", t),
- }
- }
- }
- PhysicalType::INT64 => {
- if let Some(ref logical_type) = desc.logical_type_ref() {
- match logical_type {
- lt @ LogicalType::Integer {
- bit_width,
- is_signed,
- } => match (bit_width, is_signed) {
- (64, true) => typed_reader!(Int64ColumnReader, Int64),
- (64, false) => typed_reader!(
- UInt64DecimalColumnReader,
- ArrowDataType::Decimal128(20u8, 0i8)
- ),
- _ => panic!("Unsupported INT64 annotation: {lt:?}"),
- },
- LogicalType::Decimal {
- scale: _,
- precision: _,
- } => {
- if use_decimal_128 || promotion_info.precision > DECIMAL_MAX_LONG_DIGITS
- {
- typed_reader!(
- Int64DecimalColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- } else {
- typed_reader!(
- Int64ToDecimal64ColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- }
- }
- LogicalType::Timestamp {
- is_adjusted_to_u_t_c,
- unit,
- } => {
- // To be consistent with Spark, we always store as micro-second and
- // convert milli-second to it.
- let time_unit = TimeUnit::Microsecond;
- let time_zone = if *is_adjusted_to_u_t_c {
- Some("UTC".to_string().into())
- } else {
- None
- };
- match unit {
- ParquetTimeUnit::MILLIS => {
- typed_reader!(
- Int64TimestampMillisColumnReader,
- ArrowDataType::Timestamp(time_unit, time_zone)
- )
- }
- ParquetTimeUnit::MICROS => {
- typed_reader!(
- Int64TimestampMicrosColumnReader,
- ArrowDataType::Timestamp(time_unit, time_zone)
- )
- }
- ParquetTimeUnit::NANOS => {
- typed_reader!(
- Int64TimestampNanosColumnReader,
- ArrowDataType::Int64
- )
- }
- }
- }
- lt => panic!("Unsupported logical type for INT64: {lt:?}"),
- }
- } else {
- match promotion_info.physical_type {
- PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- if promotion_info.precision <= DECIMAL_MAX_LONG_DIGITS
- && promotion_info.scale < 1
- {
- typed_reader!(Int64ColumnReader, Int64)
- } else {
- typed_reader!(
- Int64DecimalColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- }
- }
- // By default it is INT(64, true)
- _ => typed_reader!(Int64ColumnReader, Int64),
- }
- }
- }
- PhysicalType::INT96 => {
- typed_reader!(
- Int96ColumnReader,
- ArrowDataType::Timestamp(TimeUnit::Microsecond, Some("UTC".to_string().into()))
- )
- }
- PhysicalType::FLOAT => match promotion_info.physical_type {
- // We support type promotion from float to double
- PhysicalType::FLOAT => typed_reader!(FloatColumnReader, Float32),
- PhysicalType::DOUBLE => typed_reader!(FloatToDoubleColumnReader, Float64),
- t => panic!("Unsupported read physical type: {t} for FLOAT"),
- },
-
- PhysicalType::DOUBLE => typed_reader!(DoubleColumnReader, Float64),
- PhysicalType::BYTE_ARRAY => {
- if let Some(logical_type) = desc.logical_type_ref() {
- match logical_type {
- LogicalType::String => typed_reader!(StringColumnReader, Utf8),
- // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
- // "enum type should interpret ENUM annotated field as a UTF-8"
- LogicalType::Enum => typed_reader!(StringColumnReader, Utf8),
- lt => panic!("Unsupported logical type for BYTE_ARRAY: {lt:?}"),
- }
- } else {
- typed_reader!(ByteArrayColumnReader, Binary)
- }
- }
- PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- if let Some(logical_type) = desc.logical_type_ref() {
- match logical_type {
- LogicalType::Decimal {
- precision,
- scale: _,
- } => {
- if !use_decimal_128 && precision <= &DECIMAL_MAX_INT_DIGITS {
- typed_reader!(FLBADecimal32ColumnReader, Int32)
- } else if !use_decimal_128
- && promotion_info.precision <= DECIMAL_MAX_LONG_DIGITS
- {
- typed_reader!(
- FLBADecimal64ColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- } else {
- typed_reader!(
- FLBADecimalColumnReader,
- ArrowDataType::Decimal128(
- promotion_info.precision as u8,
- promotion_info.scale as i8
- )
- )
- }
- }
- LogicalType::Uuid => {
- let type_length = desc.type_length();
- typed_reader!(
- FLBAColumnReader,
- ArrowDataType::FixedSizeBinary(type_length)
- )
- }
- t => panic!("Unsupported logical type for FIXED_LEN_BYTE_ARRAY: {t:?}"),
- }
- } else {
- let type_length = desc.type_length();
- typed_reader!(
- FLBAColumnReader,
- ArrowDataType::FixedSizeBinary(type_length)
- )
- }
- }
- }
- }
-}
-
-macro_rules! make_func {
- ($self:ident, $func:ident $(,$args:ident)*) => ({
- match *$self {
- Self::BoolColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int8ColumnReader(ref typed) => typed.$func($($args),*),
- Self::UInt8ColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int16ColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int16ToDoubleColumnReader(ref typed) => typed.$func($($args), *),
- Self::UInt16ColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int32ColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int32To64ColumnReader(ref typed) => typed.$func($($args), *),
- Self::Int32ToDecimal64ColumnReader(ref typed) => typed.$func($($args), *),
- Self::Int32ToDoubleColumnReader(ref typed) => typed.$func($($args), *),
- Self::UInt32ColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int32DateColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int32DecimalColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int32TimestampMicrosColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int64ColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int64ToDecimal64ColumnReader(ref typed) => typed.$func($($args), *),
- Self::UInt64DecimalColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int64DecimalColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int64TimestampMillisColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int64TimestampMicrosColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int64TimestampNanosColumnReader(ref typed) => typed.$func($($args),*),
- Self::FloatColumnReader(ref typed) => typed.$func($($args),*),
- Self::DoubleColumnReader(ref typed) => typed.$func($($args),*),
- Self::FloatToDoubleColumnReader(ref typed) => typed.$func($($args),*),
- Self::ByteArrayColumnReader(ref typed) => typed.$func($($args),*),
- Self::StringColumnReader(ref typed) => typed.$func($($args),*),
- Self::FLBADecimalColumnReader(ref typed) => typed.$func($($args),*),
- Self::FLBADecimal32ColumnReader(ref typed) => typed.$func($($args),*),
- Self::FLBADecimal64ColumnReader(ref typed) => typed.$func($($args),*),
- Self::FLBAColumnReader(ref typed) => typed.$func($($args),*),
- Self::Int96ColumnReader(ref typed) => typed.$func($($args),*),
- }
- });
-}
-
-macro_rules! make_func_mut {
- ($self:ident, $func:ident $(,$args:ident)*) => ({
- match *$self {
- Self::BoolColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int8ColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::UInt8ColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int16ColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int16ToDoubleColumnReader(ref mut typed) => typed.$func($($args), *),
- Self::UInt16ColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int32ColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int32To64ColumnReader(ref mut typed) => typed.$func($($args), *),
- Self::Int32ToDecimal64ColumnReader(ref mut typed) => typed.$func($($args), *),
- Self::Int32ToDoubleColumnReader(ref mut typed) => typed.$func($($args), *),
- Self::UInt32ColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int32DateColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int32DecimalColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int32TimestampMicrosColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int64ColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int64ToDecimal64ColumnReader(ref mut typed) => typed.$func($($args), *),
- Self::UInt64DecimalColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int64DecimalColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int64TimestampMillisColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int64TimestampMicrosColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int64TimestampNanosColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::FloatColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::DoubleColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::FloatToDoubleColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::ByteArrayColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::StringColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::FLBADecimalColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::FLBADecimal32ColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::FLBADecimal64ColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::FLBAColumnReader(ref mut typed) => typed.$func($($args),*),
- Self::Int96ColumnReader(ref mut typed) => typed.$func($($args),*),
- }
- });
-}
-
-impl ColumnReader {
- #[inline]
- pub fn get_descriptor(&self) -> &ColumnDescriptor {
- make_func!(self, get_descriptor)
- }
-
- #[inline]
- pub fn set_dictionary_page(
- &mut self,
- page_value_count: usize,
- page_data: Buffer,
- encoding: Encoding,
- ) {
- make_func_mut!(
- self,
- set_dictionary_page,
- page_value_count,
- page_data,
- encoding
- )
- }
-
- #[inline]
- pub fn set_page_v1(&mut self, page_value_count: usize, page_data: Buffer, encoding: Encoding) {
- make_func_mut!(self, set_page_v1, page_value_count, page_data, encoding)
- }
-
- #[inline]
- pub fn set_page_v2(
- &mut self,
- page_value_count: usize,
- def_level_data: Buffer,
- rep_level_data: Buffer,
- value_data: Buffer,
- encoding: Encoding,
- ) {
- make_func_mut!(
- self,
- set_page_v2,
- page_value_count,
- def_level_data,
- rep_level_data,
- value_data,
- encoding
- )
- }
-
- #[inline]
- pub fn reset_batch(&mut self) {
- make_func_mut!(self, reset_batch)
- }
-
- #[inline]
- pub fn current_batch(&mut self) -> Result<ArrayData, ExecutionError> {
- make_func_mut!(self, current_batch)
- }
-
- #[inline]
- pub fn read_batch(&mut self, total: usize, null_pad_size: usize) -> (usize, usize) {
- make_func_mut!(self, read_batch, total, null_pad_size)
- }
-
- #[inline]
- pub fn skip_batch(&mut self, total: usize, put_nulls: bool) -> usize {
- make_func_mut!(self, skip_batch, total, put_nulls)
- }
-}
-
-/// A batched reader for a primitive Parquet column.
-pub struct TypedColumnReader<T: DataType> {
- desc: ColumnDescPtr,
- arrow_type: ArrowDataType,
- rep_level_decoder: Option<LevelDecoder>,
- def_level_decoder: Option<LevelDecoder>,
- value_decoder: Option<Box<dyn Decoder>>,
-
- /// The remaining number of values to read in the current page
- num_values_in_page: usize,
- /// The value vector for this column reader; reused across batches.
- vector: ParquetMutableVector,
- /// The batch size for this column vector.
- capacity: usize,
- /// Number of bits used to represent one value in Parquet.
- bit_width: usize,
- // Options for reading Parquet
- read_options: ReadOptions,
-
- /// Marker to allow `T` in the generic parameter of the struct.
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> TypedColumnReader<T> {
- pub fn new(
- desc: ColumnDescriptor,
- capacity: usize,
- arrow_type: ArrowDataType,
- read_options: ReadOptions,
- ) -> Self {
- let vector = ParquetMutableVector::new(capacity, &arrow_type);
- let bit_width = ParquetMutableVector::bit_width(&arrow_type);
- Self {
- desc: Arc::new(desc),
- arrow_type,
- rep_level_decoder: None,
- def_level_decoder: None,
- value_decoder: None,
- num_values_in_page: 0,
- vector,
- capacity,
- bit_width,
- read_options,
- _phantom: PhantomData,
- }
- }
-
- #[inline]
- pub fn get_descriptor(&self) -> &ColumnDescriptor {
- &self.desc
- }
-
- /// Reset the current batch. This will clear all the content of the current columnar vector as
- /// well as reset all of its internal states.
- #[inline]
- pub fn reset_batch(&mut self) {
- self.vector.reset()
- }
-
- /// Returns the current batch that's been constructed.
- ///
- /// Note: the caller must make sure the returned Arrow vector is fully consumed before calling
- /// `read_batch` again.
- #[inline]
- pub fn current_batch(&mut self) -> Result<ArrayData, ExecutionError> {
- self.vector.get_array_data()
- }
-
- /// Reads a batch of at most `total` values from the current page this reader has. Returns a
- /// tuple where the first element is the actual number of values read (including both nulls and
- /// non-nulls), and the second element is the actual number of nulls read.
- ///
- /// Pad nulls for the amount of `null_pad_size` before reading.
- ///
- /// If the return number of values is < `total`, it means the current page is drained and the
- /// caller should call `set_page_v1` or `set_page_v2` before calling next `read_batch`.
- pub fn read_batch(&mut self, total: usize, null_pad_size: usize) -> (usize, usize) {
- debug_assert!(
- self.value_decoder.is_some() && self.def_level_decoder.is_some(),
- "set_page_v1/v2 should have been called"
- );
-
- let n = ::std::cmp::min(self.num_values_in_page, total);
- self.num_values_in_page -= n;
- let value_decoder = self.value_decoder.as_mut().unwrap();
- let dl_decoder = self.def_level_decoder.as_mut().unwrap();
-
- let previous_num_nulls = self.vector.num_nulls;
- self.vector.put_nulls(null_pad_size);
- dl_decoder.read_batch(n, &mut self.vector, value_decoder.as_mut());
-
- (n, self.vector.num_nulls - previous_num_nulls)
- }
-
- /// Skips a batch of at most `total` values from the current page this reader has, and returns
- /// the actual number of values skipped.
- ///
- /// If the return value is < `total`, it means the current page is drained and the caller should
- /// call `set_page_v1` or `set_page_v2` before calling next `skip_batch`.
- pub fn skip_batch(&mut self, total: usize, put_nulls: bool) -> usize {
- debug_assert!(
- self.value_decoder.is_some() && self.def_level_decoder.is_some(),
- "set_page_v1/v2 should have been called"
- );
-
- let n = ::std::cmp::min(self.num_values_in_page, total);
- self.num_values_in_page -= n;
- let value_decoder = self.value_decoder.as_mut().unwrap();
- let dl_decoder = self.def_level_decoder.as_mut().unwrap();
-
- dl_decoder.skip_batch(n, &mut self.vector, value_decoder.as_mut(), put_nulls);
-
- n
- }
-
- /// Sets the dictionary page for this column reader and eagerly reads it.
- ///
- /// # Panics
- ///
- /// - If being called more than once during the lifetime of this column reader. A Parquet column
- /// chunk should only contain a single dictionary page.
- /// - If the input `encoding` is neither `PLAIN`, `PLAIN_DICTIONARY` nor `RLE_DICTIONARY`.
- pub fn set_dictionary_page(
- &mut self,
- page_value_count: usize,
- page_data: Buffer,
- mut encoding: Encoding,
- ) {
- // In Parquet format v1, both dictionary page and data page use the same encoding
- // `PLAIN_DICTIONARY`, while in v2, dictioanry page uses `PLAIN` and data page uses
- // `RLE_DICTIONARY`.
- //
- // Here, we convert `PLAIN` from v2 dictionary page to `PLAIN_DICTIONARY`, so that v1 and v2
- // shares the same encoding. Later on, `get_decoder` will use the `PlainDecoder` for this
- // case.
- if encoding == Encoding::PLAIN {
- encoding = Encoding::PLAIN_DICTIONARY;
- }
-
- if encoding != Encoding::PLAIN_DICTIONARY {
- panic!("Invalid encoding type for Parquet dictionary: {encoding}");
- }
-
- if self.vector.dictionary.is_some() {
- panic!("Parquet column cannot have more than one dictionary");
- }
-
- // Create a new vector for dictionary values
- let mut value_vector = ParquetMutableVector::new(page_value_count, &self.arrow_type);
-
- let mut dictionary = self.get_decoder(page_data, encoding);
- dictionary.read_batch(&mut value_vector, page_value_count);
- value_vector.num_values = page_value_count;
-
- // Re-create the parent vector since it is initialized with the dictionary value type, not
- // the key type (which is always integer).
- self.vector = ParquetMutableVector::new(self.capacity, &ArrowDataType::Int32);
- self.vector.set_dictionary(value_vector);
- }
-
- /// Resets the Parquet data page for this column reader.
- pub fn set_page_v1(
- &mut self,
- page_value_count: usize,
- page_data: Buffer,
- mut encoding: Encoding,
- ) {
- // In v1, when data is encoded with dictionary, data page uses `PLAIN_DICTIONARY`, while v2
- // uses `RLE_DICTIONARY`. To consolidate the two, here we convert `PLAIN_DICTIONARY` to
- // `RLE_DICTIONARY` following v2. Later on, `get_decoder` will use `DictDecoder` for this
- // case.
- if encoding == Encoding::PLAIN_DICTIONARY {
- encoding = Encoding::RLE_DICTIONARY;
- }
-
- self.num_values_in_page = page_value_count;
- self.check_dictionary(&encoding);
-
- let mut page_buffer = page_data;
-
- let bit_width = log2(self.desc.max_rep_level() as u64 + 1) as u8;
- let mut rl_decoder = LevelDecoder::new(Arc::clone(&self.desc), bit_width, true);
- let offset = rl_decoder.set_data(page_value_count, &page_buffer);
- self.rep_level_decoder = Some(rl_decoder);
- page_buffer = page_buffer.slice(offset);
-
- let bit_width = log2(self.desc.max_def_level() as u64 + 1) as u8;
- let mut dl_decoder = LevelDecoder::new(Arc::clone(&self.desc), bit_width, true);
- let offset = dl_decoder.set_data(page_value_count, &page_buffer);
- self.def_level_decoder = Some(dl_decoder);
- page_buffer = page_buffer.slice(offset);
-
- let value_decoder = self.get_decoder(page_buffer, encoding);
- self.value_decoder = Some(value_decoder);
- }
-
- /// Resets the Parquet data page for this column reader.
- pub fn set_page_v2(
- &mut self,
- page_value_count: usize,
- def_level_data: Buffer,
- rep_level_data: Buffer,
- value_data: Buffer,
- encoding: Encoding,
- ) {
- self.num_values_in_page = page_value_count;
- self.check_dictionary(&encoding);
-
- let bit_width = log2(self.desc.max_rep_level() as u64 + 1) as u8;
- let mut rl_decoder = LevelDecoder::new(Arc::clone(&self.desc), bit_width, false);
- rl_decoder.set_data(page_value_count, &rep_level_data);
- self.rep_level_decoder = Some(rl_decoder);
-
- let bit_width = log2(self.desc.max_def_level() as u64 + 1) as u8;
- let mut dl_decoder = LevelDecoder::new(Arc::clone(&self.desc), bit_width, false);
- dl_decoder.set_data(page_value_count, &def_level_data);
- self.def_level_decoder = Some(dl_decoder);
-
- let value_decoder = self.get_decoder(value_data, encoding);
- self.value_decoder = Some(value_decoder);
- }
-
- fn check_dictionary(&mut self, encoding: &Encoding) {
- // The column has a dictionary while the new page is of PLAIN encoding. In this case, we
- // should eagerly decode all the dictionary indices and convert the underlying vector to a
- // plain encoded vector.
- if self.vector.dictionary.is_some() && *encoding == Encoding::PLAIN {
- let new_vector = ParquetMutableVector::new(self.capacity, &self.arrow_type);
- let old_vector = std::mem::replace(&mut self.vector, new_vector);
- T::decode_dict(old_vector, &mut self.vector, self.bit_width);
- debug_assert!(self.vector.dictionary.is_none());
- }
- }
-
- fn get_decoder(&self, value_data: Buffer, encoding: Encoding) -> Box<dyn Decoder> {
- get_decoder::<T>(
- value_data,
- encoding,
- Arc::clone(&self.desc),
- self.read_options,
- )
- }
-}
diff --git a/native/core/src/parquet/read/levels.rs b/native/core/src/parquet/read/levels.rs
deleted file mode 100644
index 27dd3568af..0000000000
--- a/native/core/src/parquet/read/levels.rs
+++ /dev/null
@@ -1,229 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::mem;
-
-use super::values::Decoder;
-use crate::{
- common::bit::{self, read_u32, BitReader},
- parquet::ParquetMutableVector,
-};
-use arrow::buffer::Buffer;
-use datafusion_comet_spark_expr::utils::unlikely;
-use parquet::schema::types::ColumnDescPtr;
-
-const INITIAL_BUF_LEN: usize = 16;
-
-enum Mode {
- RLE,
- BitPacked,
-}
-
-/// A decoder for Parquet definition & repetition levels.
-pub struct LevelDecoder {
- /// The descriptor of the column that this level decoder is associated to.
- desc: ColumnDescPtr,
- /// Number of bits used to represent the levels.
- bit_width: u8,
- /// Mode for the current run.
- mode: Mode,
- /// Total number of values (including both null and non-null) to be decoded.
- num_values: usize,
- /// The current value in a RLE run. Unused if BitPacked.
- current_value: i32,
- /// The number of total values in the current RLE run. Unused if BitPacked.
- current_count: usize,
- /// The current buffer used in a BitPacked run. Unused if RLE.
- /// This will be resized if the total number of values in the BitPacked run is larger than its
- /// capacity.
- current_buffer: Vec<i32>, // TODO: double check this
- /// The index into `current_buffer` in a BitPacked run. Unused if RLE.
- current_buffer_idx: usize,
- /// A bit reader wrapping the input buffer for levels.
- bit_reader: Option<BitReader>,
- /// Whether we need to read the length of data. This is typically true for Parquet page V1, but
- /// not for V2, since it uses separate buffer for definition & repetition levels.
- need_length: bool,
-}
-
-impl LevelDecoder {
- pub fn new(desc: ColumnDescPtr, bit_width: u8, need_length: bool) -> Self {
- Self {
- desc,
- bit_width,
- mode: Mode::RLE,
- num_values: 0,
- current_value: 0,
- current_count: 0,
- current_buffer: vec![0; INITIAL_BUF_LEN],
- current_buffer_idx: 0,
- bit_reader: None,
- need_length,
- }
- }
-
- /// Sets data for this level decoder, and returns total number of bytes consumed. This is used
- /// for reading DataPage v1 levels.
- pub fn set_data(&mut self, page_value_count: usize, page_data: &Buffer) -> usize {
- self.num_values = page_value_count;
- if self.bit_width == 0 {
- // Special case where the page doesn't have encoded rl/dl data. Here we'll treat it as
- // an RLE run of `page_value_count` number of 0s.
- self.mode = Mode::RLE;
- self.current_count = page_value_count;
- 0
- } else if self.need_length {
- let u32_size = mem::size_of::<u32>();
- let data_size = read_u32(page_data.as_slice()) as usize;
- self.bit_reader = Some(BitReader::new(page_data.slice(u32_size), data_size));
- u32_size + data_size
- } else {
- // No need to read length, just read the whole buffer
- self.bit_reader = Some(BitReader::new_all(page_data.to_owned()));
- 0
- }
- }
-
- /// Reads a batch of `total` values into `vector`. The value decoding is done by
- /// `value_decoder`.
- pub fn read_batch(
- &mut self,
- total: usize,
- vector: &mut ParquetMutableVector,
- value_decoder: &mut dyn Decoder,
- ) {
- let mut left = total;
- while left > 0 {
- if unlikely(self.current_count == 0) {
- self.read_next_group();
- }
-
- debug_assert!(self.current_count > 0);
-
- let n = ::std::cmp::min(left, self.current_count);
- let max_def_level = self.desc.max_def_level();
-
- match self.mode {
- Mode::RLE => {
- if self.current_value as i16 == max_def_level {
- bit::set_bits(vector.validity_buffer.as_slice_mut(), vector.num_values, n);
- value_decoder.read_batch(vector, n);
- vector.num_values += n;
- } else {
- vector.put_nulls(n);
- }
- }
- Mode::BitPacked => {
- for i in 0..n {
- if self.current_buffer[self.current_buffer_idx + i] == max_def_level as i32
- {
- bit::set_bit(vector.validity_buffer.as_slice_mut(), vector.num_values);
- value_decoder.read(vector);
- vector.num_values += 1;
- } else {
- vector.put_null();
- }
- }
- self.current_buffer_idx += n;
- }
- }
-
- left -= n;
- self.current_count -= n;
- }
- }
-
- /// Skips a batch of `total` values. The value decoding is done by `value_decoder`.
- pub fn skip_batch(
- &mut self,
- total: usize,
- vector: &mut ParquetMutableVector,
- value_decoder: &mut dyn Decoder,
- put_nulls: bool,
- ) {
- let mut skip = total;
- while skip > 0 {
- if unlikely(self.current_count == 0) {
- self.read_next_group();
- }
-
- debug_assert!(self.current_count > 0);
-
- let n = ::std::cmp::min(skip, self.current_count);
- let max_def_level = self.desc.max_def_level();
-
- match self.mode {
- Mode::RLE => {
- if self.current_value as i16 == max_def_level {
- value_decoder.skip_batch(n);
- }
- }
- Mode::BitPacked => {
- let mut num_skips = 0;
- for i in 0..n {
- if self.current_buffer[self.current_buffer_idx + i] == max_def_level as i32
- {
- num_skips += 1;
- }
- }
- value_decoder.skip_batch(num_skips);
- self.current_buffer_idx += n;
- }
- }
- if put_nulls {
- vector.put_nulls(n);
- }
-
- skip -= n;
- self.current_count -= n;
- }
- }
-
- /// Loads the next group from this RLE/BitPacked hybrid reader.
- fn read_next_group(&mut self) {
- let bit_reader = self.bit_reader.as_mut().expect("bit_reader should be set");
- if let Some(indicator_value) = bit_reader.get_vlq_int() {
- self.mode = if indicator_value & 1 == 1 {
- Mode::BitPacked
- } else {
- Mode::RLE
- };
-
- match self.mode {
- Mode::BitPacked => {
- self.current_count = ((indicator_value >> 1) * 8) as usize;
- if self.current_buffer.len() < self.current_count {
- self.current_buffer.resize(self.current_count, 0);
- }
- self.current_buffer_idx = 0;
- bit_reader.get_batch(
- &mut self.current_buffer[..self.current_count],
- self.bit_width as usize,
- );
- }
- Mode::RLE => {
- // RLE
- self.current_count = (indicator_value >> 1) as usize;
- let value_width = (self.bit_width as usize).div_ceil(8);
- self.current_value = bit_reader
- .get_aligned::<i32>(value_width)
- .expect("current value should be set");
- }
- }
- }
- }
-}
diff --git a/native/core/src/parquet/read/mod.rs b/native/core/src/parquet/read/mod.rs
deleted file mode 100644
index f66f9da3da..0000000000
--- a/native/core/src/parquet/read/mod.rs
+++ /dev/null
@@ -1,108 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-pub mod column;
-pub mod levels;
-pub mod values;
-
-pub use column::ColumnReader;
-use parquet::schema::types::ColumnDescPtr;
-
-use super::ParquetMutableVector;
-use crate::common::bit::{self, BitReader};
-use arrow::buffer::Buffer;
-use bytes::Buf;
-
-#[derive(Clone, Copy)]
-pub struct ReadOptions {
- // Whether to read legacy dates/timestamps as it is. If false, throw exceptions.
- pub(crate) use_legacy_date_timestamp_or_ntz: bool,
-}
-
-/// Internal states for PLAIN decoder. Used in combination of `PlainDecoding`.
-pub struct PlainDecoderInner {
- /// The input buffer containing values to be decoded
- data: Buffer,
-
- /// The current offset in `data`, in bytes.
- offset: usize,
-
- /// Reads `data` bit by bit, used if `T` is [`BoolType`].
- bit_reader: BitReader,
-
- /// Options for reading Parquet
- read_options: ReadOptions,
-
- /// The Parquet column descriptor
- desc: ColumnDescPtr,
-}
-
-/// A trait for [`super::DataType`] to implement how PLAIN encoded data is to be decoded into Arrow
-/// format given an input and output buffer.
-///
-/// The actual implementations of this trait is in `read/values.rs`.
-pub trait PlainDecoding {
- /// Decodes `num` of items from `src`, and store the result into `dst`, in Arrow format.
- ///
- /// Note: this assumes the `src` has data for at least `num` elements, and won't do any
- /// bound checking. The condition MUST be guaranteed from the caller side.
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize);
-
- /// Skip `num` of items from `src`
- ///
- /// Note: this assumes the `src` has data for at least `num` elements, and won't do any
- /// bound checking. The condition MUST be guaranteed from the caller side.
- fn skip(src: &mut PlainDecoderInner, num: usize);
-}
-
-pub trait PlainDictDecoding {
- /// Eagerly decode vector `src` which must have dictionary associated. The decoded values are
- /// appended into `dst`.
- fn decode_dict(src: ParquetMutableVector, dst: &mut ParquetMutableVector, bit_width: usize) {
- assert!(dst.dictionary.is_none());
- assert!(src.dictionary.is_some());
-
- let mut value_buf = src.value_buffer.as_slice();
- let validity_buf = src.validity_buffer.as_slice();
- let dictionary = src.dictionary.as_ref().unwrap();
-
- for i in 0..src.num_values {
- if bit::get_bit(validity_buf, i) {
- // non-null value: lookup the value position and copy its value into `dst`
- let val_idx = value_buf.get_u32_le();
- Self::decode_dict_one(i, val_idx as usize, dictionary, dst, bit_width);
- dst.num_values += 1;
- } else {
- value_buf.advance(4);
- dst.put_null();
- }
- }
-
- dst.validity_buffer = src.validity_buffer;
- }
-
- /// Decode a single value from `src`, whose position in the dictionary indices (i.e., keys)
- /// is `idx` and the positions in the dictionary values is `val_idx`. The decoded value is
- /// appended to `dst`.
- fn decode_dict_one(
- idx: usize,
- val_idx: usize,
- src: &ParquetMutableVector,
- dst: &mut ParquetMutableVector,
- bit_width: usize,
- );
-}
diff --git a/native/core/src/parquet/read/values.rs b/native/core/src/parquet/read/values.rs
deleted file mode 100644
index 58f083eca9..0000000000
--- a/native/core/src/parquet/read/values.rs
+++ /dev/null
@@ -1,1131 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::{marker::PhantomData, mem};
-
-use arrow::buffer::Buffer;
-use bytes::Buf;
-use log::debug;
-use parquet::{basic::Encoding, schema::types::ColumnDescPtr};
-
-use super::{PlainDecoderInner, PlainDecoding, PlainDictDecoding, ReadOptions};
-use crate::write_null;
-use crate::write_val_or_null;
-use crate::{
- common::bit::{self, BitReader},
- parquet::{data_type::*, ParquetMutableVector},
-};
-use arrow::datatypes::DataType as ArrowDataType;
-use datafusion_comet_spark_expr::utils::unlikely;
-
-pub fn get_decoder<T: DataType>(
- value_data: Buffer,
- encoding: Encoding,
- desc: ColumnDescPtr,
- read_options: ReadOptions,
-) -> Box<dyn Decoder> {
- let decoder: Box<dyn Decoder> = match encoding {
- Encoding::PLAIN | Encoding::PLAIN_DICTIONARY => {
- Box::new(PlainDecoder::<T>::new(value_data, desc, read_options))
- }
- // This is for dictionary indices
- Encoding::RLE_DICTIONARY => Box::new(DictDecoder::new(value_data)),
- _ => panic!("Unsupported encoding: {encoding}"),
- };
- decoder
-}
-
-/// A Parquet decoder for values within a Parquet data page.
-pub trait Decoder {
- /// Consumes a single value from the decoder and stores it into `dst`.
- ///
- /// # Preconditions
- ///
- /// * `dst` have enough length to hold at least one value.
- /// * `data` of this decoder should have enough bytes left to be decoded.
- fn read(&mut self, dst: &mut ParquetMutableVector);
-
- /// Consumes a batch of `num` values from the data and stores them to `dst`.
- ///
- /// # Preconditions
- ///
- /// * `dst` should have length >= `num * T::type_size()` .
- /// * `data` of this decoder should have >= `num * T::type_size()` bytes left to be decoded.
- fn read_batch(&mut self, dst: &mut ParquetMutableVector, num: usize);
-
- /// Skips a batch of `num` values from the data.
- ///
- /// # Preconditions
- ///
- /// * `data` of this decoder should have >= `num * T::type_size()` bytes left to be decoded.
- fn skip_batch(&mut self, num: usize);
-
- /// Returns the encoding for this decoder.
- fn encoding(&self) -> Encoding;
-}
-
-/// The switch off date between Julian and Gregorian calendar. See
-/// https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html
-const JULIAN_GREGORIAN_SWITCH_OFF_DAY: i32 = -141427;
-
-/// The switch off timestamp (in micros) between Julian and Gregorian calendar. See
-/// https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html
-const JULIAN_GREGORIAN_SWITCH_OFF_TS: i64 = -2208988800000000;
-
-/// See http://stackoverflow.com/questions/466321/convert-unix-timestamp-to-julian
-/// Also see Spark's `DateTimeUtils.JULIAN_DAY_OF_EPOCH`
-const JULIAN_DAY_OF_EPOCH: i32 = 2440588;
-
-/// Number of micro seconds per milli second.
-const MICROS_PER_MILLIS: i64 = 1000;
-
-const MICROS_PER_DAY: i64 = 24_i64 * 60 * 60 * 1000 * 1000;
-
-pub struct PlainDecoder<T: DataType> {
- /// Internal states for this decoder.
- inner: PlainDecoderInner,
-
- /// Marker to allow `T` in the generic parameter of the struct.
- _phantom: PhantomData<T>,
-}
-
-impl<T: DataType> PlainDecoder<T> {
- pub fn new(value_data: Buffer, desc: ColumnDescPtr, read_options: ReadOptions) -> Self {
- let len = value_data.len();
- let inner = PlainDecoderInner {
- data: value_data.clone(),
- offset: 0,
- bit_reader: BitReader::new(value_data, len),
- read_options,
- desc,
- };
- Self {
- inner,
- _phantom: PhantomData,
- }
- }
-}
-
-macro_rules! make_plain_default_impl {
- ($($ty: ident), *) => {
- $(
- impl PlainDecoding for $ty {
- /// Default implementation for PLAIN encoding. Uses `mempcy` when the physical
- /// layout is the same between Parquet and Arrow.
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- let src_data = &src.data;
- let byte_width = src.desc.type_length() as usize;
- let num_bytes = byte_width * num;
- let dst_offset = byte_width * dst.num_values;
-
- bit::memcpy(
- &src_data[src.offset..src.offset + num_bytes],
- &mut dst.value_buffer[dst_offset..]);
- src.offset += num_bytes;
- }
-
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- let num_bytes = src.desc.type_length() as usize * num;
- src.offset += num_bytes;
- }
- }
- )*
- };
-}
-
-make_plain_default_impl! { Int32Type, Int64Type, FloatType, DoubleType, FLBAType }
-
-macro_rules! make_plain_dict_impl {
- ($($ty: ident), *) => {
- $(
- impl PlainDictDecoding for $ty {
- fn decode_dict_one(
- idx: usize,
- val_idx: usize,
- src: &ParquetMutableVector,
- dst: &mut ParquetMutableVector,
- bit_width: usize,
- ) {
- let byte_width = bit_width / 8;
- bit::memcpy(
- &src.value_buffer[val_idx * byte_width..(val_idx+1) * byte_width],
- &mut dst.value_buffer[idx * byte_width..],
- );
- }
- }
- )*
- };
-}
-
-make_plain_dict_impl! { Int8Type, UInt8Type, Int16Type, UInt16Type, Int32Type, UInt32Type }
-make_plain_dict_impl! { Int32DateType, Int64Type, FloatType, Int64ToDecimal64Type, FLBAType }
-make_plain_dict_impl! { DoubleType, Int64TimestampMillisType, Int64TimestampMicrosType }
-
-macro_rules! make_int_variant_dict_impl {
- ($ty:ty, $src_ty:ty, $dst_ty:ty) => {
- impl PlainDictDecoding for $ty {
- fn decode_dict_one(
- idx: usize,
- val_idx: usize,
- src: &ParquetMutableVector,
- dst: &mut ParquetMutableVector,
- _: usize,
- ) {
- let src_ptr = src.value_buffer.as_ptr() as *const $src_ty;
- let dst_ptr = dst.value_buffer.as_mut_ptr() as *mut $dst_ty;
- unsafe {
- // SAFETY the caller must ensure `idx`th pointer is in bounds
- dst_ptr
- .add(idx)
- .write_unaligned(src_ptr.add(val_idx).read_unaligned() as $dst_ty);
- }
- }
- }
- };
-}
-
-make_int_variant_dict_impl!(Int16ToDoubleType, i16, f64);
-make_int_variant_dict_impl!(Int32To64Type, i32, i64);
-make_int_variant_dict_impl!(Int32ToDecimal64Type, i32, i64);
-make_int_variant_dict_impl!(Int32ToDoubleType, i32, f64);
-make_int_variant_dict_impl!(Int32TimestampMicrosType, i32, i64);
-make_int_variant_dict_impl!(FloatToDoubleType, f32, f64);
-make_int_variant_dict_impl!(Int32DecimalType, i128, i128);
-make_int_variant_dict_impl!(Int64DecimalType, i128, i128);
-make_int_variant_dict_impl!(UInt64Type, u128, u128);
-
-impl PlainDecoding for Int32DateType {
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- let src_data = &src.data;
- let byte_width = src.desc.type_length() as usize;
- let num_bytes = byte_width * num;
- let dst_offset = byte_width * dst.num_values;
-
- if !src.read_options.use_legacy_date_timestamp_or_ntz {
- // By default we panic if the date value is before the switch date between Julian
- // calendar and Gregorian calendar, which is 1582-10-15, and -141727 days
- // before the unix epoch date 1970-01-01.
- let mut offset = src.offset;
- for _ in 0..num {
- let v = &src_data[offset..offset + byte_width] as *const [u8] as *const u8
- as *const i32;
-
- // TODO: optimize this further as checking value one by one is not very efficient
- unsafe {
- if unlikely(v.read_unaligned() < JULIAN_GREGORIAN_SWITCH_OFF_DAY) {
- panic!(
- "Encountered date value {}, which is before 1582-10-15 (counting backwards \
- from Unix epoch date 1970-01-01), and could be ambigous depending on \
- whether a legacy Julian/Gregorian hybrid calendar is used, or a Proleptic \
- Gregorian calendar is used.",
- *v
- );
- }
- }
-
- offset += byte_width;
- }
- }
-
- bit::memcpy(
- &src_data[src.offset..src.offset + num_bytes],
- &mut dst.value_buffer[dst_offset..],
- );
-
- src.offset += num_bytes;
- }
-
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- let num_bytes = src.desc.type_length() as usize * num;
- src.offset += num_bytes;
- }
-}
-
-impl PlainDecoding for Int32TimestampMicrosType {
- #[inline]
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- let src_data = &src.data;
- let byte_width = src.desc.type_length() as usize;
- let num_bytes = byte_width * num;
-
- {
- let mut offset = src.offset;
- for _ in 0..num {
- let v = src_data[offset..offset + byte_width].as_ptr() as *const i32;
-
- // TODO: optimize this further as checking value one by one is not very efficient
- unsafe {
- if unlikely(v.read_unaligned() < JULIAN_GREGORIAN_SWITCH_OFF_DAY) {
- panic!(
- "Encountered timestamp value {}, which is before 1582-10-15 (counting \
- backwards from Unix epoch date 1970-01-01), and could be ambigous \
- depending on whether a legacy Julian/Gregorian hybrid calendar is used, \
- or a Proleptic Gregorian calendar is used.",
- *v
- );
- }
- }
-
- offset += byte_width;
- }
- }
-
- let mut offset = src.offset;
- let dst_byte_width = byte_width * 2;
- let mut dst_offset = dst_byte_width * dst.num_values;
- for _ in 0..num {
- let v = src_data[offset..offset + byte_width].as_ptr() as *const i32;
- let v = unsafe { v.read_unaligned() };
- let v = (v as i64).wrapping_mul(MICROS_PER_DAY);
- bit::memcpy_value(&v, dst_byte_width, &mut dst.value_buffer[dst_offset..]);
- offset += byte_width;
- dst_offset += dst_byte_width;
- }
-
- src.offset += num_bytes;
- }
-
- #[inline]
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- let num_bytes = src.desc.type_length() as usize * num;
- src.offset += num_bytes;
- }
-}
-
-impl PlainDecoding for Int64TimestampMillisType {
- #[inline]
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- let src_data = &src.data;
- let byte_width = src.desc.type_length() as usize;
- let num_bytes = byte_width * num;
-
- if !src.read_options.use_legacy_date_timestamp_or_ntz {
- let mut offset = src.offset;
- for _ in 0..num {
- unsafe {
- let v = &src_data[offset..offset + byte_width] as *const [u8] as *const u8
- as *const i64;
- let v = v.read_unaligned() * MICROS_PER_MILLIS;
-
- // TODO: optimize this further as checking value one by one is not very
- // efficient
- if unlikely(v < JULIAN_GREGORIAN_SWITCH_OFF_TS) {
- panic!(
- "Encountered timestamp value {v}, which is before 1582-10-15 (counting \
- backwards from Unix epoch date 1970-01-01), and could be ambigous \
- depending on whether a legacy Julian/Gregorian hybrid calendar is used, \
- or a Proleptic Gregorian calendar is used."
- );
- }
-
- offset += byte_width;
- }
- }
- }
-
- unsafe {
- let mut offset = src.offset;
- let mut dst_offset = byte_width * dst.num_values;
- for _ in 0..num {
- let v = &src_data[offset..offset + byte_width] as *const [u8] as *const u8
- as *const i64;
- let v = v.read_unaligned() * MICROS_PER_MILLIS;
- bit::memcpy_value(&v, byte_width, &mut dst.value_buffer[dst_offset..]);
- offset += byte_width;
- dst_offset += byte_width;
- }
- }
-
- src.offset += num_bytes;
- }
-
- #[inline]
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- let num_bytes = src.desc.type_length() as usize * num;
- src.offset += num_bytes;
- }
-}
-
-impl PlainDecoding for Int64TimestampMicrosType {
- #[inline]
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- let src_data = &src.data;
- let byte_width = src.desc.type_length() as usize;
- let num_bytes = byte_width * num;
- let dst_offset = byte_width * dst.num_values;
-
- if !src.read_options.use_legacy_date_timestamp_or_ntz {
- let mut offset = src.offset;
- for _ in 0..num {
- unsafe {
- let v = &src_data[offset..offset + byte_width] as *const [u8] as *const u8
- as *const i64;
-
- // TODO: optimize this further as checking value one by one is not very
- // efficient
- if unlikely(v.read_unaligned() < JULIAN_GREGORIAN_SWITCH_OFF_TS) {
- panic!(
- "Encountered timestamp value {}, which is before 1582-10-15 (counting \
- backwards from Unix epoch date 1970-01-01), and could be ambigous \
- depending on whether a legacy Julian/Gregorian hybrid calendar is used, \
- or a Proleptic Gregorian calendar is used.",
- *v
- );
- }
-
- offset += byte_width;
- }
- }
- }
-
- bit::memcpy(
- &src_data[src.offset..src.offset + num_bytes],
- &mut dst.value_buffer[dst_offset..],
- );
-
- src.offset += num_bytes;
- }
-
- #[inline]
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- let num_bytes = src.desc.type_length() as usize * num;
- src.offset += num_bytes;
- }
-}
-
-impl PlainDecoding for BoolType {
- /// Specific implementation for PLAIN encoding of boolean type. Even though both Parquet and
- /// Arrow share the same physical layout for the type (which is 1 bit for each value), we'll
- /// need to treat the number of bytes specifically.
- #[inline]
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- src.bit_reader
- .get_bits(&mut dst.value_buffer, dst.num_values, num);
- }
-
- #[inline]
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- src.bit_reader.skip_bits(num);
- }
-}
-
-// Does it make sense to encode booleans with dictionary?
-impl PlainDictDecoding for BoolType {
- #[inline]
- fn decode_dict_one(
- idx: usize,
- val_idx: usize,
- src: &ParquetMutableVector,
- dst: &mut ParquetMutableVector,
- _: usize,
- ) {
- let v = bit::get_bit(src.value_buffer.as_slice(), val_idx);
- if v {
- bit::set_bit(dst.value_buffer.as_slice_mut(), idx);
- } // `dst` should be zero initialized so no need to call `unset_bit`.
- }
-}
-
-macro_rules! make_int_variant_impl {
- ($dst_type:ty, $copy_fn:ident, $type_width:expr) => {
- impl PlainDecoding for $dst_type {
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- let dst_slice = dst.value_buffer.as_slice_mut();
- let dst_offset = dst.num_values * $type_width;
- $copy_fn(&src.data[src.offset..], &mut dst_slice[dst_offset..], num);
- src.offset += 4 * num; // Parquet stores Int8/Int16 using 4 bytes
- }
-
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- src.offset += 4 * num; // Parquet stores Int8/Int16 using 4 bytes
- }
- }
- };
-}
-
-make_int_variant_impl!(Int8Type, copy_i32_to_i8, 1);
-make_int_variant_impl!(Int16Type, copy_i32_to_i16, 2);
-make_int_variant_impl!(Int16ToDoubleType, copy_i32_to_f64, 8); // Parquet uses Int16 using 4 bytes
-make_int_variant_impl!(Int32To64Type, copy_i32_to_i64, 8);
-make_int_variant_impl!(Int32ToDoubleType, copy_i32_to_f64, 8);
-make_int_variant_impl!(FloatToDoubleType, copy_f32_to_f64, 8);
-
-// unsigned type require double the width and zeroes are written for the second half
-// because they are implemented as the next size up signed type
-make_int_variant_impl!(UInt8Type, copy_i32_to_u8, 2);
-make_int_variant_impl!(UInt16Type, copy_i32_to_u16, 4);
-make_int_variant_impl!(UInt32Type, copy_i32_to_u32, 8);
-
-macro_rules! make_int_decimal_variant_impl {
- ($ty:ty, $copy_fn:ident, $type_width:expr, $dst_type:ty) => {
- impl PlainDecoding for $ty {
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- let dst_slice = dst.value_buffer.as_slice_mut();
- let dst_offset = dst.num_values * std::mem::size_of::<$dst_type>();
- $copy_fn(&src.data[src.offset..], &mut dst_slice[dst_offset..], num);
-
- let src_precision = src.desc.type_precision() as u32;
- let src_scale = ::std::cmp::max(src.desc.type_scale(), 0) as u32;
- let (dst_precision, dst_scale) = match dst.arrow_type {
- ArrowDataType::Decimal128(p, s) if s >= 0 => (p as u32, s as u32),
- _ => unreachable!(),
- };
- let upper = (10 as $dst_type).pow(dst_precision);
- let v = dst_slice[dst_offset..].as_mut_ptr() as *mut $dst_type;
- if dst_scale > src_scale {
- let mul = (10 as $dst_type).pow(dst_scale - src_scale);
- for i in 0..num {
- unsafe {
- // SAFETY the caller must ensure `i`th pointer is in bounds
- let v = v.add(i);
- write_val_or_null!(v, v.read_unaligned() * mul, upper, dst, i);
- }
- }
- } else if dst_scale < src_scale {
- let div = (10 as $dst_type).pow(src_scale - dst_scale);
- for i in 0..num {
- unsafe {
- // SAFETY the caller must ensure `i`th pointer is in bounds
- let v = v.add(i);
- write_val_or_null!(v, v.read_unaligned() / div, upper, dst, i);
- }
- }
- } else if src_precision > dst_precision {
- for i in 0..num {
- unsafe {
- // SAFETY the caller must ensure `i`th pointer is in bounds
- let v = v.add(i);
- write_null!(v.read_unaligned(), upper, dst, i);
- }
- }
- }
-
- src.offset += $type_width * num;
- }
-
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- src.offset += $type_width * num;
- }
- }
- };
-}
-make_int_decimal_variant_impl!(Int32ToDecimal64Type, copy_i32_to_i64, 4, i64);
-make_int_decimal_variant_impl!(Int32DecimalType, copy_i32_to_i128, 4, i128);
-make_int_decimal_variant_impl!(Int64ToDecimal64Type, copy_i64_to_i64, 8, i64);
-make_int_decimal_variant_impl!(Int64DecimalType, copy_i64_to_i128, 8, i128);
-make_int_decimal_variant_impl!(UInt64Type, copy_u64_to_u128, 8, u128);
-
-#[macro_export]
-macro_rules! write_val_or_null {
- ($v: expr, $adjusted: expr, $upper: expr, $dst: expr, $i: expr) => {
- let adjusted = $adjusted;
- $v.write_unaligned(adjusted);
- write_null!(adjusted, $upper, $dst, $i);
- };
-}
-
-#[macro_export]
-macro_rules! write_null {
- ($val: expr, $upper: expr, $dst: expr, $i: expr) => {
- if $upper <= $val {
- bit::unset_bit($dst.validity_buffer.as_slice_mut(), $dst.num_values + $i);
- $dst.num_nulls += 1;
- }
- };
-}
-
-macro_rules! generate_cast_to_unsigned {
- ($name: ident, $src_type:ty, $dst_type:ty, $zero_value:expr) => {
- pub fn $name(src: &[u8], dst: &mut [u8], num: usize) {
- debug_assert!(
- src.len() >= num * std::mem::size_of::<$src_type>(),
- "Source slice is too small"
- );
- debug_assert!(
- dst.len() >= num * std::mem::size_of::<$dst_type>() * 2,
- "Destination slice is too small"
- );
-
- let src_ptr = src.as_ptr() as *const $src_type;
- let dst_ptr = dst.as_mut_ptr() as *mut $dst_type;
- unsafe {
- for i in 0..num {
- dst_ptr
- .add(2 * i)
- .write_unaligned(src_ptr.add(i).read_unaligned() as $dst_type);
- // write zeroes
- dst_ptr.add(2 * i + 1).write_unaligned($zero_value);
- }
- }
- }
- };
-}
-
-generate_cast_to_unsigned!(copy_i32_to_u32, i32, u32, 0_u32);
-
-macro_rules! generate_cast_to_signed {
- ($name: ident, $src_type:ty, $dst_type:ty) => {
- pub fn $name(src: &[u8], dst: &mut [u8], num: usize) {
- debug_assert!(
- src.len() >= num * std::mem::size_of::<$src_type>(),
- "Source slice is too small"
- );
- debug_assert!(
- dst.len() >= num * std::mem::size_of::<$dst_type>(),
- "Destination slice is too small"
- );
-
- let src_ptr = src.as_ptr() as *const $src_type;
- let dst_ptr = dst.as_mut_ptr() as *mut $dst_type;
- unsafe {
- for i in 0..num {
- dst_ptr
- .add(i)
- .write_unaligned(src_ptr.add(i).read_unaligned() as $dst_type);
- }
- }
- }
- };
-}
-
-generate_cast_to_signed!(copy_i32_to_i8, i32, i8);
-generate_cast_to_signed!(copy_i32_to_i16, i32, i16);
-generate_cast_to_signed!(copy_i32_to_i64, i32, i64);
-generate_cast_to_signed!(copy_i32_to_i128, i32, i128);
-generate_cast_to_signed!(copy_i32_to_f64, i32, f64);
-generate_cast_to_signed!(copy_i64_to_i64, i64, i64);
-generate_cast_to_signed!(copy_i64_to_i128, i64, i128);
-generate_cast_to_signed!(copy_u64_to_u128, u64, u128);
-generate_cast_to_signed!(copy_f32_to_f64, f32, f64);
-// even for u8/u16, need to copy full i16/i32 width for Spark compatibility
-generate_cast_to_signed!(copy_i32_to_u8, i32, i16);
-generate_cast_to_signed!(copy_i32_to_u16, i32, i32);
-
-// Shared implementation for variants of Binary type
-macro_rules! make_plain_binary_impl {
- ($($ty: ident), *) => {
- $(
- impl PlainDecoding for $ty {
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- let src_data = &src.data;
- let mut src_offset = src.offset;
-
- let mut offset_offset = dst.num_values * 4;
- let offset_buf = &mut dst.value_buffer.as_slice_mut();
- let mut offset_value = read_num_bytes!(i32, 4, &offset_buf[offset_offset..]);
- offset_offset += 4;
-
- // The actual content of a byte array is stored contiguously in the child vector
- let child = &mut dst.children[0];
- let mut value_offset = child.num_values; // num_values == num of bytes
-
- (0..num).for_each(|_| {
- let len = read_num_bytes!(i32, 4, &src_data[src_offset..]) as usize;
- offset_value += len as i32;
-
- // Copy offset for the current string value into the offset buffer
- bit::memcpy_value(&offset_value, 4, &mut offset_buf[offset_offset..]);
-
- // Reserve additional space in child value buffer if not enough
- let value_buf_len = child.value_buffer.len();
-
- if unlikely(value_buf_len < value_offset + len) {
- let new_capacity = ::std::cmp::max(value_offset + len, value_buf_len * 2);
- debug!("Reserving additional space ({} -> {} bytes) for value buffer",
- value_buf_len, new_capacity);
- child.value_buffer.resize(new_capacity);
- }
-
- // Copy the actual string content into the value buffer
- src_offset += mem::size_of::<u32>();
- bit::memcpy(
- &src_data[src_offset..src_offset + len],
- &mut child.value_buffer.as_slice_mut()[value_offset..],
- );
-
- value_offset += len;
- src_offset += len;
- offset_offset += 4;
- });
-
- src.offset = src_offset;
- child.num_values = value_offset;
- }
-
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- let src_data = &src.data;
- let mut src_offset = src.offset;
-
- (0..num).for_each(|_| {
- let len = read_num_bytes!(i32, 4, &src_data[src_offset..]) as usize;
- src_offset += mem::size_of::<u32>();
- src_offset += len;
- });
-
- src.offset = src_offset;
- }
- }
- )*
- };
-}
-
-make_plain_binary_impl! { ByteArrayType, StringType }
-
-macro_rules! make_plain_dict_binary_impl {
- ($($ty: ident), *) => {
- $(
- impl PlainDictDecoding for $ty {
- #[inline]
- fn decode_dict_one(
- idx: usize,
- val_idx: usize,
- src: &ParquetMutableVector,
- dst: &mut ParquetMutableVector,
- _: usize,
- ) {
- debug_assert!(src.children.len() == 1);
- debug_assert!(dst.children.len() == 1);
-
- let src_child = &src.children[0];
- let dst_child = &mut dst.children[0];
-
- // get the offset & data for the binary value at index `val_idx`
- let mut start_slice = &src.value_buffer[val_idx * 4..];
- let start = start_slice.get_u32_le() as usize;
- let mut end_slice = &src.value_buffer[(val_idx + 1) * 4..];
- let end = end_slice.get_u32_le() as usize;
-
- debug_assert!(end >= start);
-
- let len = end - start;
- let curr_offset = read_num_bytes!(u32, 4, &dst.value_buffer[idx * 4..]) as usize;
-
- // Reserve additional space in child value buffer if not enough
- let value_buf_len = dst_child.value_buffer.len();
-
- if unlikely(value_buf_len < curr_offset + len) {
- let new_capacity = ::std::cmp::max(curr_offset + len, value_buf_len * 2);
- debug!("Reserving additional space ({} -> {} bytes) for value buffer \
- during dictionary fallback", value_buf_len,
- new_capacity);
- dst_child.value_buffer.resize(new_capacity);
- }
-
- bit::memcpy(
- &src_child.value_buffer[start..end],
- &mut dst_child.value_buffer[curr_offset..],
- );
-
- bit::memcpy_value(
- &((curr_offset + len) as u32),
- 4,
- &mut dst.value_buffer[(idx + 1) * 4..],
- );
-
- dst_child.num_values += len;
- }
- }
- )*
- };
-}
-
-make_plain_dict_binary_impl! { ByteArrayType, StringType }
-
-macro_rules! make_plain_decimal_int_impl {
- ($($ty: ident; $dst_type:ty), *) => {
- $(
- impl PlainDecoding for $ty {
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- let num_bytes = std::mem::size_of::<$dst_type>();
- let byte_width = src.desc.type_length() as usize;
- let num_bits = num_bytes.saturating_sub(byte_width) * 8;
-
- let src_data = &src.data[src.offset..];
- let dst_data = &mut dst.value_buffer[dst.num_values * num_bytes..];
-
- let mut src_offset = 0;
-
- let src_precision = src.desc.type_precision() as u32;
- let src_scale = ::std::cmp::max(src.desc.type_scale(), 0) as u32;
- let (dst_precision, dst_scale) = match dst.arrow_type {
- ArrowDataType::Decimal128(p, s) if s >= 0 => (p as u32, s as u32),
- _ => (src_precision, src_scale),
- };
- let upper = (10 as $dst_type).pow(dst_precision);
- let mul_div = (10 as $dst_type).pow(dst_scale.abs_diff(src_scale));
-
- for i in 0..num {
- let mut unscaled: $dst_type = 0;
- for _ in 0..byte_width {
- unscaled = unscaled << 8 | src_data[src_offset] as $dst_type;
- src_offset += 1;
- }
- unscaled = (unscaled << num_bits) >> num_bits;
- if dst_scale > src_scale {
- unscaled *= mul_div;
- } else if dst_scale < src_scale {
- unscaled /= mul_div;
- }
- bit::memcpy_value(&unscaled, num_bytes, &mut dst_data[i * num_bytes..]);
- if src_precision > dst_precision {
- write_null!(unscaled, upper, dst, i);
- }
- }
-
- src.offset += num * byte_width;
- }
-
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- let num_bytes_to_skip = num * src.desc.type_length() as usize;
- src.offset += num_bytes_to_skip;
- }
- }
-
- impl PlainDictDecoding for $ty {
- #[inline]
- fn decode_dict_one(_: usize, val_idx: usize, src: &ParquetMutableVector, dst: &mut ParquetMutableVector, _: usize) {
- let num_bytes = std::mem::size_of::<$dst_type>();
- bit::memcpy(
- &src.value_buffer[val_idx * num_bytes..(val_idx + 1) * num_bytes],
- &mut dst.value_buffer[dst.num_values * num_bytes..],
- );
- }
- }
- )*
- };
-}
-
-make_plain_decimal_int_impl!(FLBADecimal32Type; i32, FLBADecimal64Type; i64, FLBADecimalType; i128);
-
-// Int96 contains 12 bytes
-const INT96_SRC_BYTE_WIDTH: usize = 12;
-// We convert INT96 to micros and store using i64
-const INT96_DST_BYTE_WIDTH: usize = 8;
-
-fn int96_to_microsecond(v: &[u8]) -> i64 {
- let nanos = &v[..INT96_DST_BYTE_WIDTH] as *const [u8] as *const u8 as *const i64;
- let day = &v[INT96_DST_BYTE_WIDTH..] as *const [u8] as *const u8 as *const i32;
-
- unsafe {
- ((day.read_unaligned() - JULIAN_DAY_OF_EPOCH) as i64)
- .wrapping_mul(MICROS_PER_DAY)
- .wrapping_add(nanos.read_unaligned() / 1000)
- }
-}
-
-/// Decode timestamps represented as INT96 into i64 with micros precision
-impl PlainDecoding for Int96TimestampMicrosType {
- #[inline]
- fn decode(src: &mut PlainDecoderInner, dst: &mut ParquetMutableVector, num: usize) {
- let src_data = &src.data;
-
- if !src.read_options.use_legacy_date_timestamp_or_ntz {
- let mut offset = src.offset;
- for _ in 0..num {
- // TODO: optimize this further as checking value one by one is not very efficient
- let micros = int96_to_microsecond(&src_data[offset..offset + INT96_SRC_BYTE_WIDTH]);
-
- if unlikely(micros < JULIAN_GREGORIAN_SWITCH_OFF_TS) {
- panic!(
- "Encountered timestamp value {micros}, which is before 1582-10-15 (counting \
- backwards from Unix epoch date 1970-01-01), and could be ambigous \
- depending on whether a legacy Julian/Gregorian hybrid calendar is used, \
- or a Proleptic Gregorian calendar is used."
- );
- }
-
- offset += INT96_SRC_BYTE_WIDTH;
- }
- }
-
- let mut offset = src.offset;
- let mut dst_offset = INT96_DST_BYTE_WIDTH * dst.num_values;
- for _ in 0..num {
- let micros = int96_to_microsecond(&src_data[offset..offset + INT96_SRC_BYTE_WIDTH]);
-
- bit::memcpy_value(
- µs,
- INT96_DST_BYTE_WIDTH,
- &mut dst.value_buffer[dst_offset..],
- );
-
- offset += INT96_SRC_BYTE_WIDTH;
- dst_offset += INT96_DST_BYTE_WIDTH;
- }
-
- src.offset = offset;
- }
-
- #[inline]
- fn skip(src: &mut PlainDecoderInner, num: usize) {
- src.offset += INT96_SRC_BYTE_WIDTH * num;
- }
-}
-
-impl PlainDictDecoding for Int96TimestampMicrosType {
- fn decode_dict_one(
- _: usize,
- val_idx: usize,
- src: &ParquetMutableVector,
- dst: &mut ParquetMutableVector,
- _: usize,
- ) {
- let src_offset = val_idx * INT96_DST_BYTE_WIDTH;
- let dst_offset = dst.num_values * INT96_DST_BYTE_WIDTH;
-
- bit::memcpy(
- &src.value_buffer[src_offset..src_offset + INT96_DST_BYTE_WIDTH],
- &mut dst.value_buffer[dst_offset..dst_offset + INT96_DST_BYTE_WIDTH],
- );
- }
-}
-
-impl<T: DataType> Decoder for PlainDecoder<T> {
- #[inline]
- fn read(&mut self, dst: &mut ParquetMutableVector) {
- self.read_batch(dst, 1)
- }
-
- /// Default implementation for PLAIN encoding, which uses a `memcpy` to copy from Parquet to the
- /// Arrow vector. NOTE: this only works if the Parquet physical type has the same type width as
- /// the Arrow's physical type (e.g., Parquet INT32 vs Arrow INT32). For other cases, we should
- /// have special implementations.
- #[inline]
- fn read_batch(&mut self, dst: &mut ParquetMutableVector, num: usize) {
- T::decode(&mut self.inner, dst, num);
- }
-
- #[inline]
- fn skip_batch(&mut self, num: usize) {
- T::skip(&mut self.inner, num);
- }
-
- #[inline]
- fn encoding(&self) -> Encoding {
- Encoding::PLAIN
- }
-}
-
-/// A decoder for Parquet dictionary indices, which is always of integer type, and encoded with
-/// RLE/BitPacked encoding.
-pub struct DictDecoder {
- /// Number of bits used to represent dictionary indices. Must be between `[0, 64]`.
- bit_width: usize,
-
- /// Bit reader
- bit_reader: BitReader,
-
- /// Number of values left in the current RLE run
- rle_left: usize,
-
- /// Number of values left in the current BIT_PACKED run
- bit_packed_left: usize,
-
- /// Current value in the RLE run. Unused if BIT_PACKED
- current_value: u32,
-}
-
-impl DictDecoder {
- pub fn new(buf: Buffer) -> Self {
- let bit_width = buf.as_bytes()[0] as usize;
-
- Self {
- bit_width,
- bit_reader: BitReader::new_all(buf.slice(1)),
- rle_left: 0,
- bit_packed_left: 0,
- current_value: 0,
- }
- }
-}
-
-impl DictDecoder {
- /// Reads the header of the next RLE/BitPacked run, and update the internal state such as # of
- /// values for the next run, as well as the current value in case it's RLE.
- fn reload(&mut self) {
- if let Some(indicator_value) = self.bit_reader.get_vlq_int() {
- if indicator_value & 1 == 1 {
- self.bit_packed_left = ((indicator_value >> 1) * 8) as usize;
- } else {
- self.rle_left = (indicator_value >> 1) as usize;
- let value_width = self.bit_width.div_ceil(8);
- self.current_value = self.bit_reader.get_aligned::<u32>(value_width).unwrap();
- }
- } else {
- panic!("Can't read VLQ int from BitReader");
- }
- }
-}
-
-impl Decoder for DictDecoder {
- fn read(&mut self, dst: &mut ParquetMutableVector) {
- let dst_slice = dst.value_buffer.as_slice_mut();
- let dst_offset = dst.num_values * 4;
-
- // We've finished the current run. Now load the next.
- if self.rle_left == 0 && self.bit_packed_left == 0 {
- self.reload();
- }
-
- let value = if self.rle_left > 0 {
- self.rle_left -= 1;
- self.current_value
- } else {
- self.bit_packed_left -= 1;
- self.bit_reader.get_u32_value(self.bit_width)
- };
-
- // Directly copy the value into the destination buffer
- unsafe {
- let dst = &mut dst_slice[dst_offset..] as *mut [u8] as *mut u8 as *mut u32;
- *dst = value;
- }
- }
-
- fn read_batch(&mut self, dst: &mut ParquetMutableVector, num: usize) {
- let mut values_read = 0;
- let dst_slice = dst.value_buffer.as_slice_mut();
- let mut dst_offset = dst.num_values * 4;
-
- while values_read < num {
- let num_to_read = num - values_read;
- let mut dst = &mut dst_slice[dst_offset..] as *mut [u8] as *mut u8 as *mut u32;
-
- if self.rle_left > 0 {
- let n = std::cmp::min(num_to_read, self.rle_left);
- unsafe {
- // Copy the current RLE value into the destination buffer.
- for _ in 0..n {
- *dst = self.current_value;
- dst = dst.offset(1);
- }
- dst_offset += 4 * n;
- }
- self.rle_left -= n;
- values_read += n;
- } else if self.bit_packed_left > 0 {
- let n = std::cmp::min(num_to_read, self.bit_packed_left);
- unsafe {
- // Decode the next `n` BitPacked values into u32 and put the result directly to
- // `dst`.
- self.bit_reader.get_u32_batch(dst, n, self.bit_width);
- }
- dst_offset += 4 * n;
- self.bit_packed_left -= n;
- values_read += n;
- } else {
- self.reload();
- }
- }
- }
-
- fn skip_batch(&mut self, num: usize) {
- let mut values_skipped = 0;
-
- while values_skipped < num {
- let num_to_skip = num - values_skipped;
-
- if self.rle_left > 0 {
- let n = std::cmp::min(num_to_skip, self.rle_left);
- self.rle_left -= n;
- values_skipped += n;
- } else if self.bit_packed_left > 0 {
- let n = std::cmp::min(num_to_skip, self.bit_packed_left);
- self.bit_reader.skip_bits(n * self.bit_width);
- self.bit_packed_left -= n;
- values_skipped += n;
- } else {
- self.reload();
- }
- }
- }
-
- fn encoding(&self) -> Encoding {
- Encoding::RLE_DICTIONARY
- }
-}
-
-#[cfg(test)]
-mod test {
- use super::*;
- use parquet::data_type::AsBytes;
-
- #[test]
- fn test_i32_to_i8() {
- let source =
- hex::decode("8a000000dbffffff1800000034ffffff300000001d000000abffffff37fffffff1000000")
- .unwrap();
- let expected = hex::decode("8adb1834301dab37f1").unwrap();
- let num = source.len() / 4;
- let mut dest: Vec<u8> = vec![b' '; num];
- copy_i32_to_i8(source.as_bytes(), dest.as_mut_slice(), num);
- assert_eq!(expected.as_bytes(), dest.as_bytes());
- }
-
- #[test]
- fn test_i32_to_u8() {
- let source =
- hex::decode("8a000000dbffffff1800000034ffffff300000001d000000abffffff37fffffff1000000")
- .unwrap();
- let expected = hex::decode("8a00dbff180034ff30001d00abff37fff100").unwrap();
- let num = source.len() / 4;
- let mut dest: Vec<u8> = vec![b' '; num * 2];
- copy_i32_to_u8(source.as_bytes(), dest.as_mut_slice(), num);
- assert_eq!(expected.as_bytes(), dest.as_bytes());
- }
-
- #[test]
- fn test_i32_to_i16() {
- let source =
- hex::decode("8a0e0000db93ffff1826000034f4ffff300200001d2b0000abe3ffff378dfffff1470000")
- .unwrap();
- let expected = hex::decode("8a0edb93182634f430021d2babe3378df147").unwrap();
- let num = source.len() / 4;
- let mut dest: Vec<u8> = vec![b' '; num * 2];
- copy_i32_to_i16(source.as_bytes(), dest.as_mut_slice(), num);
- assert_eq!(expected.as_bytes(), dest.as_bytes());
- }
-
- #[test]
- fn test_i32_to_u16() {
- let source = hex::decode(
- "ff7f0000008000000180000002800000038000000480000005800000068000000780000008800000",
- )
- .unwrap();
- let expected = hex::decode(
- "ff7f0000008000000180000002800000038000000480000005800000068000000780000008800000",
- )
- .unwrap();
- let num = source.len() / 4;
- let mut dest: Vec<u8> = vec![b' '; num * 4];
- copy_i32_to_u16(source.as_bytes(), dest.as_mut_slice(), num);
- assert_eq!(expected.as_bytes(), dest.as_bytes());
- }
-
- #[test]
- fn test_i32_to_u32() {
- let source = hex::decode(
- "ffffff7f000000800100008002000080030000800400008005000080060000800700008008000080",
- )
- .unwrap();
- let expected = hex::decode("ffffff7f00000000000000800000000001000080000000000200008000000000030000800000000004000080000000000500008000000000060000800000000007000080000000000800008000000000").unwrap();
- let num = source.len() / 4;
- let mut dest: Vec<u8> = vec![b' '; num * 8];
- copy_i32_to_u32(source.as_bytes(), dest.as_mut_slice(), num);
- assert_eq!(expected.as_bytes(), dest.as_bytes());
- }
-}
diff --git a/native/core/src/parquet/util/bit_packing.rs b/native/core/src/parquet/util/bit_packing.rs
deleted file mode 100644
index fd39ba8fa3..0000000000
--- a/native/core/src/parquet/util/bit_packing.rs
+++ /dev/null
@@ -1,3658 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-/// Unpack 32 values with bit width `num_bits` from `in_ptr`, and write to `out_ptr`.
-/// Return the `in_ptr` where the starting offset points to the first byte after all the
-/// bytes that were consumed.
-// TODO: may be better to make these more compact using if-else conditions.
-// However, this may require const generics:
-// https://github.com/rust-lang/rust/issues/44580
-// to eliminate the branching cost.
-// TODO: we should use SIMD instructions to further optimize this. I have explored
-// https://github.com/tantivy-search/bitpacking
-// but the layout it uses for SIMD is different from Parquet.
-// TODO: support packing as well, which is used for encoding.
-pub unsafe fn unpack32(mut in_ptr: *const u32, out_ptr: *mut u32, num_bits: usize) -> *const u32 {
- in_ptr = match num_bits {
- 0 => nullunpacker32(in_ptr, out_ptr),
- 1 => unpack1_32(in_ptr, out_ptr),
- 2 => unpack2_32(in_ptr, out_ptr),
- 3 => unpack3_32(in_ptr, out_ptr),
- 4 => unpack4_32(in_ptr, out_ptr),
- 5 => unpack5_32(in_ptr, out_ptr),
- 6 => unpack6_32(in_ptr, out_ptr),
- 7 => unpack7_32(in_ptr, out_ptr),
- 8 => unpack8_32(in_ptr, out_ptr),
- 9 => unpack9_32(in_ptr, out_ptr),
- 10 => unpack10_32(in_ptr, out_ptr),
- 11 => unpack11_32(in_ptr, out_ptr),
- 12 => unpack12_32(in_ptr, out_ptr),
- 13 => unpack13_32(in_ptr, out_ptr),
- 14 => unpack14_32(in_ptr, out_ptr),
- 15 => unpack15_32(in_ptr, out_ptr),
- 16 => unpack16_32(in_ptr, out_ptr),
- 17 => unpack17_32(in_ptr, out_ptr),
- 18 => unpack18_32(in_ptr, out_ptr),
- 19 => unpack19_32(in_ptr, out_ptr),
- 20 => unpack20_32(in_ptr, out_ptr),
- 21 => unpack21_32(in_ptr, out_ptr),
- 22 => unpack22_32(in_ptr, out_ptr),
- 23 => unpack23_32(in_ptr, out_ptr),
- 24 => unpack24_32(in_ptr, out_ptr),
- 25 => unpack25_32(in_ptr, out_ptr),
- 26 => unpack26_32(in_ptr, out_ptr),
- 27 => unpack27_32(in_ptr, out_ptr),
- 28 => unpack28_32(in_ptr, out_ptr),
- 29 => unpack29_32(in_ptr, out_ptr),
- 30 => unpack30_32(in_ptr, out_ptr),
- 31 => unpack31_32(in_ptr, out_ptr),
- 32 => unpack32_32(in_ptr, out_ptr),
- _ => unimplemented!(),
- };
- in_ptr
-}
-
-unsafe fn nullunpacker32(in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- for _ in 0..32 {
- *out = 0;
- out = out.offset(1);
- }
- in_buf
-}
-
-unsafe fn unpack1_32(in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 1) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 2) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 3) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 4) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 5) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 6) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 7) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 9) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 11) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 13) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 15) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 17) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 19) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 21) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 22) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 23) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 25) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 26) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 27) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 28) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 29) & 1;
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 30) & 1;
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack2_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 22) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 26) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 28) % (1u32 << 2);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
- *out = (in_buf.read_unaligned()) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 22) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 26) % (1u32 << 2);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 28) % (1u32 << 2);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack3_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 9) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 15) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 21) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 27) % (1u32 << 3);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (3 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 13) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 19) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 22) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 25) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 28) % (1u32 << 3);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (3 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 11) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 17) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 23) % (1u32 << 3);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 26) % (1u32 << 3);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack4_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 28) % (1u32 << 4);
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 28) % (1u32 << 4);
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 28) % (1u32 << 4);
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 4);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 28) % (1u32 << 4);
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack5_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 15) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 25) % (1u32 << 5);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (5 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 13) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 23) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 28) % (1u32 << 5);
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (5 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 11) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 21) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 26) % (1u32 << 5);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (5 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 9) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 19) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 5);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (5 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 17) % (1u32 << 5);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 22) % (1u32 << 5);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack6_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 6);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (6 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 22) % (1u32 << 6);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (6 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 6);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 6);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (6 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 22) % (1u32 << 6);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (6 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 6);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 6);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack7_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 21) % (1u32 << 7);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (7 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 17) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 24) % (1u32 << 7);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (7 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 13) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 7);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (7 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 9) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 23) % (1u32 << 7);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (7 - 5);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 19) % (1u32 << 7);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (7 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 15) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 22) % (1u32 << 7);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (7 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 11) % (1u32 << 7);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 7);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 25;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack8_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 8);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 8);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack9_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 9) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 9);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (9 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 13) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 22) % (1u32 << 9);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (9 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 17) % (1u32 << 9);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (9 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 21) % (1u32 << 9);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 7)) << (9 - 7);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 9);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (9 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 11) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 9);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (9 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 15) % (1u32 << 9);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (9 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 19) % (1u32 << 9);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (9 - 5);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 9);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 9);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 23;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack10_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 10);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (10 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 10);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (10 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 10);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (10 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 10);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (10 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 10);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 10);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (10 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 10);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (10 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 10);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (10 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 10);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (10 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 10);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 10);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack11_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 11);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 11) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (11 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 11);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (11 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 11);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 13) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (11 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 11);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (11 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 11);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 15) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (11 - 5);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 11);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (11 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 11);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 17) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 7)) << (11 - 7);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 11);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (11 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 11);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 19) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 9)) << (11 - 9);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 9) % (1u32 << 11);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 20) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (11 - 10);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 11);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 21;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack12_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 12);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (12 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 12);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (12 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 12);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (12 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 12);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (12 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 12);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (12 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 12);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (12 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 12);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (12 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 12);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (12 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 12);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack13_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 13);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 13) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 7)) << (13 - 7);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (13 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 13);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (13 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (13 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 13);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 15) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 9)) << (13 - 9);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 9) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (13 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 13);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (13 - 10);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (13 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 13);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 17) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 11)) << (13 - 11);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 11) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (13 - 5);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 13);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 18) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (13 - 12);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (13 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 13);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 19;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack14_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 14);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (14 - 10);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (14 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (14 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 14);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (14 - 12);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (14 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (14 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 14);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (14 - 10);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (14 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (14 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 14);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (14 - 12);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (14 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (14 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 14);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 18;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack15_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 15);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 15) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 13)) << (15 - 13);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 13) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 11)) << (15 - 11);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 11) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 9)) << (15 - 9);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 9) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 7)) << (15 - 7);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (15 - 5);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (15 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (15 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 15);
- out = out.offset(1);
- *out = ((in_buf.read_unaligned()) >> 16) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (15 - 14);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (15 - 12);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (15 - 10);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (15 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (15 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (15 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (15 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 15);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 17;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack16_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
- out = out.offset(1);
- in_buf = in_buf.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 16);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 16;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack17_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (17 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (17 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (17 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (17 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (17 - 10);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (17 - 12);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (17 - 14);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 14) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (17 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (17 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (17 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (17 - 5);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 7)) << (17 - 7);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 9)) << (17 - 9);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 9) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 11)) << (17 - 11);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 11) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 13)) << (17 - 13);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 13) % (1u32 << 17);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 15)) << (17 - 15);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 15;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack18_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (18 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (18 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (18 - 12);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (18 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (18 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (18 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (18 - 10);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (18 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (18 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (18 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (18 - 12);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (18 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (18 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (18 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (18 - 10);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 18);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (18 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack19_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (19 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (19 - 12);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 12) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 18)) << (19 - 18);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (19 - 5);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 11)) << (19 - 11);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 11) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 17)) << (19 - 17);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (19 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (19 - 10);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (19 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (19 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 9)) << (19 - 9);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 9) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 15)) << (19 - 15);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (19 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (19 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (19 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (19 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 7)) << (19 - 7);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 19);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 13)) << (19 - 13);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 13;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack20_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (20 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (20 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (20 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (20 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (20 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (20 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (20 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (20 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (20 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (20 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (20 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (20 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (20 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (20 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (20 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 20);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (20 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack21_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (21 - 10);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 10) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (21 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 9)) << (21 - 9);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 9) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 19)) << (21 - 19);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (21 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 18)) << (21 - 18);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 7)) << (21 - 7);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 17)) << (21 - 17);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (21 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (21 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (21 - 5);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 15)) << (21 - 15);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (21 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (21 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (21 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 13)) << (21 - 13);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 13;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (21 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (21 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (21 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 21);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 11)) << (21 - 11);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 11;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack22_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 22);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (22 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (22 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 22);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (22 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (22 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 22);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (22 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (22 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 22);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 18)) << (22 - 18);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (22 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 22);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (22 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (22 - 10);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 10;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 22);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (22 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (22 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 22);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (22 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (22 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 22);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (22 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (22 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 22);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 18)) << (22 - 18);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (22 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 22);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (22 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (22 - 10);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 10;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack23_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 23);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (23 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (23 - 5);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 23);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 19)) << (23 - 19);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (23 - 10);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (23 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 23);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 15)) << (23 - 15);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (23 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 23);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (23 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 11)) << (23 - 11);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 11;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (23 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 23);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (23 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 7)) << (23 - 7);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 7) % (1u32 << 23);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 21)) << (23 - 21);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (23 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (23 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 23);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 17)) << (23 - 17);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (23 - 8);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 8) % (1u32 << 23);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 22)) << (23 - 22);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 13)) << (23 - 13);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 13;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (23 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 23);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 18)) << (23 - 18);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 9)) << (23 - 9);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 9;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack24_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 24);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 24);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 24);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 24);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 24);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 24);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 24);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 24);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (24 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (24 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack25_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 25);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 18)) << (25 - 18);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 11)) << (25 - 11);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 11;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (25 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 25);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 22)) << (25 - 22);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 15)) << (25 - 15);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (25 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (25 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 25);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 19)) << (25 - 19);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (25 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (25 - 5);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 5) % (1u32 << 25);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 23)) << (25 - 23);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (25 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 9)) << (25 - 9);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 9;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (25 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 25);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (25 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 13)) << (25 - 13);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 13;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (25 - 6);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 6) % (1u32 << 25);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 24)) << (25 - 24);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 17)) << (25 - 17);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (25 - 10);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (25 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 25);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 21)) << (25 - 21);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (25 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 7)) << (25 - 7);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 7;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack26_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 26);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (26 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (26 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (26 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (26 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 26);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 22)) << (26 - 22);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (26 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (26 - 10);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (26 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 26);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 24)) << (26 - 24);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 18)) << (26 - 18);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (26 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (26 - 6);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 6;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 26);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (26 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (26 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (26 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (26 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 26);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 22)) << (26 - 22);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (26 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (26 - 10);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (26 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 26);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 24)) << (26 - 24);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 18)) << (26 - 18);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (26 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (26 - 6);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 6;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack27_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 27);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 27;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 22)) << (27 - 22);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 22;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 17)) << (27 - 17);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (27 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 7)) << (27 - 7);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 7;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 2)) << (27 - 2);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 2) % (1u32 << 27);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 24)) << (27 - 24);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 19)) << (27 - 19);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 19;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (27 - 14);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 14;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 9)) << (27 - 9);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 9;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (27 - 4);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 4) % (1u32 << 27);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 31;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 26)) << (27 - 26);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 21)) << (27 - 21);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 21;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (27 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 11)) << (27 - 11);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 11;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 6)) << (27 - 6);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 6;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 1)) << (27 - 1);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 1) % (1u32 << 27);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 23)) << (27 - 23);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 18)) << (27 - 18);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 18;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 13)) << (27 - 13);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 13;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (27 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 3)) << (27 - 3);
- out = out.offset(1);
-
- *out = ((in_buf.read_unaligned()) >> 3) % (1u32 << 27);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 30;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 25)) << (27 - 25);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 25;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (27 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 15)) << (27 - 15);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 15;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 10)) << (27 - 10);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 10;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 5)) << (27 - 5);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 5;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack28_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 28);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 24)) << (28 - 24);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (28 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (28 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (28 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (28 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (28 - 4);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 4;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 28);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 24)) << (28 - 24);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (28 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (28 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (28 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (28 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (28 - 4);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 4;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 28);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 24)) << (28 - 24);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (28 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (28 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (28 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (28 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (28 - 4);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 4;
- in_buf = in_buf.offset(1);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) % (1u32 << 28);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 28;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 24)) << (28 - 24);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 24;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (28 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 16)) << (28 - 16);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 16;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 12)) << (28 - 12);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 12;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 8)) << (28 - 8);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 8;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 4)) << (28 - 4);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 4;
-
- in_buf.offset(1)
-}
-
-unsafe fn unpack29_32(mut in_buf: *const u32, mut out: *mut u32) -> *const u32 {
- *out = (in_buf.read_unaligned()) % (1u32 << 29);
- out = out.offset(1);
- *out = (in_buf.read_unaligned()) >> 29;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 26)) << (29 - 26);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 26;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 23)) << (29 - 23);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 23;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 20)) << (29 - 20);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 20;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 17)) << (29 - 17);
- out = out.offset(1);
-
- *out = (in_buf.read_unaligned()) >> 17;
- in_buf = in_buf.offset(1);
- *out |= ((in_buf.read_unaligned()) % (1u32 << 14)) << (29 - 14);
- out = out.offset(1);
-
... 13623 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org