You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@parquet.apache.org by "ASF GitHub Bot (JIRA)" <ji...@apache.org> on 2018/12/06 07:56:04 UTC
[jira] [Commented] (PARQUET-1228) parquet-format code changes for
encryption support
[ https://issues.apache.org/jira/browse/PARQUET-1228?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16711092#comment-16711092 ]
ASF GitHub Bot commented on PARQUET-1228:
-----------------------------------------
ggershinsky closed pull request #532: PARQUET-1228: parquet-format code changes for encryption support
URL: https://github.com/apache/parquet-mr/pull/532
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/parquet-arrow/src/main/java/org/apache/parquet/arrow/schema/SchemaConverter.java b/parquet-arrow/src/main/java/org/apache/parquet/arrow/schema/SchemaConverter.java
index e02b03b5d..51057c589 100644
--- a/parquet-arrow/src/main/java/org/apache/parquet/arrow/schema/SchemaConverter.java
+++ b/parquet-arrow/src/main/java/org/apache/parquet/arrow/schema/SchemaConverter.java
@@ -23,6 +23,7 @@
import static java.util.Optional.of;
import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MICROS;
import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MILLIS;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.NANOS;
import static org.apache.parquet.schema.LogicalTypeAnnotation.dateType;
import static org.apache.parquet.schema.LogicalTypeAnnotation.decimalType;
import static org.apache.parquet.schema.LogicalTypeAnnotation.intType;
@@ -246,6 +247,8 @@ public TypeMapping visit(Time type) {
return primitive(INT32, timeType(false, MILLIS));
} else if (bitWidth == 64 && timeUnit == TimeUnit.MICROSECOND) {
return primitive(INT64, timeType(false, MICROS));
+ } else if (bitWidth == 64 && timeUnit == TimeUnit.NANOSECOND) {
+ return primitive(INT64, timeType(false, NANOS));
}
throw new UnsupportedOperationException("Unsupported type " + type);
}
@@ -257,6 +260,8 @@ public TypeMapping visit(Timestamp type) {
return primitive(INT64, timestampType(isUtcNormalized(type), MILLIS));
} else if (timeUnit == TimeUnit.MICROSECOND) {
return primitive(INT64, timestampType(isUtcNormalized(type), MICROS));
+ } else if (timeUnit == TimeUnit.NANOSECOND) {
+ return primitive(INT64, timestampType(isUtcNormalized(type), NANOS));
}
throw new UnsupportedOperationException("Unsupported type " + type);
}
@@ -460,6 +465,8 @@ public TypeMapping convertINT64(PrimitiveTypeName primitiveTypeName) throws Runt
public Optional<TypeMapping> visit(LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeLogicalType) {
if (timeLogicalType.getUnit() == MICROS) {
return of(field(new ArrowType.Time(TimeUnit.MICROSECOND, 64)));
+ } else if (timeLogicalType.getUnit() == NANOS) {
+ return of(field(new ArrowType.Time(TimeUnit.NANOSECOND, 64)));
}
return empty();
}
@@ -471,6 +478,8 @@ public TypeMapping convertINT64(PrimitiveTypeName primitiveTypeName) throws Runt
return of(field(new ArrowType.Timestamp(TimeUnit.MICROSECOND, getTimeZone(timestampLogicalType))));
case MILLIS:
return of(field(new ArrowType.Timestamp(TimeUnit.MILLISECOND, getTimeZone(timestampLogicalType))));
+ case NANOS:
+ return of(field(new ArrowType.Timestamp(TimeUnit.NANOSECOND, getTimeZone(timestampLogicalType))));
}
return empty();
}
diff --git a/parquet-arrow/src/test/java/org/apache/parquet/arrow/schema/TestSchemaConverter.java b/parquet-arrow/src/test/java/org/apache/parquet/arrow/schema/TestSchemaConverter.java
index 2817de263..c962b5456 100644
--- a/parquet-arrow/src/test/java/org/apache/parquet/arrow/schema/TestSchemaConverter.java
+++ b/parquet-arrow/src/test/java/org/apache/parquet/arrow/schema/TestSchemaConverter.java
@@ -21,6 +21,7 @@
import static java.util.Arrays.asList;
import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MICROS;
import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MILLIS;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.NANOS;
import static org.apache.parquet.schema.LogicalTypeAnnotation.timeType;
import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType;
import static org.apache.parquet.schema.OriginalType.DATE;
@@ -66,6 +67,7 @@
import org.apache.parquet.arrow.schema.SchemaMapping.TypeMappingVisitor;
import org.apache.parquet.arrow.schema.SchemaMapping.UnionTypeMapping;
import org.apache.parquet.example.Paper;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.Types;
import org.junit.Assert;
@@ -93,6 +95,7 @@ private static Field field(String name, ArrowType type, Field... children) {
field("f", new ArrowType.FixedSizeList(1), field(null, new ArrowType.Date(DateUnit.DAY))),
field("g", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
field("h", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")),
+ field("i", new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")),
field("j", new ArrowType.Timestamp(TimeUnit.MILLISECOND, null)),
field("k", new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC")),
field("l", new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)),
@@ -112,6 +115,7 @@ private static Field field(String name, ArrowType type, Field... children) {
.named("f"))
.addField(Types.optional(FLOAT).named("g"))
.addField(Types.optional(INT64).as(timestampType(true, MILLIS)).named("h"))
+ .addField(Types.optional(INT64).as(timestampType(true, NANOS)).named("i"))
.addField(Types.optional(INT64).as(timestampType(false, MILLIS)).named("j"))
.addField(Types.optional(INT64).as(timestampType(true, MICROS)).named("k"))
.addField(Types.optional(INT64).as(timestampType(false, MICROS)).named("l"))
@@ -144,8 +148,10 @@ private static Field field(String name, ArrowType type, Field... children) {
field("m", new ArrowType.Time(TimeUnit.MILLISECOND, 32)),
field("n", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")),
field("o", new ArrowType.Interval(IntervalUnit.DAY_TIME)),
- field("o1", new ArrowType.Interval(IntervalUnit.YEAR_MONTH))
- ));
+ field("o1", new ArrowType.Interval(IntervalUnit.YEAR_MONTH)),
+ field("p", new ArrowType.Time(TimeUnit.NANOSECOND, 64)),
+ field("q", new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"))
+ ));
private final MessageType allTypesParquetSchema = Types.buildMessage()
.addField(Types.optional(BINARY).named("a"))
@@ -182,6 +188,8 @@ private static Field field(String name, ArrowType type, Field... children) {
.addField(Types.optional(INT64).as(TIMESTAMP_MILLIS).named("n"))
.addField(Types.optional(FIXED_LEN_BYTE_ARRAY).length(12).as(INTERVAL).named("o"))
.addField(Types.optional(FIXED_LEN_BYTE_ARRAY).length(12).as(INTERVAL).named("o1"))
+ .addField(Types.optional(INT64).as(timeType(false, NANOS)).named("p"))
+ .addField(Types.optional(INT64).as(timestampType(true, NANOS)).named("q"))
.named("root");
private final Schema supportedTypesArrowSchema = new Schema(asList(
@@ -205,7 +213,9 @@ private static Field field(String name, ArrowType type, Field... children) {
field("j2", new ArrowType.Decimal(25, 5)),
field("k", new ArrowType.Date(DateUnit.DAY)),
field("l", new ArrowType.Time(TimeUnit.MILLISECOND, 32)),
- field("m", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"))
+ field("m", new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC")),
+ field("n", new ArrowType.Time(TimeUnit.NANOSECOND, 64)),
+ field("o", new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"))
));
private final MessageType supportedTypesParquetSchema = Types.buildMessage()
@@ -234,6 +244,8 @@ private static Field field(String name, ArrowType type, Field... children) {
.addField(Types.optional(INT32).as(DATE).named("k"))
.addField(Types.optional(INT32).as(TIME_MILLIS).named("l"))
.addField(Types.optional(INT64).as(TIMESTAMP_MILLIS).named("m"))
+ .addField(Types.optional(INT64).as(timeType(true, NANOS)).named("n"))
+ .addField(Types.optional(INT64).as(timestampType(true, NANOS)).named("o"))
.named("root");
private final Schema paperArrowSchema = new Schema(asList(
@@ -307,7 +319,7 @@ private void compareFields(List<Field> left, List<Field> right) {
@Test
public void testAllMap() throws IOException {
SchemaMapping map = converter.map(allTypesArrowSchema, allTypesParquetSchema);
- Assert.assertEquals("p, s<p>, l<p>, l<p>, u<p>, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p", toSummaryString(map));
+ Assert.assertEquals("p, s<p>, l<p>, l<p>, u<p>, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p", toSummaryString(map));
}
private String toSummaryString(SchemaMapping map) {
@@ -387,13 +399,6 @@ public void testArrowTimeMicrosecondToParquet() {
Types.buildMessage().addField(Types.optional(INT64).as(timeType(false, MICROS)).named("a")).named("root"));
}
- @Test(expected = UnsupportedOperationException.class)
- public void testArrowTimeNanosecondToParquet() {
- converter.fromArrow(new Schema(asList(
- field("a", new ArrowType.Time(TimeUnit.NANOSECOND, 64))
- ))).getParquetSchema();
- }
-
@Test
public void testParquetInt32TimeMillisToArrow() {
MessageType parquet = Types.buildMessage()
@@ -449,13 +454,6 @@ public void testArrowTimestampMicrosecondToParquet() {
Assert.assertEquals(expected, Types.buildMessage().addField(Types.optional(INT64).as(TIMESTAMP_MICROS).named("a")).named("root"));
}
- @Test(expected = UnsupportedOperationException.class)
- public void testArrowTimestampNanosecondToParquet() {
- converter.fromArrow(new Schema(asList(
- field("a", new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"))
- ))).getParquetSchema();
- }
-
@Test
public void testParquetInt64TimestampMillisToArrow() {
MessageType parquet = Types.buildMessage()
diff --git a/parquet-cli/src/main/java/org/apache/parquet/cli/Main.java b/parquet-cli/src/main/java/org/apache/parquet/cli/Main.java
index 990193c73..fa69ce7a4 100644
--- a/parquet-cli/src/main/java/org/apache/parquet/cli/Main.java
+++ b/parquet-cli/src/main/java/org/apache/parquet/cli/Main.java
@@ -32,6 +32,7 @@
import org.apache.parquet.cli.commands.ConvertCommand;
import org.apache.parquet.cli.commands.ParquetMetadataCommand;
import org.apache.parquet.cli.commands.SchemaCommand;
+import org.apache.parquet.cli.commands.ShowColumnIndexCommand;
import org.apache.parquet.cli.commands.ShowDictionaryCommand;
import org.apache.parquet.cli.commands.ShowPagesCommand;
import org.apache.parquet.cli.commands.ToAvroCommand;
@@ -87,6 +88,7 @@
jc.addCommand("to-avro", new ToAvroCommand(console));
jc.addCommand("cat", new CatCommand(console, 0));
jc.addCommand("head", new CatCommand(console, 10));
+ jc.addCommand("column-index", new ShowColumnIndexCommand(console));
}
@Override
diff --git a/parquet-cli/src/main/java/org/apache/parquet/cli/commands/ShowColumnIndexCommand.java b/parquet-cli/src/main/java/org/apache/parquet/cli/commands/ShowColumnIndexCommand.java
new file mode 100644
index 000000000..38a7094b8
--- /dev/null
+++ b/parquet-cli/src/main/java/org/apache/parquet/cli/commands/ShowColumnIndexCommand.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.cli.commands;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.parquet.cli.BaseCommand;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.io.InputFile;
+import org.slf4j.Logger;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/**
+ * parquet-cli command to print column and offset indexes.
+ */
+@Parameters(commandDescription = "Prints the column and offset indexes of a Parquet file")
+public class ShowColumnIndexCommand extends BaseCommand {
+ public ShowColumnIndexCommand(Logger console) {
+ super(console);
+ }
+
+ @Parameter(description = "<parquet path>")
+ List<String> files;
+
+ @Parameter(names = { "-c", "--column" }, description = "Shows the column/offset indexes for the given column only")
+ List<String> ColumnPaths;
+
+ @Parameter(names = { "-r",
+ "--row-group" }, description = "Shows the column/offset indexes for the given row-groups only; "
+ + "row-groups are referenced by their indexes from 0")
+ List<String> rowGroupIndexes;
+
+ @Parameter(names = { "-i", "--column-index" }, description = "Shows the column indexes; "
+ + "active by default unless -o is used")
+ boolean showColumnIndex;
+
+ @Parameter(names = { "-o", "--offset-index" }, description = "Shows the offset indexes; "
+ + "active by default unless -i is used")
+ boolean showOffsetIndex;
+
+ @Override
+ public List<String> getExamples() {
+ return Lists.newArrayList(
+ "# Show only column indexes for column 'col' from a Parquet file",
+ "-c col -i sample.parquet");
+ }
+
+ @Override
+ public int run() throws IOException {
+ Preconditions.checkArgument(files != null && files.size() >= 1,
+ "A Parquet file is required.");
+ Preconditions.checkArgument(files.size() == 1,
+ "Cannot process multiple Parquet files.");
+
+ InputFile in = HadoopInputFile.fromPath(qualifiedPath(files.get(0)), getConf());
+ if (!showColumnIndex && !showOffsetIndex) {
+ showColumnIndex = true;
+ showOffsetIndex = true;
+ }
+
+ Set<String> rowGroupIndexSet = new HashSet<>();
+ if (rowGroupIndexes != null) {
+ rowGroupIndexSet.addAll(rowGroupIndexes);
+ }
+
+ try (ParquetFileReader reader = ParquetFileReader.open(in)) {
+ boolean firstBlock = true;
+ int rowGroupIndex = 0;
+ for (BlockMetaData block : reader.getFooter().getBlocks()) {
+ if (!rowGroupIndexSet.isEmpty() && !rowGroupIndexSet.contains(Integer.toString(rowGroupIndex))) {
+ ++rowGroupIndex;
+ continue;
+ }
+ if (!firstBlock) {
+ console.info("");
+ }
+ firstBlock = false;
+ console.info("row-group {}:", rowGroupIndex);
+ for (ColumnChunkMetaData column : getColumns(block)) {
+ String path = column.getPath().toDotString();
+ if (showColumnIndex) {
+ console.info("column index for column {}:", path);
+ ColumnIndex columnIndex = reader.readColumnIndex(column);
+ if (columnIndex == null) {
+ console.info("NONE");
+ } else {
+ console.info(columnIndex.toString());
+ }
+ }
+ if (showOffsetIndex) {
+ console.info("offset index for column {}:", path);
+ OffsetIndex offsetIndex = reader.readOffsetIndex(column);
+ if (offsetIndex == null) {
+ console.info("NONE");
+ } else {
+ console.info(offsetIndex.toString());
+ }
+ }
+ }
+ ++rowGroupIndex;
+ }
+ }
+ return 0;
+ }
+
+ private List<ColumnChunkMetaData> getColumns(BlockMetaData block) {
+ List<ColumnChunkMetaData> columns = block.getColumns();
+ if (ColumnPaths == null || ColumnPaths.isEmpty()) {
+ return columns;
+ }
+ Map<String, ColumnChunkMetaData> pathMap = new HashMap<>();
+ for (ColumnChunkMetaData column : columns) {
+ pathMap.put(column.getPath().toDotString(), column);
+ }
+
+ List<ColumnChunkMetaData> filtered = new ArrayList<>();
+ for (String path : ColumnPaths) {
+ ColumnChunkMetaData column = pathMap.get(path);
+ if (column != null) {
+ filtered.add(column);
+ }
+ }
+ return filtered;
+ }
+
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ColumnReader.java b/parquet-column/src/main/java/org/apache/parquet/column/ColumnReader.java
index 52d269ef0..6d93eeed5 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/ColumnReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/ColumnReader.java
@@ -41,7 +41,10 @@
/**
* @return the totalCount of values to be consumed
+ * @deprecated will be removed in 2.0.0; Total values might not be able to be counted before reading the values (e.g.
+ * in case of column index based filtering)
*/
+ @Deprecated
long getTotalValueCount();
/**
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java
index 39b65da9f..b17323933 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java
@@ -47,6 +47,7 @@
public static final boolean DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK = true;
public static final int DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK = 100;
public static final int DEFAULT_MAXIMUM_RECORD_COUNT_FOR_CHECK = 10000;
+ public static final int DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH = 64;
public static final ValuesWriterFactory DEFAULT_VALUES_WRITER_FACTORY = new DefaultValuesWriterFactory();
@@ -83,10 +84,11 @@ public static WriterVersion fromString(String name) {
private final boolean estimateNextSizeCheck;
private final ByteBufferAllocator allocator;
private final ValuesWriterFactory valuesWriterFactory;
+ private final int columnIndexTruncateLength;
private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPageSize, boolean enableDict, int minRowCountForPageSizeCheck,
int maxRowCountForPageSizeCheck, boolean estimateNextSizeCheck, ByteBufferAllocator allocator,
- ValuesWriterFactory writerFactory) {
+ ValuesWriterFactory writerFactory, int columnIndexMinMaxTruncateLength) {
this.pageSizeThreshold = pageSize;
this.initialSlabSize = CapacityByteArrayOutputStream
.initialSlabSizeHeuristic(MIN_SLAB_SIZE, pageSizeThreshold, 10);
@@ -99,6 +101,7 @@ private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPag
this.allocator = allocator;
this.valuesWriterFactory = writerFactory;
+ this.columnIndexTruncateLength = columnIndexMinMaxTruncateLength;
}
public ValuesWriter newRepetitionLevelWriter(ColumnDescriptor path) {
@@ -163,7 +166,7 @@ public ColumnWriteStore newColumnWriteStore(MessageType schema,
PageWriteStore pageStore) {
switch (writerVersion) {
case PARQUET_1_0:
- return new ColumnWriteStoreV1(pageStore, this);
+ return new ColumnWriteStoreV1(schema, pageStore, this);
case PARQUET_2_0:
return new ColumnWriteStoreV2(schema, pageStore, this);
default:
@@ -183,6 +186,10 @@ public ValuesWriterFactory getValuesWriterFactory() {
return valuesWriterFactory;
}
+ public int getColumnIndexTruncateLength() {
+ return columnIndexTruncateLength;
+ }
+
public boolean estimateNextSizeCheck() {
return estimateNextSizeCheck;
}
@@ -205,6 +212,7 @@ public static Builder copy(ParquetProperties toCopy) {
private boolean estimateNextSizeCheck = DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK;
private ByteBufferAllocator allocator = new HeapByteBufferAllocator();
private ValuesWriterFactory valuesWriterFactory = DEFAULT_VALUES_WRITER_FACTORY;
+ private int columnIndexTruncateLength = DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH;
private Builder() {
}
@@ -299,11 +307,17 @@ public Builder withValuesWriterFactory(ValuesWriterFactory factory) {
return this;
}
+ public Builder withColumnIndexTruncateLength(int length) {
+ Preconditions.checkArgument(length > 0, "Invalid column index min/max truncate length (negative) : %s", length);
+ this.columnIndexTruncateLength = length;
+ return this;
+ }
+
public ParquetProperties build() {
ParquetProperties properties =
new ParquetProperties(writerVersion, pageSize, dictPageSize,
enableDict, minRowCountForPageSizeCheck, maxRowCountForPageSizeCheck,
- estimateNextSizeCheck, allocator, valuesWriterFactory);
+ estimateNextSizeCheck, allocator, valuesWriterFactory, columnIndexTruncateLength);
// we pass a constructed but uninitialized factory to ParquetProperties above as currently
// creation of ValuesWriters is invoked from within ParquetProperties. In the future
// we'd like to decouple that and won't need to pass an object to properties and then pass the
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReadStoreImpl.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReadStoreImpl.java
index e582908ca..b7e159775 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReadStoreImpl.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReadStoreImpl.java
@@ -18,6 +18,9 @@
*/
package org.apache.parquet.column.impl;
+import java.util.Optional;
+import java.util.PrimitiveIterator;
+
import org.apache.parquet.VersionParser;
import org.apache.parquet.VersionParser.ParsedVersion;
import org.apache.parquet.VersionParser.VersionParseException;
@@ -72,7 +75,14 @@ public ColumnReadStoreImpl(PageReadStore pageReadStore,
@Override
public ColumnReader getColumnReader(ColumnDescriptor path) {
- return newMemColumnReader(path, pageReadStore.getPageReader(path));
+ PrimitiveConverter converter = getPrimitiveConverter(path);
+ PageReader pageReader = pageReadStore.getPageReader(path);
+ Optional<PrimitiveIterator.OfLong> rowIndexes = pageReadStore.getRowIndexes();
+ if (rowIndexes.isPresent()) {
+ return new SynchronizingColumnReader(path, pageReader, converter, writerVersion, rowIndexes.get());
+ } else {
+ return new ColumnReaderImpl(path, pageReader, converter, writerVersion);
+ }
}
public ColumnReaderImpl newMemColumnReader(ColumnDescriptor path, PageReader pageReader) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderBase.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderBase.java
new file mode 100644
index 000000000..c929431c6
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderBase.java
@@ -0,0 +1,760 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.impl;
+
+import static java.lang.String.format;
+import static org.apache.parquet.Preconditions.checkNotNull;
+import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL;
+import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL;
+import static org.apache.parquet.column.ValuesType.VALUES;
+
+import java.io.IOException;
+
+import org.apache.parquet.CorruptDeltaByteArrays;
+import org.apache.parquet.VersionParser.ParsedVersion;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.ColumnReader;
+import org.apache.parquet.column.Dictionary;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.page.DataPage;
+import org.apache.parquet.column.page.DataPageV1;
+import org.apache.parquet.column.page.DataPageV2;
+import org.apache.parquet.column.page.DictionaryPage;
+import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.column.values.RequiresPreviousReader;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder;
+import org.apache.parquet.io.ParquetDecodingException;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.PrimitiveConverter;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeNameConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base superclass for {@link ColumnReader} implementations.
+ */
+abstract class ColumnReaderBase implements ColumnReader {
+ private static final Logger LOG = LoggerFactory.getLogger(ColumnReaderBase.class);
+
+ /**
+ * binds the lower level page decoder to the record converter materializing the records
+ */
+ private static abstract class Binding {
+
+ /**
+ * read one value from the underlying page
+ */
+ abstract void read();
+
+ /**
+ * skip one value from the underlying page
+ */
+ abstract void skip();
+
+ /**
+ * Skips n values from the underlying page
+ *
+ * @param n
+ * the number of values to be skipped
+ */
+ abstract void skip(int n);
+
+ /**
+ * write current value to converter
+ */
+ abstract void writeValue();
+
+ /**
+ * @return current value
+ */
+ public int getDictionaryId() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * @return current value
+ */
+ public int getInteger() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * @return current value
+ */
+ public boolean getBoolean() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * @return current value
+ */
+ public long getLong() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * @return current value
+ */
+ public Binary getBinary() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * @return current value
+ */
+ public float getFloat() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * @return current value
+ */
+ public double getDouble() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private final ParsedVersion writerVersion;
+ private final ColumnDescriptor path;
+ private final long totalValueCount;
+ private final PageReader pageReader;
+ private final Dictionary dictionary;
+
+ private IntIterator repetitionLevelColumn;
+ private IntIterator definitionLevelColumn;
+ protected ValuesReader dataColumn;
+ private Encoding currentEncoding;
+
+ private int repetitionLevel;
+ private int definitionLevel;
+ private int dictionaryId;
+
+ private long endOfPageValueCount;
+ private long readValues = 0;
+ private int pageValueCount = 0;
+
+ private final PrimitiveConverter converter;
+ private Binding binding;
+ private final int maxDefinitionLevel;
+
+ // this is needed because we will attempt to read the value twice when filtering
+ // TODO: rework that
+ private boolean valueRead;
+
+ private void bindToDictionary(final Dictionary dictionary) {
+ binding =
+ new Binding() {
+ void read() {
+ dictionaryId = dataColumn.readValueDictionaryId();
+ }
+ public void skip() {
+ dataColumn.skip();
+ }
+ @Override
+ void skip(int n) {
+ dataColumn.skip(n);
+ }
+ public int getDictionaryId() {
+ return dictionaryId;
+ }
+ void writeValue() {
+ converter.addValueFromDictionary(dictionaryId);
+ }
+ public int getInteger() {
+ return dictionary.decodeToInt(dictionaryId);
+ }
+ public boolean getBoolean() {
+ return dictionary.decodeToBoolean(dictionaryId);
+ }
+ public long getLong() {
+ return dictionary.decodeToLong(dictionaryId);
+ }
+ public Binary getBinary() {
+ return dictionary.decodeToBinary(dictionaryId);
+ }
+ public float getFloat() {
+ return dictionary.decodeToFloat(dictionaryId);
+ }
+ public double getDouble() {
+ return dictionary.decodeToDouble(dictionaryId);
+ }
+ };
+ }
+
+ private void bind(PrimitiveTypeName type) {
+ binding = type.convert(new PrimitiveTypeNameConverter<Binding, RuntimeException>() {
+ @Override
+ public Binding convertFLOAT(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
+ return new Binding() {
+ float current;
+ void read() {
+ current = dataColumn.readFloat();
+ }
+ public void skip() {
+ current = 0;
+ dataColumn.skip();
+ }
+ @Override
+ void skip(int n) {
+ current = 0;
+ dataColumn.skip(n);
+ }
+ public float getFloat() {
+ return current;
+ }
+ void writeValue() {
+ converter.addFloat(current);
+ }
+ };
+ }
+ @Override
+ public Binding convertDOUBLE(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
+ return new Binding() {
+ double current;
+ void read() {
+ current = dataColumn.readDouble();
+ }
+ public void skip() {
+ current = 0;
+ dataColumn.skip();
+ }
+ @Override
+ void skip(int n) {
+ current = 0;
+ dataColumn.skip(n);
+ }
+ public double getDouble() {
+ return current;
+ }
+ void writeValue() {
+ converter.addDouble(current);
+ }
+ };
+ }
+ @Override
+ public Binding convertINT32(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
+ return new Binding() {
+ int current;
+ void read() {
+ current = dataColumn.readInteger();
+ }
+ public void skip() {
+ current = 0;
+ dataColumn.skip();
+ }
+ @Override
+ void skip(int n) {
+ current = 0;
+ dataColumn.skip(n);
+ }
+ @Override
+ public int getInteger() {
+ return current;
+ }
+ void writeValue() {
+ converter.addInt(current);
+ }
+ };
+ }
+ @Override
+ public Binding convertINT64(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
+ return new Binding() {
+ long current;
+ void read() {
+ current = dataColumn.readLong();
+ }
+ public void skip() {
+ current = 0;
+ dataColumn.skip();
+ }
+ @Override
+ void skip(int n) {
+ current = 0;
+ dataColumn.skip(n);
+ }
+ @Override
+ public long getLong() {
+ return current;
+ }
+ void writeValue() {
+ converter.addLong(current);
+ }
+ };
+ }
+ @Override
+ public Binding convertINT96(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
+ return this.convertBINARY(primitiveTypeName);
+ }
+ @Override
+ public Binding convertFIXED_LEN_BYTE_ARRAY(
+ PrimitiveTypeName primitiveTypeName) throws RuntimeException {
+ return this.convertBINARY(primitiveTypeName);
+ }
+ @Override
+ public Binding convertBOOLEAN(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
+ return new Binding() {
+ boolean current;
+ void read() {
+ current = dataColumn.readBoolean();
+ }
+ public void skip() {
+ current = false;
+ dataColumn.skip();
+ }
+ @Override
+ void skip(int n) {
+ current = false;
+ dataColumn.skip(n);
+ }
+ @Override
+ public boolean getBoolean() {
+ return current;
+ }
+ void writeValue() {
+ converter.addBoolean(current);
+ }
+ };
+ }
+ @Override
+ public Binding convertBINARY(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
+ return new Binding() {
+ Binary current;
+ void read() {
+ current = dataColumn.readBytes();
+ }
+ public void skip() {
+ current = null;
+ dataColumn.skip();
+ }
+ @Override
+ void skip(int n) {
+ current = null;
+ dataColumn.skip(n);
+ }
+ @Override
+ public Binary getBinary() {
+ return current;
+ }
+ void writeValue() {
+ converter.addBinary(current);
+ }
+ };
+ }
+ });
+ }
+
+ /**
+ * creates a reader for triplets
+ * @param path the descriptor for the corresponding column
+ * @param pageReader the underlying store to read from
+ * @param converter a converter that materializes the values in this column in the current record
+ * @param writerVersion writer version string from the Parquet file being read
+ */
+ ColumnReaderBase(ColumnDescriptor path, PageReader pageReader, PrimitiveConverter converter, ParsedVersion writerVersion) {
+ this.path = checkNotNull(path, "path");
+ this.pageReader = checkNotNull(pageReader, "pageReader");
+ this.converter = checkNotNull(converter, "converter");
+ this.writerVersion = writerVersion;
+ this.maxDefinitionLevel = path.getMaxDefinitionLevel();
+ DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
+ if (dictionaryPage != null) {
+ try {
+ this.dictionary = dictionaryPage.getEncoding().initDictionary(path, dictionaryPage);
+ if (converter.hasDictionarySupport()) {
+ converter.setDictionary(dictionary);
+ }
+ } catch (IOException e) {
+ throw new ParquetDecodingException("could not decode the dictionary for " + path, e);
+ }
+ } else {
+ this.dictionary = null;
+ }
+ this.totalValueCount = pageReader.getTotalValueCount();
+ if (totalValueCount <= 0) {
+ throw new ParquetDecodingException("totalValueCount '" + totalValueCount + "' <= 0");
+ }
+ }
+
+ boolean isFullyConsumed() {
+ return readValues >= totalValueCount;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#writeCurrentValueToConverter()
+ */
+ @Override
+ public void writeCurrentValueToConverter() {
+ readValue();
+ this.binding.writeValue();
+ }
+
+ @Override
+ public int getCurrentValueDictionaryID() {
+ readValue();
+ return binding.getDictionaryId();
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#getInteger()
+ */
+ @Override
+ public int getInteger() {
+ readValue();
+ return this.binding.getInteger();
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#getBoolean()
+ */
+ @Override
+ public boolean getBoolean() {
+ readValue();
+ return this.binding.getBoolean();
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#getLong()
+ */
+ @Override
+ public long getLong() {
+ readValue();
+ return this.binding.getLong();
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#getBinary()
+ */
+ @Override
+ public Binary getBinary() {
+ readValue();
+ return this.binding.getBinary();
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#getFloat()
+ */
+ @Override
+ public float getFloat() {
+ readValue();
+ return this.binding.getFloat();
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#getDouble()
+ */
+ @Override
+ public double getDouble() {
+ readValue();
+ return this.binding.getDouble();
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#getCurrentRepetitionLevel()
+ */
+ @Override
+ public int getCurrentRepetitionLevel() {
+ return repetitionLevel;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#getDescriptor()
+ */
+ @Override
+ public ColumnDescriptor getDescriptor() {
+ return path;
+ }
+
+ /**
+ * Reads the value into the binding.
+ */
+ public void readValue() {
+ try {
+ if (!valueRead) {
+ binding.read();
+ valueRead = true;
+ }
+ } catch (RuntimeException e) {
+ if (CorruptDeltaByteArrays.requiresSequentialReads(writerVersion, currentEncoding) &&
+ e instanceof ArrayIndexOutOfBoundsException) {
+ // this is probably PARQUET-246, which may happen if reading data with
+ // MR because this can't be detected without reading all footers
+ throw new ParquetDecodingException("Read failure possibly due to " +
+ "PARQUET-246: try setting parquet.split.files to false",
+ new ParquetDecodingException(
+ format("Can't read value in column %s at value %d out of %d, " +
+ "%d out of %d in currentPage. repetition level: " +
+ "%d, definition level: %d",
+ path, readValues, totalValueCount,
+ readValues - (endOfPageValueCount - pageValueCount),
+ pageValueCount, repetitionLevel, definitionLevel),
+ e));
+ }
+ throw new ParquetDecodingException(
+ format("Can't read value in column %s at value %d out of %d, " +
+ "%d out of %d in currentPage. repetition level: " +
+ "%d, definition level: %d",
+ path, readValues, totalValueCount,
+ readValues - (endOfPageValueCount - pageValueCount),
+ pageValueCount, repetitionLevel, definitionLevel),
+ e);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#skip()
+ */
+ @Override
+ public void skip() {
+ if (!valueRead) {
+ binding.skip();
+ valueRead = true;
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#getCurrentDefinitionLevel()
+ */
+ @Override
+ public int getCurrentDefinitionLevel() {
+ return definitionLevel;
+ }
+
+ private void checkRead() {
+ int rl, dl;
+ int skipValues = 0;
+ for (;;) {
+ if (isPageFullyConsumed()) {
+ if (isFullyConsumed()) {
+ LOG.debug("end reached");
+ repetitionLevel = 0; // the next repetition level
+ return;
+ }
+ readPage();
+ skipValues = 0;
+ }
+ rl = repetitionLevelColumn.nextInt();
+ dl = definitionLevelColumn.nextInt();
+ ++readValues;
+ if (!skipRL(rl)) {
+ break;
+ }
+ if (dl == maxDefinitionLevel) {
+ ++skipValues;
+ }
+ }
+ binding.skip(skipValues);
+ repetitionLevel = rl;
+ definitionLevel = dl;
+ }
+
+ /*
+ * Returns if current levels / value shall be skipped based on the specified repetition level.
+ */
+ abstract boolean skipRL(int rl);
+
+ private void readPage() {
+ LOG.debug("loading page");
+ DataPage page = pageReader.readPage();
+ page.accept(new DataPage.Visitor<Void>() {
+ @Override
+ public Void visit(DataPageV1 dataPageV1) {
+ readPageV1(dataPageV1);
+ return null;
+ }
+ @Override
+ public Void visit(DataPageV2 dataPageV2) {
+ readPageV2(dataPageV2);
+ return null;
+ }
+ });
+ }
+
+ private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount) {
+ ValuesReader previousReader = this.dataColumn;
+
+ this.currentEncoding = dataEncoding;
+ this.pageValueCount = valueCount;
+ this.endOfPageValueCount = readValues + pageValueCount;
+
+ if (dataEncoding.usesDictionary()) {
+ if (dictionary == null) {
+ throw new ParquetDecodingException(
+ "could not read page in col " + path + " as the dictionary was missing for encoding " + dataEncoding);
+ }
+ this.dataColumn = dataEncoding.getDictionaryBasedValuesReader(path, VALUES, dictionary);
+ } else {
+ this.dataColumn = dataEncoding.getValuesReader(path, VALUES);
+ }
+
+ if (dataEncoding.usesDictionary() && converter.hasDictionarySupport()) {
+ bindToDictionary(dictionary);
+ } else {
+ bind(path.getType());
+ }
+
+ try {
+ dataColumn.initFromPage(pageValueCount, in);
+ } catch (IOException e) {
+ throw new ParquetDecodingException("could not read page in col " + path, e);
+ }
+
+ if (CorruptDeltaByteArrays.requiresSequentialReads(writerVersion, dataEncoding) &&
+ previousReader != null && previousReader instanceof RequiresPreviousReader) {
+ // previous reader can only be set if reading sequentially
+ ((RequiresPreviousReader) dataColumn).setPreviousReader(previousReader);
+ }
+ }
+
+ private void readPageV1(DataPageV1 page) {
+ ValuesReader rlReader = page.getRlEncoding().getValuesReader(path, REPETITION_LEVEL);
+ ValuesReader dlReader = page.getDlEncoding().getValuesReader(path, DEFINITION_LEVEL);
+ this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader);
+ this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader);
+ int valueCount = page.getValueCount();
+ try {
+ BytesInput bytes = page.getBytes();
+ LOG.debug("page size {} bytes and {} values", bytes.size(), valueCount);
+ LOG.debug("reading repetition levels at 0");
+ ByteBufferInputStream in = bytes.toInputStream();
+ rlReader.initFromPage(valueCount, in);
+ LOG.debug("reading definition levels at {}", in.position());
+ dlReader.initFromPage(valueCount, in);
+ LOG.debug("reading data at {}", in.position());
+ initDataReader(page.getValueEncoding(), in, valueCount);
+ } catch (IOException e) {
+ throw new ParquetDecodingException("could not read page " + page + " in col " + path, e);
+ }
+ newPageInitialized(page);
+ }
+
+ private void readPageV2(DataPageV2 page) {
+ this.repetitionLevelColumn = newRLEIterator(path.getMaxRepetitionLevel(), page.getRepetitionLevels());
+ this.definitionLevelColumn = newRLEIterator(path.getMaxDefinitionLevel(), page.getDefinitionLevels());
+ int valueCount = page.getValueCount();
+ LOG.debug("page data size {} bytes and {} values", page.getData().size(), valueCount);
+ try {
+ initDataReader(page.getDataEncoding(), page.getData().toInputStream(), valueCount);
+ } catch (IOException e) {
+ throw new ParquetDecodingException("could not read page " + page + " in col " + path, e);
+ }
+ newPageInitialized(page);
+ }
+
+ final int getPageValueCount() {
+ return pageValueCount;
+ }
+
+ abstract void newPageInitialized(DataPage page);
+
+ private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) {
+ try {
+ if (maxLevel == 0) {
+ return new NullIntIterator();
+ }
+ return new RLEIntIterator(
+ new RunLengthBitPackingHybridDecoder(
+ BytesUtils.getWidthFromMaxInt(maxLevel),
+ bytes.toInputStream()));
+ } catch (IOException e) {
+ throw new ParquetDecodingException("could not read levels in page for col " + path, e);
+ }
+ }
+
+ boolean isPageFullyConsumed() {
+ return readValues >= endOfPageValueCount;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#consume()
+ */
+ @Override
+ public void consume() {
+ checkRead();
+ valueRead = false;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see org.apache.parquet.column.ColumnReader#getTotalValueCount()
+ */
+ @Deprecated
+ @Override
+ public long getTotalValueCount() {
+ return totalValueCount;
+ }
+
+ static abstract class IntIterator {
+ abstract int nextInt();
+ }
+
+ static class ValuesReaderIntIterator extends IntIterator {
+ ValuesReader delegate;
+
+ public ValuesReaderIntIterator(ValuesReader delegate) {
+ super();
+ this.delegate = delegate;
+ }
+
+ @Override
+ int nextInt() {
+ return delegate.readInteger();
+ }
+ }
+
+ static class RLEIntIterator extends IntIterator {
+ RunLengthBitPackingHybridDecoder delegate;
+
+ public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ int nextInt() {
+ try {
+ return delegate.readInt();
+ } catch (IOException e) {
+ throw new ParquetDecodingException(e);
+ }
+ }
+ }
+
+ private static final class NullIntIterator extends IntIterator {
+ @Override
+ int nextInt() {
+ return 0;
+ }
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java
index 8c85b37f8..0413d621c 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java
@@ -1,4 +1,4 @@
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -18,675 +18,41 @@
*/
package org.apache.parquet.column.impl;
-import static java.lang.String.format;
-import static org.apache.parquet.Preconditions.checkNotNull;
-import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL;
-import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL;
-import static org.apache.parquet.column.ValuesType.VALUES;
-
-import java.io.IOException;
-
-import org.apache.parquet.CorruptDeltaByteArrays;
import org.apache.parquet.VersionParser.ParsedVersion;
-import org.apache.parquet.bytes.ByteBufferInputStream;
-import org.apache.parquet.bytes.BytesInput;
-import org.apache.parquet.bytes.BytesUtils;
import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.column.ColumnReader;
-import org.apache.parquet.column.Dictionary;
-import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.page.DataPage;
-import org.apache.parquet.column.page.DataPageV1;
-import org.apache.parquet.column.page.DataPageV2;
-import org.apache.parquet.column.page.DictionaryPage;
import org.apache.parquet.column.page.PageReader;
-import org.apache.parquet.column.values.RequiresPreviousReader;
-import org.apache.parquet.column.values.ValuesReader;
-import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder;
-import org.apache.parquet.io.ParquetDecodingException;
-import org.apache.parquet.io.api.Binary;
import org.apache.parquet.io.api.PrimitiveConverter;
-import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
-import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeNameConverter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
- * ColumnReader implementation
+ * ColumnReader implementation for the scenario when column indexes are not used (all values are read)
*/
-public class ColumnReaderImpl implements ColumnReader {
- private static final Logger LOG = LoggerFactory.getLogger(ColumnReaderImpl.class);
-
- /**
- * binds the lower level page decoder to the record converter materializing the records
- */
- private static abstract class Binding {
-
- /**
- * read one value from the underlying page
- */
- abstract void read();
-
- /**
- * skip one value from the underlying page
- */
- abstract void skip();
-
- /**
- * write current value to converter
- */
- abstract void writeValue();
-
- /**
- * @return current value
- */
- public int getDictionaryId() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * @return current value
- */
- public int getInteger() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * @return current value
- */
- public boolean getBoolean() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * @return current value
- */
- public long getLong() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * @return current value
- */
- public Binary getBinary() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * @return current value
- */
- public float getFloat() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * @return current value
- */
- public double getDouble() {
- throw new UnsupportedOperationException();
- }
- }
-
- private final ParsedVersion writerVersion;
- private final ColumnDescriptor path;
- private final long totalValueCount;
- private final PageReader pageReader;
- private final Dictionary dictionary;
-
- private IntIterator repetitionLevelColumn;
- private IntIterator definitionLevelColumn;
- protected ValuesReader dataColumn;
- private Encoding currentEncoding;
-
- private int repetitionLevel;
- private int definitionLevel;
- private int dictionaryId;
-
- private long endOfPageValueCount;
- private long readValues = 0;
- private int pageValueCount = 0;
-
- private final PrimitiveConverter converter;
- private Binding binding;
-
- // this is needed because we will attempt to read the value twice when filtering
- // TODO: rework that
- private boolean valueRead;
-
- private void bindToDictionary(final Dictionary dictionary) {
- binding =
- new Binding() {
- void read() {
- dictionaryId = dataColumn.readValueDictionaryId();
- }
- public void skip() {
- dataColumn.skip();
- }
- public int getDictionaryId() {
- return dictionaryId;
- }
- void writeValue() {
- converter.addValueFromDictionary(dictionaryId);
- }
- public int getInteger() {
- return dictionary.decodeToInt(dictionaryId);
- }
- public boolean getBoolean() {
- return dictionary.decodeToBoolean(dictionaryId);
- }
- public long getLong() {
- return dictionary.decodeToLong(dictionaryId);
- }
- public Binary getBinary() {
- return dictionary.decodeToBinary(dictionaryId);
- }
- public float getFloat() {
- return dictionary.decodeToFloat(dictionaryId);
- }
- public double getDouble() {
- return dictionary.decodeToDouble(dictionaryId);
- }
- };
- }
-
- private void bind(PrimitiveTypeName type) {
- binding = type.convert(new PrimitiveTypeNameConverter<Binding, RuntimeException>() {
- @Override
- public Binding convertFLOAT(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
- return new Binding() {
- float current;
- void read() {
- current = dataColumn.readFloat();
- }
- public void skip() {
- current = 0;
- dataColumn.skip();
- }
- public float getFloat() {
- return current;
- }
- void writeValue() {
- converter.addFloat(current);
- }
- };
- }
- @Override
- public Binding convertDOUBLE(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
- return new Binding() {
- double current;
- void read() {
- current = dataColumn.readDouble();
- }
- public void skip() {
- current = 0;
- dataColumn.skip();
- }
- public double getDouble() {
- return current;
- }
- void writeValue() {
- converter.addDouble(current);
- }
- };
- }
- @Override
- public Binding convertINT32(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
- return new Binding() {
- int current;
- void read() {
- current = dataColumn.readInteger();
- }
- public void skip() {
- current = 0;
- dataColumn.skip();
- }
- @Override
- public int getInteger() {
- return current;
- }
- void writeValue() {
- converter.addInt(current);
- }
- };
- }
- @Override
- public Binding convertINT64(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
- return new Binding() {
- long current;
- void read() {
- current = dataColumn.readLong();
- }
- public void skip() {
- current = 0;
- dataColumn.skip();
- }
- @Override
- public long getLong() {
- return current;
- }
- void writeValue() {
- converter.addLong(current);
- }
- };
- }
- @Override
- public Binding convertINT96(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
- return this.convertBINARY(primitiveTypeName);
- }
- @Override
- public Binding convertFIXED_LEN_BYTE_ARRAY(
- PrimitiveTypeName primitiveTypeName) throws RuntimeException {
- return this.convertBINARY(primitiveTypeName);
- }
- @Override
- public Binding convertBOOLEAN(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
- return new Binding() {
- boolean current;
- void read() {
- current = dataColumn.readBoolean();
- }
- public void skip() {
- current = false;
- dataColumn.skip();
- }
- @Override
- public boolean getBoolean() {
- return current;
- }
- void writeValue() {
- converter.addBoolean(current);
- }
- };
- }
- @Override
- public Binding convertBINARY(PrimitiveTypeName primitiveTypeName) throws RuntimeException {
- return new Binding() {
- Binary current;
- void read() {
- current = dataColumn.readBytes();
- }
- public void skip() {
- current = null;
- dataColumn.skip();
- }
- @Override
- public Binary getBinary() {
- return current;
- }
- void writeValue() {
- converter.addBinary(current);
- }
- };
- }
- });
- }
+public class ColumnReaderImpl extends ColumnReaderBase {
/**
* creates a reader for triplets
- * @param path the descriptor for the corresponding column
- * @param pageReader the underlying store to read from
- * @param converter a converter that materializes the values in this column in the current record
- * @param writerVersion writer version string from the Parquet file being read
- */
- public ColumnReaderImpl(ColumnDescriptor path, PageReader pageReader, PrimitiveConverter converter, ParsedVersion writerVersion) {
- this.path = checkNotNull(path, "path");
- this.pageReader = checkNotNull(pageReader, "pageReader");
- this.converter = checkNotNull(converter, "converter");
- this.writerVersion = writerVersion;
- DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
- if (dictionaryPage != null) {
- try {
- this.dictionary = dictionaryPage.getEncoding().initDictionary(path, dictionaryPage);
- if (converter.hasDictionarySupport()) {
- converter.setDictionary(dictionary);
- }
- } catch (IOException e) {
- throw new ParquetDecodingException("could not decode the dictionary for " + path, e);
- }
- } else {
- this.dictionary = null;
- }
- this.totalValueCount = pageReader.getTotalValueCount();
- if (totalValueCount <= 0) {
- throw new ParquetDecodingException("totalValueCount '" + totalValueCount + "' <= 0");
- }
+ *
+ * @param path
+ * the descriptor for the corresponding column
+ * @param pageReader
+ * the underlying store to read from
+ * @param converter
+ * a converter that materializes the values in this column in the current record
+ * @param writerVersion
+ * writer version string from the Parquet file being read
+ */
+ public ColumnReaderImpl(ColumnDescriptor path, PageReader pageReader, PrimitiveConverter converter,
+ ParsedVersion writerVersion) {
+ super(path, pageReader, converter, writerVersion);
consume();
}
- private boolean isFullyConsumed() {
- return readValues >= totalValueCount;
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#writeCurrentValueToConverter()
- */
- @Override
- public void writeCurrentValueToConverter() {
- readValue();
- this.binding.writeValue();
- }
-
- @Override
- public int getCurrentValueDictionaryID() {
- readValue();
- return binding.getDictionaryId();
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#getInteger()
- */
- @Override
- public int getInteger() {
- readValue();
- return this.binding.getInteger();
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#getBoolean()
- */
- @Override
- public boolean getBoolean() {
- readValue();
- return this.binding.getBoolean();
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#getLong()
- */
- @Override
- public long getLong() {
- readValue();
- return this.binding.getLong();
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#getBinary()
- */
- @Override
- public Binary getBinary() {
- readValue();
- return this.binding.getBinary();
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#getFloat()
- */
- @Override
- public float getFloat() {
- readValue();
- return this.binding.getFloat();
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#getDouble()
- */
- @Override
- public double getDouble() {
- readValue();
- return this.binding.getDouble();
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#getCurrentRepetitionLevel()
- */
- @Override
- public int getCurrentRepetitionLevel() {
- return repetitionLevel;
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#getDescriptor()
- */
- @Override
- public ColumnDescriptor getDescriptor() {
- return path;
- }
-
- /**
- * Reads the value into the binding.
- */
- public void readValue() {
- try {
- if (!valueRead) {
- binding.read();
- valueRead = true;
- }
- } catch (RuntimeException e) {
- if (CorruptDeltaByteArrays.requiresSequentialReads(writerVersion, currentEncoding) &&
- e instanceof ArrayIndexOutOfBoundsException) {
- // this is probably PARQUET-246, which may happen if reading data with
- // MR because this can't be detected without reading all footers
- throw new ParquetDecodingException("Read failure possibly due to " +
- "PARQUET-246: try setting parquet.split.files to false",
- new ParquetDecodingException(
- format("Can't read value in column %s at value %d out of %d, " +
- "%d out of %d in currentPage. repetition level: " +
- "%d, definition level: %d",
- path, readValues, totalValueCount,
- readValues - (endOfPageValueCount - pageValueCount),
- pageValueCount, repetitionLevel, definitionLevel),
- e));
- }
- throw new ParquetDecodingException(
- format("Can't read value in column %s at value %d out of %d, " +
- "%d out of %d in currentPage. repetition level: " +
- "%d, definition level: %d",
- path, readValues, totalValueCount,
- readValues - (endOfPageValueCount - pageValueCount),
- pageValueCount, repetitionLevel, definitionLevel),
- e);
- }
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#skip()
- */
- @Override
- public void skip() {
- if (!valueRead) {
- binding.skip();
- valueRead = true;
- }
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#getCurrentDefinitionLevel()
- */
- @Override
- public int getCurrentDefinitionLevel() {
- return definitionLevel;
- }
-
- // TODO: change the logic around read() to not tie together reading from the 3 columns
- private void readRepetitionAndDefinitionLevels() {
- repetitionLevel = repetitionLevelColumn.nextInt();
- definitionLevel = definitionLevelColumn.nextInt();
- ++readValues;
- }
-
- private void checkRead() {
- if (isPageFullyConsumed()) {
- if (isFullyConsumed()) {
- LOG.debug("end reached");
- repetitionLevel = 0; // the next repetition level
- return;
- }
- readPage();
- }
- readRepetitionAndDefinitionLevels();
- }
-
- private void readPage() {
- LOG.debug("loading page");
- DataPage page = pageReader.readPage();
- page.accept(new DataPage.Visitor<Void>() {
- @Override
- public Void visit(DataPageV1 dataPageV1) {
- readPageV1(dataPageV1);
- return null;
- }
- @Override
- public Void visit(DataPageV2 dataPageV2) {
- readPageV2(dataPageV2);
- return null;
- }
- });
- }
-
- private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount) {
- ValuesReader previousReader = this.dataColumn;
-
- this.currentEncoding = dataEncoding;
- this.pageValueCount = valueCount;
- this.endOfPageValueCount = readValues + pageValueCount;
-
- if (dataEncoding.usesDictionary()) {
- if (dictionary == null) {
- throw new ParquetDecodingException(
- "could not read page in col " + path + " as the dictionary was missing for encoding " + dataEncoding);
- }
- this.dataColumn = dataEncoding.getDictionaryBasedValuesReader(path, VALUES, dictionary);
- } else {
- this.dataColumn = dataEncoding.getValuesReader(path, VALUES);
- }
-
- if (dataEncoding.usesDictionary() && converter.hasDictionarySupport()) {
- bindToDictionary(dictionary);
- } else {
- bind(path.getType());
- }
-
- try {
- dataColumn.initFromPage(pageValueCount, in);
- } catch (IOException e) {
- throw new ParquetDecodingException("could not read page in col " + path, e);
- }
-
- if (CorruptDeltaByteArrays.requiresSequentialReads(writerVersion, dataEncoding) &&
- previousReader != null && previousReader instanceof RequiresPreviousReader) {
- // previous reader can only be set if reading sequentially
- ((RequiresPreviousReader) dataColumn).setPreviousReader(previousReader);
- }
- }
-
- private void readPageV1(DataPageV1 page) {
- ValuesReader rlReader = page.getRlEncoding().getValuesReader(path, REPETITION_LEVEL);
- ValuesReader dlReader = page.getDlEncoding().getValuesReader(path, DEFINITION_LEVEL);
- this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader);
- this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader);
- try {
- BytesInput bytes = page.getBytes();
- LOG.debug("page size {} bytes and {} records", bytes.size(), pageValueCount);
- LOG.debug("reading repetition levels at 0");
- ByteBufferInputStream in = bytes.toInputStream();
- rlReader.initFromPage(pageValueCount, in);
- LOG.debug("reading definition levels at {}", in.position());
- dlReader.initFromPage(pageValueCount, in);
- LOG.debug("reading data at {}", in.position());
- initDataReader(page.getValueEncoding(), in, page.getValueCount());
- } catch (IOException e) {
- throw new ParquetDecodingException("could not read page " + page + " in col " + path, e);
- }
- }
-
- private void readPageV2(DataPageV2 page) {
- this.repetitionLevelColumn = newRLEIterator(path.getMaxRepetitionLevel(), page.getRepetitionLevels());
- this.definitionLevelColumn = newRLEIterator(path.getMaxDefinitionLevel(), page.getDefinitionLevels());
- LOG.debug("page data size {} bytes and {} records", page.getData().size(), pageValueCount);
- try {
- initDataReader(page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount());
- } catch (IOException e) {
- throw new ParquetDecodingException("could not read page " + page + " in col " + path, e);
- }
- }
-
- private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) {
- try {
- if (maxLevel == 0) {
- return new NullIntIterator();
- }
- return new RLEIntIterator(
- new RunLengthBitPackingHybridDecoder(
- BytesUtils.getWidthFromMaxInt(maxLevel),
- bytes.toInputStream()));
- } catch (IOException e) {
- throw new ParquetDecodingException("could not read levels in page for col " + path, e);
- }
- }
-
- private boolean isPageFullyConsumed() {
- return readValues >= endOfPageValueCount;
- }
-
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#consume()
- */
@Override
- public void consume() {
- checkRead();
- valueRead = false;
+ boolean skipRL(int rl) {
+ return false;
}
- /**
- * {@inheritDoc}
- * @see org.apache.parquet.column.ColumnReader#getTotalValueCount()
- */
@Override
- public long getTotalValueCount() {
- return totalValueCount;
- }
-
- static abstract class IntIterator {
- abstract int nextInt();
- }
-
- static class ValuesReaderIntIterator extends IntIterator {
- ValuesReader delegate;
-
- public ValuesReaderIntIterator(ValuesReader delegate) {
- super();
- this.delegate = delegate;
- }
-
- @Override
- int nextInt() {
- return delegate.readInteger();
- }
- }
-
- static class RLEIntIterator extends IntIterator {
- RunLengthBitPackingHybridDecoder delegate;
-
- public RLEIntIterator(RunLengthBitPackingHybridDecoder delegate) {
- this.delegate = delegate;
- }
-
- @Override
- int nextInt() {
- try {
- return delegate.readInt();
- } catch (IOException e) {
- throw new ParquetDecodingException(e);
- }
- }
- }
-
- private static final class NullIntIterator extends IntIterator {
- @Override
- int nextInt() {
- return 0;
- }
+ void newPageInitialized(DataPage page) {
}
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java
new file mode 100644
index 000000000..5cd7d876e
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreBase.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.impl;
+
+import static java.lang.Math.max;
+import static java.lang.Math.min;
+import static java.util.Collections.unmodifiableMap;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.ColumnWriteStore;
+import org.apache.parquet.column.ColumnWriter;
+import org.apache.parquet.column.ParquetProperties;
+import org.apache.parquet.column.page.PageWriteStore;
+import org.apache.parquet.column.page.PageWriter;
+import org.apache.parquet.schema.MessageType;
+
+/**
+ * Base implementation for {@link ColumnWriteStore} to be extended to specialize for V1 and V2 pages.
+ */
+abstract class ColumnWriteStoreBase implements ColumnWriteStore {
+
+ // Used to support the deprecated workflow of ColumnWriteStoreV1 (lazy init of ColumnWriters)
+ private interface ColumnWriterProvider {
+ ColumnWriter getColumnWriter(ColumnDescriptor path);
+ }
+
+ private final ColumnWriterProvider columnWriterProvider;
+
+ // will flush even if size bellow the threshold by this much to facilitate page alignment
+ private static final float THRESHOLD_TOLERANCE_RATIO = 0.1f; // 10 %
+
+ private final Map<ColumnDescriptor, ColumnWriterBase> columns;
+ private final ParquetProperties props;
+ private final long thresholdTolerance;
+ private long rowCount;
+ private long rowCountForNextSizeCheck;
+
+ // To be used by the deprecated constructor of ColumnWriteStoreV1
+ @Deprecated
+ ColumnWriteStoreBase(
+ final PageWriteStore pageWriteStore,
+ final ParquetProperties props) {
+ this.props = props;
+ this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO);
+
+ this.columns = new TreeMap<>();
+
+ this.rowCountForNextSizeCheck = props.getMinRowCountForPageSizeCheck();
+
+ columnWriterProvider = new ColumnWriterProvider() {
+ @Override
+ public ColumnWriter getColumnWriter(ColumnDescriptor path) {
+ ColumnWriterBase column = columns.get(path);
+ if (column == null) {
+ column = createColumnWriter(path, pageWriteStore.getPageWriter(path), props);
+ columns.put(path, column);
+ }
+ return column;
+ }
+ };
+ }
+
+ ColumnWriteStoreBase(
+ MessageType schema,
+ PageWriteStore pageWriteStore,
+ ParquetProperties props) {
+ this.props = props;
+ this.thresholdTolerance = (long) (props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO);
+ Map<ColumnDescriptor, ColumnWriterBase> mcolumns = new TreeMap<>();
+ for (ColumnDescriptor path : schema.getColumns()) {
+ PageWriter pageWriter = pageWriteStore.getPageWriter(path);
+ mcolumns.put(path, createColumnWriter(path, pageWriter, props));
+ }
+ this.columns = unmodifiableMap(mcolumns);
+
+ this.rowCountForNextSizeCheck = props.getMinRowCountForPageSizeCheck();
+
+ columnWriterProvider = new ColumnWriterProvider() {
+ @Override
+ public ColumnWriter getColumnWriter(ColumnDescriptor path) {
+ return columns.get(path);
+ }
+ };
+ }
+
+ abstract ColumnWriterBase createColumnWriter(ColumnDescriptor path, PageWriter pageWriter, ParquetProperties props);
+
+ public ColumnWriter getColumnWriter(ColumnDescriptor path) {
+ return columnWriterProvider.getColumnWriter(path);
+ }
+
+ public Set<ColumnDescriptor> getColumnDescriptors() {
+ return columns.keySet();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (Entry<ColumnDescriptor, ColumnWriterBase> entry : columns.entrySet()) {
+ sb.append(Arrays.toString(entry.getKey().getPath())).append(": ");
+ sb.append(entry.getValue().getTotalBufferedSize()).append(" bytes");
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public long getAllocatedSize() {
+ long total = 0;
+ for (ColumnWriterBase memColumn : columns.values()) {
+ total += memColumn.allocatedSize();
+ }
+ return total;
+ }
+
+ @Override
+ public long getBufferedSize() {
+ long total = 0;
+ for (ColumnWriterBase memColumn : columns.values()) {
+ total += memColumn.getTotalBufferedSize();
+ }
+ return total;
+ }
+
+ @Override
+ public void flush() {
+ for (ColumnWriterBase memColumn : columns.values()) {
+ long rows = rowCount - memColumn.getRowsWrittenSoFar();
+ if (rows > 0) {
+ memColumn.writePage();
+ }
+ memColumn.finalizeColumnChunk();
+ }
+ }
+
+ public String memUsageString() {
+ StringBuilder b = new StringBuilder("Store {\n");
+ for (ColumnWriterBase memColumn : columns.values()) {
+ b.append(memColumn.memUsageString(" "));
+ }
+ b.append("}\n");
+ return b.toString();
+ }
+
+ public long maxColMemSize() {
+ long max = 0;
+ for (ColumnWriterBase memColumn : columns.values()) {
+ max = Math.max(max, memColumn.getBufferedSizeInMemory());
+ }
+ return max;
+ }
+
+ @Override
+ public void close() {
+ flush(); // calling flush() here to keep it consistent with the behavior before merging with master
+ for (ColumnWriterBase memColumn : columns.values()) {
+ memColumn.close();
+ }
+ }
+
+ @Override
+ public void endRecord() {
+ ++rowCount;
+ if (rowCount >= rowCountForNextSizeCheck) {
+ sizeCheck();
+ }
+ }
+
+ private void sizeCheck() {
+ long minRecordToWait = Long.MAX_VALUE;
+ for (ColumnWriterBase writer : columns.values()) {
+ long usedMem = writer.getCurrentPageBufferedSize();
+ long rows = rowCount - writer.getRowsWrittenSoFar();
+ long remainingMem = props.getPageSizeThreshold() - usedMem;
+ if (remainingMem <= thresholdTolerance) {
+ writer.writePage();
+ remainingMem = props.getPageSizeThreshold();
+ }
+ long rowsToFillPage =
+ usedMem == 0 ?
+ props.getMaxRowCountForPageSizeCheck()
+ : (long) ((float) rows) / usedMem * remainingMem;
+ if (rowsToFillPage < minRecordToWait) {
+ minRecordToWait = rowsToFillPage;
+ }
+ }
+ if (minRecordToWait == Long.MAX_VALUE) {
+ minRecordToWait = props.getMinRowCountForPageSizeCheck();
+ }
+
+ if (props.estimateNextSizeCheck()) {
+ // will check again halfway if between min and max
+ rowCountForNextSizeCheck = rowCount +
+ min(
+ max(minRecordToWait / 2, props.getMinRowCountForPageSizeCheck()),
+ props.getMaxRowCountForPageSizeCheck());
+ } else {
+ rowCountForNextSizeCheck = rowCount + props.getMinRowCountForPageSizeCheck();
+ }
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV1.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV1.java
index 93a497fad..7258423fb 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV1.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV1.java
@@ -18,121 +18,26 @@
*/
package org.apache.parquet.column.impl;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-
-import org.apache.parquet.bytes.ByteBufferAllocator;
import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.column.ColumnWriteStore;
-import org.apache.parquet.column.ColumnWriter;
import org.apache.parquet.column.ParquetProperties;
-import org.apache.parquet.column.ParquetProperties.WriterVersion;
import org.apache.parquet.column.page.PageWriteStore;
import org.apache.parquet.column.page.PageWriter;
+import org.apache.parquet.schema.MessageType;
-public class ColumnWriteStoreV1 implements ColumnWriteStore {
-
- private final Map<ColumnDescriptor, ColumnWriterV1> columns = new TreeMap<ColumnDescriptor, ColumnWriterV1>();
- private final PageWriteStore pageWriteStore;
- private final ParquetProperties props;
-
- public ColumnWriteStoreV1(PageWriteStore pageWriteStore,
- ParquetProperties props) {
- this.pageWriteStore = pageWriteStore;
- this.props = props;
- }
-
- public ColumnWriter getColumnWriter(ColumnDescriptor path) {
- ColumnWriterV1 column = columns.get(path);
- if (column == null) {
- column = newMemColumn(path);
- columns.put(path, column);
- }
- return column;
- }
-
- public Set<ColumnDescriptor> getColumnDescriptors() {
- return columns.keySet();
- }
-
- private ColumnWriterV1 newMemColumn(ColumnDescriptor path) {
- PageWriter pageWriter = pageWriteStore.getPageWriter(path);
- return new ColumnWriterV1(path, pageWriter, props);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- for (Entry<ColumnDescriptor, ColumnWriterV1> entry : columns.entrySet()) {
- sb.append(Arrays.toString(entry.getKey().getPath())).append(": ");
- sb.append(entry.getValue().getBufferedSizeInMemory()).append(" bytes");
- sb.append("\n");
- }
- return sb.toString();
- }
-
- @Override
- public long getAllocatedSize() {
- Collection<ColumnWriterV1> values = columns.values();
- long total = 0;
- for (ColumnWriterV1 memColumn : values) {
- total += memColumn.allocatedSize();
- }
- return total;
- }
-
- @Override
- public long getBufferedSize() {
- Collection<ColumnWriterV1> values = columns.values();
- long total = 0;
- for (ColumnWriterV1 memColumn : values) {
- total += memColumn.getBufferedSizeInMemory();
- }
- return total;
- }
-
- @Override
- public String memUsageString() {
- StringBuilder b = new StringBuilder("Store {\n");
- Collection<ColumnWriterV1> values = columns.values();
- for (ColumnWriterV1 memColumn : values) {
- b.append(memColumn.memUsageString(" "));
- }
- b.append("}\n");
- return b.toString();
- }
+public class ColumnWriteStoreV1 extends ColumnWriteStoreBase {
- public long maxColMemSize() {
- Collection<ColumnWriterV1> values = columns.values();
- long max = 0;
- for (ColumnWriterV1 memColumn : values) {
- max = Math.max(max, memColumn.getBufferedSizeInMemory());
- }
- return max;
+ public ColumnWriteStoreV1(MessageType schema, PageWriteStore pageWriteStore, ParquetProperties props) {
+ super(schema, pageWriteStore, props);
}
- @Override
- public void flush() {
- Collection<ColumnWriterV1> values = columns.values();
- for (ColumnWriterV1 memColumn : values) {
- memColumn.flush();
- }
+ @Deprecated
+ public ColumnWriteStoreV1(final PageWriteStore pageWriteStore,
+ final ParquetProperties props) {
+ super(pageWriteStore, props);
}
@Override
- public void endRecord() {
- // V1 does not take record boundaries into account
- }
-
- public void close() {
- Collection<ColumnWriterV1> values = columns.values();
- for (ColumnWriterV1 memColumn : values) {
- memColumn.close();
- }
+ ColumnWriterBase createColumnWriter(ColumnDescriptor path, PageWriter pageWriter, ParquetProperties props) {
+ return new ColumnWriterV1(path, pageWriter, props);
}
-
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV2.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV2.java
index 7574cedf7..bf1090d0b 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV2.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriteStoreV2.java
@@ -18,158 +18,20 @@
*/
package org.apache.parquet.column.impl;
-import static java.lang.Math.max;
-import static java.lang.Math.min;
-import static java.util.Collections.unmodifiableMap;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-
import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.column.ColumnWriteStore;
-import org.apache.parquet.column.ColumnWriter;
import org.apache.parquet.column.ParquetProperties;
import org.apache.parquet.column.page.PageWriteStore;
import org.apache.parquet.column.page.PageWriter;
import org.apache.parquet.schema.MessageType;
-public class ColumnWriteStoreV2 implements ColumnWriteStore {
-
- // will flush even if size bellow the threshold by this much to facilitate page alignment
- private static final float THRESHOLD_TOLERANCE_RATIO = 0.1f; // 10 %
-
- private final Map<ColumnDescriptor, ColumnWriterV2> columns;
- private final Collection<ColumnWriterV2> writers;
- private final ParquetProperties props;
- private final long thresholdTolerance;
- private long rowCount;
- private long rowCountForNextSizeCheck;
-
- public ColumnWriteStoreV2(
- MessageType schema,
- PageWriteStore pageWriteStore,
- ParquetProperties props) {
- this.props = props;
- this.thresholdTolerance = (long)(props.getPageSizeThreshold() * THRESHOLD_TOLERANCE_RATIO);
- Map<ColumnDescriptor, ColumnWriterV2> mcolumns = new TreeMap<ColumnDescriptor, ColumnWriterV2>();
- for (ColumnDescriptor path : schema.getColumns()) {
- PageWriter pageWriter = pageWriteStore.getPageWriter(path);
- mcolumns.put(path, new ColumnWriterV2(path, pageWriter, props));
- }
- this.columns = unmodifiableMap(mcolumns);
- this.writers = this.columns.values();
-
- this.rowCountForNextSizeCheck = props.getMinRowCountForPageSizeCheck();
- }
-
- public ColumnWriter getColumnWriter(ColumnDescriptor path) {
- return columns.get(path);
- }
-
- public Set<ColumnDescriptor> getColumnDescriptors() {
- return columns.keySet();
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- for (Entry<ColumnDescriptor, ColumnWriterV2> entry : columns.entrySet()) {
- sb.append(Arrays.toString(entry.getKey().getPath())).append(": ");
- sb.append(entry.getValue().getTotalBufferedSize()).append(" bytes");
- sb.append("\n");
- }
- return sb.toString();
- }
-
- @Override
- public long getAllocatedSize() {
- long total = 0;
- for (ColumnWriterV2 memColumn : columns.values()) {
- total += memColumn.allocatedSize();
- }
- return total;
- }
-
- @Override
- public long getBufferedSize() {
- long total = 0;
- for (ColumnWriterV2 memColumn : columns.values()) {
- total += memColumn.getTotalBufferedSize();
- }
- return total;
- }
-
- @Override
- public void flush() {
- for (ColumnWriterV2 memColumn : columns.values()) {
- long rows = rowCount - memColumn.getRowsWrittenSoFar();
- if (rows > 0) {
- memColumn.writePage(rowCount);
- }
- memColumn.finalizeColumnChunk();
- }
- }
+public class ColumnWriteStoreV2 extends ColumnWriteStoreBase {
- public String memUsageString() {
- StringBuilder b = new StringBuilder("Store {\n");
- for (ColumnWriterV2 memColumn : columns.values()) {
- b.append(memColumn.memUsageString(" "));
- }
- b.append("}\n");
- return b.toString();
+ public ColumnWriteStoreV2(MessageType schema, PageWriteStore pageWriteStore, ParquetProperties props) {
+ super(schema, pageWriteStore, props);
}
@Override
- public void close() {
- flush(); // calling flush() here to keep it consistent with the behavior before merging with master
- for (ColumnWriterV2 memColumn : columns.values()) {
- memColumn.close();
- }
+ ColumnWriterBase createColumnWriter(ColumnDescriptor path, PageWriter pageWriter, ParquetProperties props) {
+ return new ColumnWriterV2(path, pageWriter, props);
}
-
- @Override
- public void endRecord() {
- ++ rowCount;
- if (rowCount >= rowCountForNextSizeCheck) {
- sizeCheck();
- }
- }
-
- private void sizeCheck() {
- long minRecordToWait = Long.MAX_VALUE;
- for (ColumnWriterV2 writer : writers) {
- long usedMem = writer.getCurrentPageBufferedSize();
- long rows = rowCount - writer.getRowsWrittenSoFar();
- long remainingMem = props.getPageSizeThreshold() - usedMem;
- if (remainingMem <= thresholdTolerance) {
- writer.writePage(rowCount);
- remainingMem = props.getPageSizeThreshold();
- }
- long rowsToFillPage =
- usedMem == 0 ?
- props.getMaxRowCountForPageSizeCheck()
- : (long)((float)rows) / usedMem * remainingMem;
- if (rowsToFillPage < minRecordToWait) {
- minRecordToWait = rowsToFillPage;
- }
- }
- if (minRecordToWait == Long.MAX_VALUE) {
- minRecordToWait = props.getMinRowCountForPageSizeCheck();
- }
-
- if(props.estimateNextSizeCheck()) {
- // will check again halfway if between min and max
- rowCountForNextSizeCheck = rowCount +
- min(
- max(minRecordToWait / 2, props.getMinRowCountForPageSizeCheck()),
- props.getMaxRowCountForPageSizeCheck());
- } else {
- rowCountForNextSizeCheck = rowCount + props.getMinRowCountForPageSizeCheck();
- }
- }
-
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java
new file mode 100644
index 000000000..3788c82e4
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterBase.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.impl;
+
+import java.io.IOException;
+
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.ColumnWriter;
+import org.apache.parquet.column.ParquetProperties;
+import org.apache.parquet.column.page.DictionaryPage;
+import org.apache.parquet.column.page.PageWriter;
+import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.column.values.ValuesWriter;
+import org.apache.parquet.io.ParquetEncodingException;
+import org.apache.parquet.io.api.Binary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base implementation for {@link ColumnWriter} to be extended to specialize for V1 and V2 pages.
+ */
+abstract class ColumnWriterBase implements ColumnWriter {
+ private static final Logger LOG = LoggerFactory.getLogger(ColumnWriterBase.class);
+
+ // By default: Debugging disabled this way (using the "if (DEBUG)" IN the methods) to allow
+ // the java compiler (not the JIT) to remove the unused statements during build time.
+ private static final boolean DEBUG = false;
+
+ final ColumnDescriptor path;
+ final PageWriter pageWriter;
+ private ValuesWriter repetitionLevelColumn;
+ private ValuesWriter definitionLevelColumn;
+ private ValuesWriter dataColumn;
+ private int valueCount;
+
+ private Statistics<?> statistics;
+ private long rowsWrittenSoFar = 0;
+ private int pageRowCount;
+
+ ColumnWriterBase(
+ ColumnDescriptor path,
+ PageWriter pageWriter,
+ ParquetProperties props) {
+ this.path = path;
+ this.pageWriter = pageWriter;
+ resetStatistics();
+
+ this.repetitionLevelColumn = createRLWriter(props, path);
+ this.definitionLevelColumn = createDLWriter(props, path);
+ this.dataColumn = props.newValuesWriter(path);
+ }
+
+ abstract ValuesWriter createRLWriter(ParquetProperties props, ColumnDescriptor path);
+
+ abstract ValuesWriter createDLWriter(ParquetProperties props, ColumnDescriptor path);
+
+ private void log(Object value, int r, int d) {
+ LOG.debug("{} {} r:{} d:{}", path, value, r, d);
+ }
+
+ private void resetStatistics() {
+ this.statistics = Statistics.createStats(path.getPrimitiveType());
+ }
+
+ private void definitionLevel(int definitionLevel) {
+ definitionLevelColumn.writeInteger(definitionLevel);
+ }
+
+ private void repetitionLevel(int repetitionLevel) {
+ repetitionLevelColumn.writeInteger(repetitionLevel);
+ assert pageRowCount == 0 ? repetitionLevel == 0 : true : "Every page shall start on record boundaries";
+ if (repetitionLevel == 0) {
+ ++pageRowCount;
+ }
+ }
+
+ /**
+ * Writes the current null value
+ *
+ * @param repetitionLevel
+ * @param definitionLevel
+ */
+ @Override
+ public void writeNull(int repetitionLevel, int definitionLevel) {
+ if (DEBUG)
+ log(null, repetitionLevel, definitionLevel);
+ repetitionLevel(repetitionLevel);
+ definitionLevel(definitionLevel);
+ statistics.incrementNumNulls();
+ ++valueCount;
+ }
+
+ @Override
+ public void close() {
+ // Close the Values writers.
+ repetitionLevelColumn.close();
+ definitionLevelColumn.close();
+ dataColumn.close();
+ }
+
+ @Override
+ public long getBufferedSizeInMemory() {
+ return repetitionLevelColumn.getBufferedSize()
+ + definitionLevelColumn.getBufferedSize()
+ + dataColumn.getBufferedSize()
+ + pageWriter.getMemSize();
+ }
+
+ /**
+ * Writes the current value
+ *
+ * @param value
+ * @param repetitionLevel
+ * @param definitionLevel
+ */
+ @Override
+ public void write(double value, int repetitionLevel, int definitionLevel) {
+ if (DEBUG)
+ log(value, repetitionLevel, definitionLevel);
+ repetitionLevel(repetitionLevel);
+ definitionLevel(definitionLevel);
+ dataColumn.writeDouble(value);
+ statistics.updateStats(value);
+ ++valueCount;
+ }
+
+ /**
+ * Writes the current value
+ *
+ * @param value
+ * @param repetitionLevel
+ * @param definitionLevel
+ */
+ @Override
+ public void write(float value, int repetitionLevel, int definitionLevel) {
+ if (DEBUG)
+ log(value, repetitionLevel, definitionLevel);
+ repetitionLevel(repetitionLevel);
+ definitionLevel(definitionLevel);
+ dataColumn.writeFloat(value);
+ statistics.updateStats(value);
+ ++valueCount;
+ }
+
+ /**
+ * Writes the current value
+ *
+ * @param value
+ * @param repetitionLevel
+ * @param definitionLevel
+ */
+ @Override
+ public void write(Binary value, int repetitionLevel, int definitionLevel) {
+ if (DEBUG)
+ log(value, repetitionLevel, definitionLevel);
+ repetitionLevel(repetitionLevel);
+ definitionLevel(definitionLevel);
+ dataColumn.writeBytes(value);
+ statistics.updateStats(value);
+ ++valueCount;
+ }
+
+ /**
+ * Writes the current value
+ *
+ * @param value
+ * @param repetitionLevel
+ * @param definitionLevel
+ */
+ @Override
+ public void write(boolean value, int repetitionLevel, int definitionLevel) {
+ if (DEBUG)
+ log(value, repetitionLevel, definitionLevel);
+ repetitionLevel(repetitionLevel);
+ definitionLevel(definitionLevel);
+ dataColumn.writeBoolean(value);
+ statistics.updateStats(value);
+ ++valueCount;
+ }
+
+ /**
+ * Writes the current value
+ *
+ * @param value
+ * @param repetitionLevel
+ * @param definitionLevel
+ */
+ @Override
+ public void write(int value, int repetitionLevel, int definitionLevel) {
+ if (DEBUG)
+ log(value, repetitionLevel, definitionLevel);
+ repetitionLevel(repetitionLevel);
+ definitionLevel(definitionLevel);
+ dataColumn.writeInteger(value);
+ statistics.updateStats(value);
+ ++valueCount;
+ }
+
+ /**
+ * Writes the current value
+ *
+ * @param value
+ * @param repetitionLevel
+ * @param definitionLevel
+ */
+ @Override
+ public void write(long value, int repetitionLevel, int definitionLevel) {
+ if (DEBUG)
+ log(value, repetitionLevel, definitionLevel);
+ repetitionLevel(repetitionLevel);
+ definitionLevel(definitionLevel);
+ dataColumn.writeLong(value);
+ statistics.updateStats(value);
+ ++valueCount;
+ }
+
+ /**
+ * Finalizes the Column chunk. Possibly adding extra pages if needed (dictionary, ...)
+ * Is called right after writePage
+ */
+ void finalizeColumnChunk() {
+ final DictionaryPage dictionaryPage = dataColumn.toDictPageAndClose();
+ if (dictionaryPage != null) {
+ if (DEBUG)
+ LOG.debug("write dictionary");
+ try {
+ pageWriter.writeDictionaryPage(dictionaryPage);
+ } catch (IOException e) {
+ throw new ParquetEncodingException("could not write dictionary page for " + path, e);
+ }
+ dataColumn.resetDictionary();
+ }
+ }
+
+ /**
+ * Used to decide when to write a page
+ *
+ * @return the number of bytes of memory used to buffer the current data
+ */
+ long getCurrentPageBufferedSize() {
+ return repetitionLevelColumn.getBufferedSize()
+ + definitionLevelColumn.getBufferedSize()
+ + dataColumn.getBufferedSize();
+ }
+
+ /**
+ * Used to decide when to write a page or row group
+ *
+ * @return the number of bytes of memory used to buffer the current data and the previously written pages
+ */
+ long getTotalBufferedSize() {
+ return repetitionLevelColumn.getBufferedSize()
+ + definitionLevelColumn.getBufferedSize()
+ + dataColumn.getBufferedSize()
+ + pageWriter.getMemSize();
+ }
+
+ /**
+ * @return actual memory used
+ */
+ long allocatedSize() {
+ return repetitionLevelColumn.getAllocatedSize()
+ + definitionLevelColumn.getAllocatedSize()
+ + dataColumn.getAllocatedSize()
+ + pageWriter.allocatedSize();
+ }
+
+ /**
+ * @param indent
+ * a prefix to format lines
+ * @return a formatted string showing how memory is used
+ */
+ String memUsageString(String indent) {
+ StringBuilder b = new StringBuilder(indent).append(path).append(" {\n");
+ b.append(indent).append(" r:").append(repetitionLevelColumn.getAllocatedSize()).append(" bytes\n");
+ b.append(indent).append(" d:").append(definitionLevelColumn.getAllocatedSize()).append(" bytes\n");
+ b.append(dataColumn.memUsageString(indent + " data:")).append("\n");
+ b.append(pageWriter.memUsageString(indent + " pages:")).append("\n");
+ b.append(indent).append(String.format(" total: %,d/%,d", getTotalBufferedSize(), allocatedSize())).append("\n");
+ b.append(indent).append("}\n");
+ return b.toString();
+ }
+
+ long getRowsWrittenSoFar() {
+ return this.rowsWrittenSoFar;
+ }
+
+ /**
+ * Writes the current data to a new page in the page store
+ */
+ void writePage() {
+ this.rowsWrittenSoFar += pageRowCount;
+ if (DEBUG)
+ LOG.debug("write page");
+ try {
+ writePage(pageRowCount, valueCount, statistics, repetitionLevelColumn, definitionLevelColumn, dataColumn);
+ } catch (IOException e) {
+ throw new ParquetEncodingException("could not write page for " + path, e);
+ }
+ repetitionLevelColumn.reset();
+ definitionLevelColumn.reset();
+ dataColumn.reset();
+ valueCount = 0;
+ resetStatistics();
+ pageRowCount = 0;
+ }
+
+ abstract void writePage(int rowCount, int valueCount, Statistics<?> statistics, ValuesWriter repetitionLevels,
+ ValuesWriter definitionLevels, ValuesWriter values) throws IOException;
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java
index c1f5d67b0..646e31aa7 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java
@@ -23,261 +23,40 @@
import java.io.IOException;
import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.column.ColumnWriter;
import org.apache.parquet.column.ParquetProperties;
-import org.apache.parquet.column.page.DictionaryPage;
import org.apache.parquet.column.page.PageWriter;
import org.apache.parquet.column.statistics.Statistics;
import org.apache.parquet.column.values.ValuesWriter;
-import org.apache.parquet.io.ParquetEncodingException;
-import org.apache.parquet.io.api.Binary;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Writes (repetition level, definition level, value) triplets and deals with writing pages to the underlying layer.
*/
-final class ColumnWriterV1 implements ColumnWriter {
- private static final Logger LOG = LoggerFactory.getLogger(ColumnWriterV1.class);
+final class ColumnWriterV1 extends ColumnWriterBase {
- // By default: Debugging disabled this way (using the "if (DEBUG)" IN the methods) to allow
- // the java compiler (not the JIT) to remove the unused statements during build time.
- private static final boolean DEBUG = false;
-
- private final ColumnDescriptor path;
- private final PageWriter pageWriter;
- private final ParquetProperties props;
-
- private ValuesWriter repetitionLevelColumn;
- private ValuesWriter definitionLevelColumn;
- private ValuesWriter dataColumn;
- private int valueCount;
- private int valueCountForNextSizeCheck;
-
- private Statistics statistics;
-
- public ColumnWriterV1(ColumnDescriptor path, PageWriter pageWriter,
- ParquetProperties props) {
- this.path = path;
- this.pageWriter = pageWriter;
- this.props = props;
-
- // initial check of memory usage. So that we have enough data to make an initial prediction
- this.valueCountForNextSizeCheck = props.getMinRowCountForPageSizeCheck();
-
- resetStatistics();
-
- this.repetitionLevelColumn = props.newRepetitionLevelWriter(path);
- this.definitionLevelColumn = props.newDefinitionLevelWriter(path);
- this.dataColumn = props.newValuesWriter(path);
- }
-
- private void log(Object value, int r, int d) {
- if (DEBUG) LOG.debug( "{} {} r:{} d:{}", path, value, r, d);
- }
-
- private void resetStatistics() {
- this.statistics = Statistics.createStats(this.path.getPrimitiveType());
- }
-
- /**
- * Counts how many values have been written and checks the memory usage to flush the page when we reach the page threshold.
- *
- * We measure the memory used when we reach the mid point toward our estimated count.
- * We then update the estimate and flush the page if we reached the threshold.
- *
- * That way we check the memory size log2(n) times.
- *
- */
- private void accountForValueWritten() {
- ++ valueCount;
- if (valueCount > valueCountForNextSizeCheck) {
- // not checking the memory used for every value
- long memSize = repetitionLevelColumn.getBufferedSize()
- + definitionLevelColumn.getBufferedSize()
- + dataColumn.getBufferedSize();
- if (memSize > props.getPageSizeThreshold()) {
- // we will write the current page and check again the size at the predicted middle of next page
- if (props.estimateNextSizeCheck()) {
- valueCountForNextSizeCheck = valueCount / 2;
- } else {
- valueCountForNextSizeCheck = props.getMinRowCountForPageSizeCheck();
- }
- writePage();
- } else if (props.estimateNextSizeCheck()) {
- // not reached the threshold, will check again midway
- valueCountForNextSizeCheck = (int)(valueCount + ((float)valueCount * props.getPageSizeThreshold() / memSize)) / 2 + 1;
- } else {
- valueCountForNextSizeCheck += props.getMinRowCountForPageSizeCheck();
- }
- }
- }
-
- private void updateStatisticsNumNulls() {
- statistics.incrementNumNulls();
- }
-
- private void updateStatistics(int value) {
- statistics.updateStats(value);
- }
-
- private void updateStatistics(long value) {
- statistics.updateStats(value);
- }
-
- private void updateStatistics(float value) {
- statistics.updateStats(value);
- }
-
- private void updateStatistics(double value) {
- statistics.updateStats(value);
- }
-
- private void updateStatistics(Binary value) {
- statistics.updateStats(value);
- }
-
- private void updateStatistics(boolean value) {
- statistics.updateStats(value);
- }
-
- private void writePage() {
- if (DEBUG) LOG.debug("write page");
- try {
- pageWriter.writePage(
- concat(repetitionLevelColumn.getBytes(), definitionLevelColumn.getBytes(), dataColumn.getBytes()),
- valueCount,
- statistics,
- repetitionLevelColumn.getEncoding(),
- definitionLevelColumn.getEncoding(),
- dataColumn.getEncoding());
- } catch (IOException e) {
- throw new ParquetEncodingException("could not write page for " + path, e);
- }
- repetitionLevelColumn.reset();
- definitionLevelColumn.reset();
- dataColumn.reset();
- valueCount = 0;
- resetStatistics();
- }
-
- @Override
- public void writeNull(int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(null, repetitionLevel, definitionLevel);
- repetitionLevelColumn.writeInteger(repetitionLevel);
- definitionLevelColumn.writeInteger(definitionLevel);
- updateStatisticsNumNulls();
- accountForValueWritten();
- }
-
- @Override
- public void write(double value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevelColumn.writeInteger(repetitionLevel);
- definitionLevelColumn.writeInteger(definitionLevel);
- dataColumn.writeDouble(value);
- updateStatistics(value);
- accountForValueWritten();
- }
-
- @Override
- public void write(float value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevelColumn.writeInteger(repetitionLevel);
- definitionLevelColumn.writeInteger(definitionLevel);
- dataColumn.writeFloat(value);
- updateStatistics(value);
- accountForValueWritten();
- }
-
- @Override
- public void write(Binary value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevelColumn.writeInteger(repetitionLevel);
- definitionLevelColumn.writeInteger(definitionLevel);
- dataColumn.writeBytes(value);
- updateStatistics(value);
- accountForValueWritten();
+ ColumnWriterV1(ColumnDescriptor path, PageWriter pageWriter, ParquetProperties props) {
+ super(path, pageWriter, props);
}
@Override
- public void write(boolean value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevelColumn.writeInteger(repetitionLevel);
- definitionLevelColumn.writeInteger(definitionLevel);
- dataColumn.writeBoolean(value);
- updateStatistics(value);
- accountForValueWritten();
+ ValuesWriter createRLWriter(ParquetProperties props, ColumnDescriptor path) {
+ return props.newRepetitionLevelWriter(path);
}
@Override
- public void write(int value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevelColumn.writeInteger(repetitionLevel);
- definitionLevelColumn.writeInteger(definitionLevel);
- dataColumn.writeInteger(value);
- updateStatistics(value);
- accountForValueWritten();
+ ValuesWriter createDLWriter(ParquetProperties props, ColumnDescriptor path) {
+ return props.newDefinitionLevelWriter(path);
}
@Override
- public void write(long value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevelColumn.writeInteger(repetitionLevel);
- definitionLevelColumn.writeInteger(definitionLevel);
- dataColumn.writeLong(value);
- updateStatistics(value);
- accountForValueWritten();
- }
-
- public void flush() {
- if (valueCount > 0) {
- writePage();
- }
- final DictionaryPage dictionaryPage = dataColumn.toDictPageAndClose();
- if (dictionaryPage != null) {
- if (DEBUG) LOG.debug("write dictionary");
- try {
- pageWriter.writeDictionaryPage(dictionaryPage);
- } catch (IOException e) {
- throw new ParquetEncodingException("could not write dictionary page for " + path, e);
- }
- dataColumn.resetDictionary();
- }
- }
-
- @Override
- public void close() {
- flush();
- // Close the Values writers.
- repetitionLevelColumn.close();
- definitionLevelColumn.close();
- dataColumn.close();
- }
-
- @Override
- public long getBufferedSizeInMemory() {
- return repetitionLevelColumn.getBufferedSize()
- + definitionLevelColumn.getBufferedSize()
- + dataColumn.getBufferedSize()
- + pageWriter.getMemSize();
- }
-
- public long allocatedSize() {
- return repetitionLevelColumn.getAllocatedSize()
- + definitionLevelColumn.getAllocatedSize()
- + dataColumn.getAllocatedSize()
- + pageWriter.allocatedSize();
- }
-
- public String memUsageString(String indent) {
- StringBuilder b = new StringBuilder(indent).append(path).append(" {\n");
- b.append(repetitionLevelColumn.memUsageString(indent + " r:")).append("\n");
- b.append(definitionLevelColumn.memUsageString(indent + " d:")).append("\n");
- b.append(dataColumn.memUsageString(indent + " data:")).append("\n");
- b.append(pageWriter.memUsageString(indent + " pages:")).append("\n");
- b.append(indent).append(String.format(" total: %,d/%,d", getBufferedSizeInMemory(), allocatedSize())).append("\n");
- b.append(indent).append("}\n");
- return b.toString();
+ void writePage(int rowCount, int valueCount, Statistics<?> statistics, ValuesWriter repetitionLevels,
+ ValuesWriter definitionLevels, ValuesWriter values) throws IOException {
+ pageWriter.writePage(
+ concat(repetitionLevels.getBytes(), definitionLevels.getBytes(), values.getBytes()),
+ valueCount,
+ rowCount,
+ statistics,
+ repetitionLevels.getEncoding(),
+ definitionLevels.getEncoding(),
+ values.getEncoding());
}
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java
index 9abdee8a5..04076c96b 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java
@@ -23,291 +23,67 @@
import org.apache.parquet.Ints;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.column.ColumnWriter;
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.ParquetProperties;
-import org.apache.parquet.column.page.DictionaryPage;
import org.apache.parquet.column.page.PageWriter;
import org.apache.parquet.column.statistics.Statistics;
import org.apache.parquet.column.values.ValuesWriter;
+import org.apache.parquet.column.values.bitpacking.DevNullValuesWriter;
import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder;
+import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter;
import org.apache.parquet.io.ParquetEncodingException;
-import org.apache.parquet.io.api.Binary;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Writes (repetition level, definition level, value) triplets and deals with writing pages to the underlying layer.
*/
-final class ColumnWriterV2 implements ColumnWriter {
- private static final Logger LOG = LoggerFactory.getLogger(ColumnWriterV2.class);
+final class ColumnWriterV2 extends ColumnWriterBase {
- // By default: Debugging disabled this way (using the "if (DEBUG)" IN the methods) to allow
- // the java compiler (not the JIT) to remove the unused statements during build time.
- private static final boolean DEBUG = false;
-
- private final ColumnDescriptor path;
- private final PageWriter pageWriter;
- private RunLengthBitPackingHybridEncoder repetitionLevelColumn;
- private RunLengthBitPackingHybridEncoder definitionLevelColumn;
- private ValuesWriter dataColumn;
- private int valueCount;
-
- private Statistics<?> statistics;
- private long rowsWrittenSoFar = 0;
-
- public ColumnWriterV2(
- ColumnDescriptor path,
- PageWriter pageWriter,
- ParquetProperties props) {
- this.path = path;
- this.pageWriter = pageWriter;
- resetStatistics();
-
- this.repetitionLevelColumn = props.newRepetitionLevelEncoder(path);
- this.definitionLevelColumn = props.newDefinitionLevelEncoder(path);
- this.dataColumn = props.newValuesWriter(path);
- }
-
- private void log(Object value, int r, int d) {
- LOG.debug("{} {} r:{} d:{}", path, value, r, d);
- }
-
- private void resetStatistics() {
- this.statistics = Statistics.createStats(path.getPrimitiveType());
- }
-
- private void definitionLevel(int definitionLevel) {
- try {
- definitionLevelColumn.writeInt(definitionLevel);
- } catch (IOException e) {
- throw new ParquetEncodingException("illegal definition level " + definitionLevel + " for column " + path, e);
+ // Extending the original implementation to not to write the size of the data as the original writer would
+ private static class RLEWriterForV2 extends RunLengthBitPackingHybridValuesWriter {
+ public RLEWriterForV2(RunLengthBitPackingHybridEncoder encoder) {
+ super(encoder);
}
- }
- private void repetitionLevel(int repetitionLevel) {
- try {
- repetitionLevelColumn.writeInt(repetitionLevel);
- } catch (IOException e) {
- throw new ParquetEncodingException("illegal repetition level " + repetitionLevel + " for column " + path, e);
- }
- }
-
- /**
- * writes the current null value
- * @param repetitionLevel
- * @param definitionLevel
- */
- public void writeNull(int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(null, repetitionLevel, definitionLevel);
- repetitionLevel(repetitionLevel);
- definitionLevel(definitionLevel);
- statistics.incrementNumNulls();
- ++ valueCount;
- }
-
- @Override
- public void close() {
- // Close the Values writers.
- repetitionLevelColumn.close();
- definitionLevelColumn.close();
- dataColumn.close();
- }
-
- @Override
- public long getBufferedSizeInMemory() {
- return repetitionLevelColumn.getBufferedSize()
- + definitionLevelColumn.getBufferedSize()
- + dataColumn.getBufferedSize()
- + pageWriter.getMemSize();
- }
-
- /**
- * writes the current value
- * @param value
- * @param repetitionLevel
- * @param definitionLevel
- */
- public void write(double value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevel(repetitionLevel);
- definitionLevel(definitionLevel);
- dataColumn.writeDouble(value);
- statistics.updateStats(value);
- ++ valueCount;
- }
-
- /**
- * writes the current value
- * @param value
- * @param repetitionLevel
- * @param definitionLevel
- */
- public void write(float value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevel(repetitionLevel);
- definitionLevel(definitionLevel);
- dataColumn.writeFloat(value);
- statistics.updateStats(value);
- ++ valueCount;
- }
-
- /**
- * writes the current value
- * @param value
- * @param repetitionLevel
- * @param definitionLevel
- */
- public void write(Binary value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevel(repetitionLevel);
- definitionLevel(definitionLevel);
- dataColumn.writeBytes(value);
- statistics.updateStats(value);
- ++ valueCount;
- }
-
- /**
- * writes the current value
- * @param value
- * @param repetitionLevel
- * @param definitionLevel
- */
- public void write(boolean value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevel(repetitionLevel);
- definitionLevel(definitionLevel);
- dataColumn.writeBoolean(value);
- statistics.updateStats(value);
- ++ valueCount;
- }
-
- /**
- * writes the current value
- * @param value
- * @param repetitionLevel
- * @param definitionLevel
- */
- public void write(int value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevel(repetitionLevel);
- definitionLevel(definitionLevel);
- dataColumn.writeInteger(value);
- statistics.updateStats(value);
- ++ valueCount;
- }
-
- /**
- * writes the current value
- * @param value
- * @param repetitionLevel
- * @param definitionLevel
- */
- public void write(long value, int repetitionLevel, int definitionLevel) {
- if (DEBUG) log(value, repetitionLevel, definitionLevel);
- repetitionLevel(repetitionLevel);
- definitionLevel(definitionLevel);
- dataColumn.writeLong(value);
- statistics.updateStats(value);
- ++ valueCount;
- }
-
- /**
- * Finalizes the Column chunk. Possibly adding extra pages if needed (dictionary, ...)
- * Is called right after writePage
- */
- public void finalizeColumnChunk() {
- final DictionaryPage dictionaryPage = dataColumn.toDictPageAndClose();
- if (dictionaryPage != null) {
- if (DEBUG) LOG.debug("write dictionary");
+ @Override
+ public BytesInput getBytes() {
try {
- pageWriter.writeDictionaryPage(dictionaryPage);
+ return encoder.toBytes();
} catch (IOException e) {
- throw new ParquetEncodingException("could not write dictionary page for " + path, e);
+ throw new ParquetEncodingException(e);
}
- dataColumn.resetDictionary();
}
}
- /**
- * used to decide when to write a page
- * @return the number of bytes of memory used to buffer the current data
- */
- public long getCurrentPageBufferedSize() {
- return repetitionLevelColumn.getBufferedSize()
- + definitionLevelColumn.getBufferedSize()
- + dataColumn.getBufferedSize();
- }
-
- /**
- * used to decide when to write a page or row group
- * @return the number of bytes of memory used to buffer the current data and the previously written pages
- */
- public long getTotalBufferedSize() {
- return repetitionLevelColumn.getBufferedSize()
- + definitionLevelColumn.getBufferedSize()
- + dataColumn.getBufferedSize()
- + pageWriter.getMemSize();
- }
+ private static final ValuesWriter NULL_WRITER = new DevNullValuesWriter();
- /**
- * @return actual memory used
- */
- public long allocatedSize() {
- return repetitionLevelColumn.getAllocatedSize()
- + definitionLevelColumn.getAllocatedSize()
- + dataColumn.getAllocatedSize()
- + pageWriter.allocatedSize();
+ ColumnWriterV2(ColumnDescriptor path, PageWriter pageWriter, ParquetProperties props) {
+ super(path, pageWriter, props);
}
- /**
- * @param indent a prefix to format lines
- * @return a formatted string showing how memory is used
- */
- public String memUsageString(String indent) {
- StringBuilder b = new StringBuilder(indent).append(path).append(" {\n");
- b.append(indent).append(" r:").append(repetitionLevelColumn.getAllocatedSize()).append(" bytes\n");
- b.append(indent).append(" d:").append(definitionLevelColumn.getAllocatedSize()).append(" bytes\n");
- b.append(dataColumn.memUsageString(indent + " data:")).append("\n");
- b.append(pageWriter.memUsageString(indent + " pages:")).append("\n");
- b.append(indent).append(String.format(" total: %,d/%,d", getTotalBufferedSize(), allocatedSize())).append("\n");
- b.append(indent).append("}\n");
- return b.toString();
+ @Override
+ ValuesWriter createRLWriter(ParquetProperties props, ColumnDescriptor path) {
+ return path.getMaxRepetitionLevel() == 0 ? NULL_WRITER : new RLEWriterForV2(props.newRepetitionLevelEncoder(path));
}
- public long getRowsWrittenSoFar() {
- return this.rowsWrittenSoFar;
+ @Override
+ ValuesWriter createDLWriter(ParquetProperties props, ColumnDescriptor path) {
+ return path.getMaxDefinitionLevel() == 0 ? NULL_WRITER : new RLEWriterForV2(props.newDefinitionLevelEncoder(path));
}
- /**
- * writes the current data to a new page in the page store
- * @param rowCount how many rows have been written so far
- */
- public void writePage(long rowCount) {
- int pageRowCount = Ints.checkedCast(rowCount - rowsWrittenSoFar);
- this.rowsWrittenSoFar = rowCount;
- if (DEBUG) LOG.debug("write page");
- try {
- // TODO: rework this API. Those must be called *in that order*
- BytesInput bytes = dataColumn.getBytes();
- Encoding encoding = dataColumn.getEncoding();
- pageWriter.writePageV2(
- pageRowCount,
- Ints.checkedCast(statistics.getNumNulls()),
- valueCount,
- path.getMaxRepetitionLevel() == 0 ? BytesInput.empty() : repetitionLevelColumn.toBytes(),
- path.getMaxDefinitionLevel() == 0 ? BytesInput.empty() : definitionLevelColumn.toBytes(),
- encoding,
- bytes,
- statistics
- );
- } catch (IOException e) {
- throw new ParquetEncodingException("could not write page for " + path, e);
- }
- repetitionLevelColumn.reset();
- definitionLevelColumn.reset();
- dataColumn.reset();
- valueCount = 0;
- resetStatistics();
+ @Override
+ void writePage(int rowCount, int valueCount, Statistics<?> statistics, ValuesWriter repetitionLevels,
+ ValuesWriter definitionLevels, ValuesWriter values) throws IOException {
+ // TODO: rework this API. The bytes shall be retrieved before the encoding (encoding might be different otherwise)
+ BytesInput bytes = values.getBytes();
+ Encoding encoding = values.getEncoding();
+ pageWriter.writePageV2(
+ rowCount,
+ Ints.checkedCast(statistics.getNumNulls()),
+ valueCount,
+ repetitionLevels.getBytes(),
+ definitionLevels.getBytes(),
+ encoding,
+ bytes,
+ statistics);
}
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/SynchronizingColumnReader.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/SynchronizingColumnReader.java
new file mode 100644
index 000000000..50f05c8af
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/SynchronizingColumnReader.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.impl;
+
+import java.util.PrimitiveIterator;
+
+import org.apache.parquet.VersionParser.ParsedVersion;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.ColumnReader;
+import org.apache.parquet.column.page.DataPage;
+import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.io.RecordReader;
+import org.apache.parquet.io.api.PrimitiveConverter;
+
+/**
+ * A {@link ColumnReader} implementation for utilizing indexes. When filtering using column indexes we might skip
+ * reading some pages for different columns. Because the rows are not aligned between the pages of the different columns
+ * it might be required to skip some values in this {@link ColumnReader} so we provide only the required values for the
+ * higher API ({@link RecordReader}) and they do not need to handle or know about the skipped pages. The values (and the
+ * related rl and dl) are skipped based on the iterator of the required row indexes and the first row index of each
+ * page.<br>
+ * For example:
+ *
+ * <pre>
+ * rows col1 col2 col3
+ * ┌──────┬──────┬──────┐
+ * 0 │ p0 │ │ │
+ * ╞══════╡ p0 │ p0 │
+ * 20 │ p1(X)│------│------│
+ * ╞══════╪══════╡ │
+ * 40 │ p2(X)│ │------│
+ * ╞══════╡ p1(X)╞══════╡
+ * 60 │ p3(X)│ │------│
+ * ╞══════╪══════╡ │
+ * 80 │ p4 │ │ p1 │
+ * ╞══════╡ p2 │ │
+ * 100 │ p5 │ │ │
+ * └──────┴──────┴──────┘
+ * </pre>
+ *
+ * The pages 1, 2, 3 in col1 are skipped so we have to skip the rows [20, 79]. Because page 1 in col2 contains values
+ * only for the rows [40, 79] we skip this entire page as well. To synchronize the row reading we have to skip the
+ * values (and the related rl and dl) for the rows [20, 39] in the end of the page 0 for col2. Similarly, we have to
+ * skip values while reading page0 and page1 for col3.
+ */
+class SynchronizingColumnReader extends ColumnReaderBase {
+
+ private final PrimitiveIterator.OfLong rowIndexes;
+ private long currentRow;
+ private long targetRow;
+ private long lastRowInPage;
+ private int valuesReadFromPage;
+
+ SynchronizingColumnReader(ColumnDescriptor path, PageReader pageReader, PrimitiveConverter converter,
+ ParsedVersion writerVersion, PrimitiveIterator.OfLong rowIndexes) {
+ super(path, pageReader, converter, writerVersion);
+ this.rowIndexes = rowIndexes;
+ targetRow = Long.MIN_VALUE;
+ consume();
+ }
+
+ @Override
+ boolean isPageFullyConsumed() {
+ return getPageValueCount() <= valuesReadFromPage || lastRowInPage < targetRow;
+ }
+
+ @Override
+ boolean isFullyConsumed() {
+ return !rowIndexes.hasNext();
+ }
+
+ @Override
+ boolean skipRL(int rl) {
+ ++valuesReadFromPage;
+ if (rl == 0) {
+ ++currentRow;
+ if (currentRow > targetRow) {
+ targetRow = rowIndexes.hasNext() ? rowIndexes.nextLong() : Long.MAX_VALUE;
+ }
+ }
+ return currentRow < targetRow;
+ }
+
+ @Override
+ protected void newPageInitialized(DataPage page) {
+ long firstRowIndex = page.getFirstRowIndex()
+ .orElseThrow(() -> new IllegalArgumentException("Missing firstRowIndex for synchronizing values"));
+ int rowCount = page.getIndexRowCount()
+ .orElseThrow(() -> new IllegalArgumentException("Missing rowCount for synchronizing values"));
+ currentRow = firstRowIndex - 1;
+ lastRowInPage = firstRowIndex + rowCount - 1;
+ valuesReadFromPage = 0;
+ }
+
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/page/DataPage.java b/parquet-column/src/main/java/org/apache/parquet/column/page/DataPage.java
index 4d8f381f5..fd1875edd 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/page/DataPage.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/page/DataPage.java
@@ -18,16 +18,24 @@
*/
package org.apache.parquet.column.page;
+import java.util.Optional;
+
/**
* one data page in a chunk
*/
abstract public class DataPage extends Page {
private final int valueCount;
+ private final long firstRowIndex;
DataPage(int compressedSize, int uncompressedSize, int valueCount) {
+ this(compressedSize, uncompressedSize, valueCount, -1);
+ }
+
+ DataPage(int compressedSize, int uncompressedSize, int valueCount, long firstRowIndex) {
super(compressedSize, uncompressedSize);
this.valueCount = valueCount;
+ this.firstRowIndex = firstRowIndex;
}
/**
@@ -37,6 +45,20 @@ public int getValueCount() {
return valueCount;
}
+ /**
+ * @return the index of the first row in this page if the related data is available (the optional column-index
+ * contains this value)
+ */
+ public Optional<Long> getFirstRowIndex() {
+ return firstRowIndex < 0 ? Optional.empty() : Optional.of(firstRowIndex);
+ }
+
+ /**
+ * @return the number of rows in this page if the related data is available (in case of pageV1 the optional
+ * column-index contains this value)
+ */
+ public abstract Optional<Integer> getIndexRowCount();
+
public abstract <T> T accept(Visitor<T> visitor);
public static interface Visitor<T> {
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/page/DataPageV1.java b/parquet-column/src/main/java/org/apache/parquet/column/page/DataPageV1.java
index 56928c381..b1f68aefb 100755
--- a/parquet-column/src/main/java/org/apache/parquet/column/page/DataPageV1.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/page/DataPageV1.java
@@ -18,6 +18,8 @@
*/
package org.apache.parquet.column.page;
+import java.util.Optional;
+
import org.apache.parquet.Ints;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.column.Encoding;
@@ -30,6 +32,7 @@
private final Encoding rlEncoding;
private final Encoding dlEncoding;
private final Encoding valuesEncoding;
+ private final int indexRowCount;
/**
* @param bytes the bytes for this page
@@ -47,6 +50,29 @@ public DataPageV1(BytesInput bytes, int valueCount, int uncompressedSize, Statis
this.rlEncoding = rlEncoding;
this.dlEncoding = dlEncoding;
this.valuesEncoding = valuesEncoding;
+ this.indexRowCount = -1;
+ }
+
+ /**
+ * @param bytes the bytes for this page
+ * @param valueCount count of values in this page
+ * @param uncompressedSize the uncompressed size of the page
+ * @param firstRowIndex the index of the first row in this page
+ * @param rowCount the number of rows in this page
+ * @param statistics of the page's values (max, min, num_null)
+ * @param rlEncoding the repetition level encoding for this page
+ * @param dlEncoding the definition level encoding for this page
+ * @param valuesEncoding the values encoding for this page
+ */
+ public DataPageV1(BytesInput bytes, int valueCount, int uncompressedSize, long firstRowIndex, int rowCount,
+ Statistics<?> statistics, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding) {
+ super(Ints.checkedCast(bytes.size()), uncompressedSize, valueCount, firstRowIndex);
+ this.bytes = bytes;
+ this.statistics = statistics;
+ this.rlEncoding = rlEncoding;
+ this.dlEncoding = dlEncoding;
+ this.valuesEncoding = valuesEncoding;
+ this.indexRowCount = rowCount;
}
/**
@@ -94,4 +120,9 @@ public String toString() {
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
+
+ @Override
+ public Optional<Integer> getIndexRowCount() {
+ return indexRowCount < 0 ? Optional.empty() : Optional.of(indexRowCount);
+ }
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/page/DataPageV2.java b/parquet-column/src/main/java/org/apache/parquet/column/page/DataPageV2.java
index 62dac8371..a1700aea0 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/page/DataPageV2.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/page/DataPageV2.java
@@ -18,6 +18,8 @@
*/
package org.apache.parquet.column.page;
+import java.util.Optional;
+
import org.apache.parquet.Ints;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.column.Encoding;
@@ -50,6 +52,32 @@ public static DataPageV2 uncompressed(
false);
}
+ /**
+ * @param rowCount count of rows
+ * @param nullCount count of nulls
+ * @param valueCount count of values
+ * @param firstRowIndex the index of the first row in this page
+ * @param repetitionLevels RLE encoded repetition levels
+ * @param definitionLevels RLE encoded definition levels
+ * @param dataEncoding encoding for the data
+ * @param data data encoded with dataEncoding
+ * @param statistics optional statistics for this page
+ * @return an uncompressed page
+ */
+ public static DataPageV2 uncompressed(
+ int rowCount, int nullCount, int valueCount, long firstRowIndex,
+ BytesInput repetitionLevels, BytesInput definitionLevels,
+ Encoding dataEncoding, BytesInput data,
+ Statistics<?> statistics) {
+ return new DataPageV2(
+ rowCount, nullCount, valueCount, firstRowIndex,
+ repetitionLevels, definitionLevels,
+ dataEncoding, data,
+ Ints.checkedCast(repetitionLevels.size() + definitionLevels.size() + data.size()),
+ statistics,
+ false);
+ }
+
/**
* @param rowCount count of rows
* @param nullCount count of nulls
@@ -104,6 +132,25 @@ public DataPageV2(
this.isCompressed = isCompressed;
}
+ private DataPageV2(
+ int rowCount, int nullCount, int valueCount, long firstRowIndex,
+ BytesInput repetitionLevels, BytesInput definitionLevels,
+ Encoding dataEncoding, BytesInput data,
+ int uncompressedSize,
+ Statistics<?> statistics,
+ boolean isCompressed) {
+ super(Ints.checkedCast(repetitionLevels.size() + definitionLevels.size() + data.size()), uncompressedSize,
+ valueCount, firstRowIndex);
+ this.rowCount = rowCount;
+ this.nullCount = nullCount;
+ this.repetitionLevels = repetitionLevels;
+ this.definitionLevels = definitionLevels;
+ this.dataEncoding = dataEncoding;
+ this.data = data;
+ this.statistics = statistics;
+ this.isCompressed = isCompressed;
+ }
+
public int getRowCount() {
return rowCount;
}
@@ -136,6 +183,11 @@ public boolean isCompressed() {
return isCompressed;
}
+ @Override
+ public Optional<Integer> getIndexRowCount() {
+ return Optional.of(rowCount);
+ }
+
@Override
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/page/PageReadStore.java b/parquet-column/src/main/java/org/apache/parquet/column/page/PageReadStore.java
index 24d582554..753bda890 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/page/PageReadStore.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/page/PageReadStore.java
@@ -18,6 +18,8 @@
*/
package org.apache.parquet.column.page;
+import java.util.Optional;
+import java.util.PrimitiveIterator;
import org.apache.parquet.column.ColumnDescriptor;
/**
@@ -29,7 +31,8 @@
/**
*
- * @param descriptor the descriptor of the column
+ * @param descriptor
+ * the descriptor of the column
* @return the page reader for that column
*/
PageReader getPageReader(ColumnDescriptor descriptor);
@@ -40,4 +43,14 @@
*/
long getRowCount();
+ /**
+ * Returns the indexes of the rows to be read/built if the related data is available. All the rows which index is not
+ * returned shall be skipped.
+ *
+ * @return the optional of the incremental iterator of the row indexes or an empty optional if the related data is not
+ * available
+ */
+ default Optional<PrimitiveIterator.OfLong> getRowIndexes() {
+ return Optional.empty();
+ }
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/page/PageWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/page/PageWriter.java
index a2d079f9c..a72be48b5 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/page/PageWriter.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/page/PageWriter.java
@@ -20,7 +20,6 @@
import java.io.IOException;
-import org.apache.parquet.bytes.ByteBufferAllocator;
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.statistics.Statistics;
@@ -39,9 +38,25 @@
* @param dlEncoding definition level encoding
* @param valuesEncoding values encoding
* @throws IOException if there is an exception while writing page data
+ * @deprecated will be removed in 2.0.0. This method does not support writing column indexes; Use
+ * {@link #writePage(BytesInput, int, int, Statistics, Encoding, Encoding, Encoding)} instead
*/
+ @Deprecated
void writePage(BytesInput bytesInput, int valueCount, Statistics<?> statistics, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding) throws IOException;
+ /**
+ * writes a single page
+ * @param bytesInput the bytes for the page
+ * @param valueCount the number of values in that page
+ * @param rowCount the number of rows in that page
+ * @param statistics the statistics for that page
+ * @param rlEncoding repetition level encoding
+ * @param dlEncoding definition level encoding
+ * @param valuesEncoding values encoding
+ * @throws IOException
+ */
+ void writePage(BytesInput bytesInput, int valueCount, int rowCount, Statistics<?> statistics, Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding) throws IOException;
+
/**
* writes a single page in the new format
* @param rowCount the number of rows in this page
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesReader.java
index 1154bc44e..06771e975 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesReader.java
@@ -179,5 +179,17 @@ public long readLong() {
* Skips the next value in the page
*/
abstract public void skip();
+
+ /**
+ * Skips the next n values in the page
+ *
+ * @param n
+ * the number of values to be skipped
+ */
+ public void skip(int n) {
+ for (int i = 0; i < n; ++i) {
+ skip();
+ }
+ }
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesReader.java
index c8a80fd30..80cfaf2b0 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesReader.java
@@ -90,6 +90,14 @@ public void skip() {
valuesRead++;
}
+ @Override
+ public void skip(int n) {
+ // checkRead() is invoked before incrementing valuesRead so increase valuesRead size in 2 steps
+ valuesRead += n - 1;
+ checkRead();
+ ++valuesRead;
+ }
+
@Override
public int readInteger() {
// TODO: probably implement it separately
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesReader.java
index 1a2ccb9b5..4dbbcb564 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesReader.java
@@ -20,8 +20,6 @@
import java.io.IOException;
-import java.nio.ByteBuffer;
-
import org.apache.parquet.bytes.ByteBufferInputStream;
import org.apache.parquet.column.values.ValuesReader;
import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesReader;
@@ -64,7 +62,15 @@ public Binary readBytes() {
@Override
public void skip() {
- int length = lengthReader.readInteger();
+ skip(1);
+ }
+
+ @Override
+ public void skip(int n) {
+ int length = 0;
+ for (int i = 0; i < n; ++i) {
+ length += lengthReader.readInteger();
+ }
try {
in.skipFully(length);
} catch (IOException e) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesReader.java
index 15ed43438..631c9084d 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesReader.java
@@ -19,7 +19,6 @@
package org.apache.parquet.column.values.plain;
import java.io.IOException;
-import java.nio.ByteBuffer;
import org.apache.parquet.bytes.ByteBufferInputStream;
import org.apache.parquet.column.values.ValuesReader;
import org.apache.parquet.io.ParquetDecodingException;
@@ -51,8 +50,13 @@ public Binary readBytes() {
@Override
public void skip() {
+ skip(1);
+ }
+
+ @Override
+ public void skip(int n) {
try {
- in.skipFully(length);
+ in.skipFully(n * length);
} catch (IOException | RuntimeException e) {
throw new ParquetDecodingException("could not skip bytes at offset " + in.position(), e);
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/PlainValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/PlainValuesReader.java
index f576528a9..127817eb0 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/PlainValuesReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/PlainValuesReader.java
@@ -41,14 +41,26 @@ public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IO
this.in = new LittleEndianDataInputStream(stream.remainingStream());
}
+ @Override
+ public void skip() {
+ skip(1);
+ }
+
+ void skipBytesFully(int n) throws IOException {
+ int skipped = 0;
+ while (skipped < n) {
+ skipped += in.skipBytes(n - skipped);
+ }
+ }
+
public static class DoublePlainValuesReader extends PlainValuesReader {
@Override
- public void skip() {
+ public void skip(int n) {
try {
- in.skipBytes(8);
+ skipBytesFully(n * 8);
} catch (IOException e) {
- throw new ParquetDecodingException("could not skip double", e);
+ throw new ParquetDecodingException("could not skip " + n + " double values", e);
}
}
@@ -65,11 +77,11 @@ public double readDouble() {
public static class FloatPlainValuesReader extends PlainValuesReader {
@Override
- public void skip() {
+ public void skip(int n) {
try {
- in.skipBytes(4);
+ skipBytesFully(n * 4);
} catch (IOException e) {
- throw new ParquetDecodingException("could not skip float", e);
+ throw new ParquetDecodingException("could not skip " + n + " floats", e);
}
}
@@ -86,11 +98,11 @@ public float readFloat() {
public static class IntegerPlainValuesReader extends PlainValuesReader {
@Override
- public void skip() {
+ public void skip(int n) {
try {
- in.skipBytes(4);
+ in.skipBytes(n * 4);
} catch (IOException e) {
- throw new ParquetDecodingException("could not skip int", e);
+ throw new ParquetDecodingException("could not skip " + n + " ints", e);
}
}
@@ -107,11 +119,11 @@ public int readInteger() {
public static class LongPlainValuesReader extends PlainValuesReader {
@Override
- public void skip() {
+ public void skip(int n) {
try {
- in.skipBytes(8);
+ in.skipBytes(n * 8);
} catch (IOException e) {
- throw new ParquetDecodingException("could not skip long", e);
+ throw new ParquetDecodingException("could not skip " + n + " longs", e);
}
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/rle/RunLengthBitPackingHybridValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/rle/RunLengthBitPackingHybridValuesWriter.java
index 3b7a5def4..a51a8c4d8 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/rle/RunLengthBitPackingHybridValuesWriter.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/rle/RunLengthBitPackingHybridValuesWriter.java
@@ -19,6 +19,7 @@
package org.apache.parquet.column.values.rle;
import java.io.IOException;
+import java.util.Objects;
import org.apache.parquet.bytes.ByteBufferAllocator;
import org.apache.parquet.Ints;
@@ -28,10 +29,14 @@
import org.apache.parquet.io.ParquetEncodingException;
public class RunLengthBitPackingHybridValuesWriter extends ValuesWriter {
- private final RunLengthBitPackingHybridEncoder encoder;
+ protected final RunLengthBitPackingHybridEncoder encoder;
public RunLengthBitPackingHybridValuesWriter(int bitWidth, int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
- this.encoder = new RunLengthBitPackingHybridEncoder(bitWidth, initialCapacity, pageSize, allocator);
+ this(new RunLengthBitPackingHybridEncoder(bitWidth, initialCapacity, pageSize, allocator));
+ }
+
+ protected RunLengthBitPackingHybridValuesWriter(RunLengthBitPackingHybridEncoder encoder) {
+ this.encoder = Objects.requireNonNull(encoder);
}
@Override
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/rle/ZeroIntegerValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/rle/ZeroIntegerValuesReader.java
index beeb0ad2e..09ca8a1a4 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/rle/ZeroIntegerValuesReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/rle/ZeroIntegerValuesReader.java
@@ -42,4 +42,8 @@ public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IO
public void skip() {
}
+ @Override
+ public void skip(int n) {
+ }
+
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java
new file mode 100644
index 000000000..490cc3e9b
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.parquet.filter2.predicate.Statistics;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.PrimitiveComparator;
+import org.apache.parquet.schema.PrimitiveType;
+
+class BinaryColumnIndexBuilder extends ColumnIndexBuilder {
+ private static class BinaryColumnIndex extends ColumnIndexBase<Binary> {
+ private Binary[] minValues;
+ private Binary[] maxValues;
+
+ private BinaryColumnIndex(PrimitiveType type) {
+ super(type);
+ }
+
+ @Override
+ ByteBuffer getMinValueAsBytes(int pageIndex) {
+ return convert(minValues[pageIndex]);
+ }
+
+ @Override
+ ByteBuffer getMaxValueAsBytes(int pageIndex) {
+ return convert(maxValues[pageIndex]);
+ }
+
+ @Override
+ String getMinValueAsString(int pageIndex) {
+ return stringifier.stringify(minValues[pageIndex]);
+ }
+
+ @Override
+ String getMaxValueAsString(int pageIndex) {
+ return stringifier.stringify(maxValues[pageIndex]);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ <T extends Comparable<T>> Statistics<T> createStats(int arrayIndex) {
+ return (Statistics<T>) new Statistics<Binary>(minValues[arrayIndex], maxValues[arrayIndex], comparator);
+ }
+
+ @Override
+ ValueComparator createValueComparator(Object value) {
+ final Binary v = (Binary) value;
+ return new ValueComparator() {
+ @Override
+ int compareValueToMin(int arrayIndex) {
+ return comparator.compare(v, minValues[arrayIndex]);
+ }
+
+ @Override
+ int compareValueToMax(int arrayIndex) {
+ return comparator.compare(v, maxValues[arrayIndex]);
+ }
+ };
+ }
+ }
+
+ private final List<Binary> minValues = new ArrayList<>();
+ private final List<Binary> maxValues = new ArrayList<>();
+ private final BinaryTruncator truncator;
+ private final int truncateLength;
+
+ private static Binary convert(ByteBuffer buffer) {
+ return Binary.fromReusedByteBuffer(buffer);
+ }
+
+ private static ByteBuffer convert(Binary value) {
+ return value.toByteBuffer();
+ }
+
+ BinaryColumnIndexBuilder(PrimitiveType type, int truncateLength) {
+ truncator = BinaryTruncator.getTruncator(type);
+ this.truncateLength = truncateLength;
+ }
+
+ @Override
+ void addMinMaxFromBytes(ByteBuffer min, ByteBuffer max) {
+ minValues.add(convert(min));
+ maxValues.add(convert(max));
+ }
+
+ @Override
+ void addMinMax(Object min, Object max) {
+ minValues.add(min == null ? null : truncator.truncateMin((Binary) min, truncateLength));
+ maxValues.add(max == null ? null : truncator.truncateMax((Binary) max, truncateLength));
+ }
+
+ @Override
+ ColumnIndexBase<Binary> createColumnIndex(PrimitiveType type) {
+ BinaryColumnIndex columnIndex = new BinaryColumnIndex(type);
+ columnIndex.minValues = minValues.toArray(new Binary[minValues.size()]);
+ columnIndex.maxValues = maxValues.toArray(new Binary[maxValues.size()]);
+ return columnIndex;
+ }
+
+ @Override
+ void clearMinMax() {
+ minValues.clear();
+ maxValues.clear();
+ }
+
+ @Override
+ int compareMinValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(minValues.get(index1), minValues.get(index2));
+ }
+
+ @Override
+ int compareMaxValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(maxValues.get(index1), maxValues.get(index2));
+ }
+
+ @Override
+ int sizeOf(Object value) {
+ return ((Binary) value).length();
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryTruncator.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryTruncator.java
new file mode 100644
index 000000000..bcc43fb86
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryTruncator.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.PrimitiveType;
+
+/**
+ * Class for truncating min/max values for binary types.
+ */
+abstract class BinaryTruncator {
+ enum Validity {
+ VALID, MALFORMED, UNMAPPABLE;
+ }
+
+ private static class CharsetValidator {
+ private final CharBuffer dummyBuffer = CharBuffer.allocate(1024);
+ private final CharsetDecoder decoder;
+
+ CharsetValidator(Charset charset) {
+ decoder = charset.newDecoder();
+ decoder.onMalformedInput(CodingErrorAction.REPORT);
+ decoder.onUnmappableCharacter(CodingErrorAction.REPORT);
+ }
+
+ Validity checkValidity(ByteBuffer buffer) {
+ int pos = buffer.position();
+ CoderResult result = CoderResult.OVERFLOW;
+ while (result.isOverflow()) {
+ dummyBuffer.clear();
+ result = decoder.decode(buffer, dummyBuffer, true);
+ }
+ buffer.position(pos);
+ if (result.isUnderflow()) {
+ return Validity.VALID;
+ } else if (result.isMalformed()) {
+ return Validity.MALFORMED;
+ } else {
+ return Validity.UNMAPPABLE;
+ }
+ }
+ }
+
+ private static final BinaryTruncator NO_OP_TRUNCATOR = new BinaryTruncator() {
+ @Override
+ Binary truncateMin(Binary minValue, int length) {
+ return minValue;
+ }
+
+ @Override
+ Binary truncateMax(Binary maxValue, int length) {
+ return maxValue;
+ }
+ };
+
+ private static final BinaryTruncator DEFAULT_UTF8_TRUNCATOR = new BinaryTruncator() {
+ private final CharsetValidator validator = new CharsetValidator(StandardCharsets.UTF_8);
+
+ @Override
+ Binary truncateMin(Binary minValue, int length) {
+ if (minValue.length() <= length) {
+ return minValue;
+ }
+ ByteBuffer buffer = minValue.toByteBuffer();
+ byte[] array;
+ if (validator.checkValidity(buffer) == Validity.VALID) {
+ array = truncateUtf8(buffer, length);
+ } else {
+ array = truncate(buffer, length);
+ }
+ return array == null ? minValue : Binary.fromConstantByteArray(array);
+ }
+
+ @Override
+ Binary truncateMax(Binary maxValue, int length) {
+ if (maxValue.length() <= length) {
+ return maxValue;
+ }
+ byte[] array;
+ ByteBuffer buffer = maxValue.toByteBuffer();
+ if (validator.checkValidity(buffer) == Validity.VALID) {
+ array = incrementUtf8(truncateUtf8(buffer, length));
+ } else {
+ array = increment(truncate(buffer, length));
+ }
+ return array == null ? maxValue : Binary.fromConstantByteArray(array);
+ }
+
+ // Simply truncate to length
+ private byte[] truncate(ByteBuffer buffer, int length) {
+ assert length < buffer.remaining();
+ byte[] array = new byte[length];
+ buffer.get(array);
+ return array;
+ }
+
+ // Trying to increment the bytes from the last one to the beginning
+ private byte[] increment(byte[] array) {
+ for (int i = array.length - 1; i >= 0; --i) {
+ byte elem = array[i];
+ ++elem;
+ array[i] = elem;
+ if (elem != 0) { // Did not overflow: 0xFF -> 0x00
+ return array;
+ }
+ }
+ return null;
+ }
+
+ // Truncates the buffer to length or less so the remaining bytes form a valid UTF-8 string
+ private byte[] truncateUtf8(ByteBuffer buffer, int length) {
+ assert length < buffer.remaining();
+ ByteBuffer newBuffer = buffer.slice();
+ newBuffer.limit(newBuffer.position() + length);
+ while (validator.checkValidity(newBuffer) != Validity.VALID) {
+ newBuffer.limit(newBuffer.limit() - 1);
+ if (newBuffer.remaining() == 0) {
+ return null;
+ }
+ }
+ byte[] array = new byte[newBuffer.remaining()];
+ newBuffer.get(array);
+ return array;
+ }
+
+ // Trying to increment the bytes from the last one to the beginning until the bytes form a valid UTF-8 string
+ private byte[] incrementUtf8(byte[] array) {
+ if (array == null) {
+ return null;
+ }
+ ByteBuffer buffer = ByteBuffer.wrap(array);
+ for (int i = array.length - 1; i >= 0; --i) {
+ byte prev = array[i];
+ byte inc = prev;
+ while (++inc != 0) { // Until overflow: 0xFF -> 0x00
+ array[i] = inc;
+ switch (validator.checkValidity(buffer)) {
+ case VALID:
+ return array;
+ case UNMAPPABLE:
+ continue; // Increment the i byte once more
+ case MALFORMED:
+ break; // Stop incrementing the i byte; go to the i-1
+ }
+ break; // MALFORMED
+ }
+ array[i] = prev;
+ }
+ return null; // All characters are the largest possible; unable to increment
+ }
+ };
+
+ static BinaryTruncator getTruncator(PrimitiveType type) {
+ if (type == null) {
+ return NO_OP_TRUNCATOR;
+ }
+ switch (type.getPrimitiveTypeName()) {
+ case INT96:
+ return NO_OP_TRUNCATOR;
+ case BINARY:
+ case FIXED_LEN_BYTE_ARRAY:
+ OriginalType originalType = type.getOriginalType();
+ if (originalType == null) {
+ return DEFAULT_UTF8_TRUNCATOR;
+ }
+ switch (originalType) {
+ case UTF8:
+ case ENUM:
+ case JSON:
+ case BSON:
+ return DEFAULT_UTF8_TRUNCATOR;
+ default:
+ return NO_OP_TRUNCATOR;
+ }
+ default:
+ throw new IllegalArgumentException("No truncator is available for the type: " + type);
+ }
+ }
+
+ abstract Binary truncateMin(Binary minValue, int length);
+
+ abstract Binary truncateMax(Binary maxValue, int length);
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BooleanColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BooleanColumnIndexBuilder.java
new file mode 100644
index 000000000..233bd1b02
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BooleanColumnIndexBuilder.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import java.nio.ByteBuffer;
+import org.apache.parquet.filter2.predicate.Statistics;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.PrimitiveComparator;
+import org.apache.parquet.schema.PrimitiveType;
+
+import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
+import it.unimi.dsi.fastutil.booleans.BooleanList;
+
+class BooleanColumnIndexBuilder extends ColumnIndexBuilder {
+ private static class BooleanColumnIndex extends ColumnIndexBase<Boolean> {
+ private boolean[] minValues;
+ private boolean[] maxValues;
+
+ private BooleanColumnIndex(PrimitiveType type) {
+ super(type);
+ }
+
+ @Override
+ ByteBuffer getMinValueAsBytes(int pageIndex) {
+ return convert(minValues[pageIndex]);
+ }
+
+ @Override
+ ByteBuffer getMaxValueAsBytes(int pageIndex) {
+ return convert(maxValues[pageIndex]);
+ }
+
+ @Override
+ String getMinValueAsString(int pageIndex) {
+ return stringifier.stringify(minValues[pageIndex]);
+ }
+
+ @Override
+ String getMaxValueAsString(int pageIndex) {
+ return stringifier.stringify(maxValues[pageIndex]);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ <T extends Comparable<T>> Statistics<T> createStats(int arrayIndex) {
+ return (Statistics<T>) new Statistics<Boolean>(minValues[arrayIndex], maxValues[arrayIndex], comparator);
+ }
+
+ @Override
+ ValueComparator createValueComparator(Object value) {
+ final boolean v = (boolean) value;
+ return new ValueComparator() {
+ @Override
+ int compareValueToMin(int arrayIndex) {
+ return comparator.compare(v, minValues[arrayIndex]);
+ }
+
+ @Override
+ int compareValueToMax(int arrayIndex) {
+ return comparator.compare(v, maxValues[arrayIndex]);
+ }
+ };
+ }
+ }
+
+ private final BooleanList minValues = new BooleanArrayList();
+ private final BooleanList maxValues = new BooleanArrayList();
+
+ private static boolean convert(ByteBuffer buffer) {
+ return buffer.get(0) != 0;
+ }
+
+ private static ByteBuffer convert(boolean value) {
+ return ByteBuffer.allocate(1).put(0, value ? (byte) 1 : 0);
+ }
+
+ @Override
+ void addMinMaxFromBytes(ByteBuffer min, ByteBuffer max) {
+ minValues.add(convert(min));
+ maxValues.add(convert(max));
+ }
+
+ @Override
+ void addMinMax(Object min, Object max) {
+ minValues.add((boolean) min);
+ maxValues.add((boolean) max);
+ }
+
+ @Override
+ ColumnIndexBase<Boolean> createColumnIndex(PrimitiveType type) {
+ BooleanColumnIndex columnIndex = new BooleanColumnIndex(type);
+ columnIndex.minValues = minValues.toBooleanArray();
+ columnIndex.maxValues = maxValues.toBooleanArray();
+ return columnIndex;
+ }
+
+ @Override
+ void clearMinMax() {
+ minValues.clear();
+ maxValues.clear();
+ }
+
+ @Override
+ int compareMinValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(minValues.get(index1), minValues.get(index2));
+ }
+
+ @Override
+ int compareMaxValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(maxValues.get(index1), maxValues.get(index2));
+ }
+
+ @Override
+ int sizeOf(Object value) {
+ return 1;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BoundaryOrder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BoundaryOrder.java
new file mode 100644
index 000000000..e47b5b3f1
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BoundaryOrder.java
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import java.util.PrimitiveIterator;
+import java.util.PrimitiveIterator.OfInt;
+
+import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder.ColumnIndexBase;
+
+/**
+ * Enum for {@link org.apache.parquet.format.BoundaryOrder}. It also contains the implementations of searching for
+ * matching page indexes for column index based filtering.
+ */
+public enum BoundaryOrder {
+ UNORDERED {
+ @Override
+ PrimitiveIterator.OfInt eq(ColumnIndexBase<?>.ValueComparator comparator) {
+ return IndexIterator.filterTranslate(comparator.arrayLength(),
+ arrayIndex -> comparator.compareValueToMin(arrayIndex) >= 0 && comparator.compareValueToMax(arrayIndex) <= 0,
+ comparator::translate);
+ }
+
+ @Override
+ PrimitiveIterator.OfInt gt(ColumnIndexBase<?>.ValueComparator comparator) {
+ return IndexIterator.filterTranslate(comparator.arrayLength(),
+ arrayIndex -> comparator.compareValueToMax(arrayIndex) < 0,
+ comparator::translate);
+ }
+
+ @Override
+ PrimitiveIterator.OfInt gtEq(ColumnIndexBase<?>.ValueComparator comparator) {
+ return IndexIterator.filterTranslate(comparator.arrayLength(),
+ arrayIndex -> comparator.compareValueToMax(arrayIndex) <= 0,
+ comparator::translate);
+ }
+
+ @Override
+ PrimitiveIterator.OfInt lt(ColumnIndexBase<?>.ValueComparator comparator) {
+ return IndexIterator.filterTranslate(comparator.arrayLength(),
+ arrayIndex -> comparator.compareValueToMin(arrayIndex) > 0,
+ comparator::translate);
+ }
+
+ @Override
+ PrimitiveIterator.OfInt ltEq(ColumnIndexBase<?>.ValueComparator comparator) {
+ return IndexIterator.filterTranslate(comparator.arrayLength(),
+ arrayIndex -> comparator.compareValueToMin(arrayIndex) >= 0,
+ comparator::translate);
+ }
+
+ @Override
+ PrimitiveIterator.OfInt notEq(ColumnIndexBase<?>.ValueComparator comparator) {
+ return IndexIterator.filterTranslate(comparator.arrayLength(),
+ arrayIndex -> comparator.compareValueToMin(arrayIndex) != 0 || comparator.compareValueToMax(arrayIndex) != 0,
+ comparator::translate);
+ }
+ },
+ ASCENDING {
+ @Override
+ OfInt eq(ColumnIndexBase<?>.ValueComparator comparator) {
+ Bounds bounds = findBounds(comparator);
+ if (bounds == null) {
+ return IndexIterator.EMPTY;
+ }
+ return IndexIterator.rangeTranslate(bounds.lower, bounds.upper, comparator::translate);
+ }
+
+ @Override
+ OfInt gt(ColumnIndexBase<?>.ValueComparator comparator) {
+ int length = comparator.arrayLength();
+ int left = 0;
+ int right = length;
+ do {
+ int i = floorMid(left, right);
+ if (comparator.compareValueToMax(i) >= 0) {
+ left = i + 1;
+ } else {
+ right = i;
+ }
+ } while (left < right);
+ return IndexIterator.rangeTranslate(right, length - 1, comparator::translate);
+ }
+
+ @Override
+ OfInt gtEq(ColumnIndexBase<?>.ValueComparator comparator) {
+ int length = comparator.arrayLength();
+ int left = 0;
+ int right = length;
+ do {
+ int i = floorMid(left, right);
+ if (comparator.compareValueToMax(i) > 0) {
+ left = i + 1;
+ } else {
+ right = i;
+ }
+ } while (left < right);
+ return IndexIterator.rangeTranslate(right, length - 1, comparator::translate);
+ }
+
+ @Override
+ OfInt lt(ColumnIndexBase<?>.ValueComparator comparator) {
+ int length = comparator.arrayLength();
+ int left = -1;
+ int right = length - 1;
+ do {
+ int i = ceilingMid(left, right);
+ if (comparator.compareValueToMin(i) <= 0) {
+ right = i - 1;
+ } else {
+ left = i;
+ }
+ } while (left < right);
+ return IndexIterator.rangeTranslate(0, left, comparator::translate);
+ }
+
+ @Override
+ OfInt ltEq(ColumnIndexBase<?>.ValueComparator comparator) {
+ int length = comparator.arrayLength();
+ int left = -1;
+ int right = length - 1;
+ do {
+ int i = ceilingMid(left, right);
+ if (comparator.compareValueToMin(i) < 0) {
+ right = i - 1;
+ } else {
+ left = i;
+ }
+ } while (left < right);
+ return IndexIterator.rangeTranslate(0, left, comparator::translate);
+ }
+
+ @Override
+ OfInt notEq(ColumnIndexBase<?>.ValueComparator comparator) {
+ Bounds bounds = findBounds(comparator);
+ int length = comparator.arrayLength();
+ if (bounds == null) {
+ return IndexIterator.all(comparator);
+ }
+ return IndexIterator.filterTranslate(
+ length,
+ i -> i < bounds.lower || i > bounds.upper || comparator.compareValueToMin(i) != 0
+ || comparator.compareValueToMax(i) != 0,
+ comparator::translate);
+ }
+
+ private Bounds findBounds(ColumnIndexBase<?>.ValueComparator comparator) {
+ int length = comparator.arrayLength();
+ int lowerLeft = 0;
+ int upperLeft = 0;
+ int lowerRight = length - 1;
+ int upperRight = length - 1;
+ do {
+ if (lowerLeft > lowerRight) {
+ return null;
+ }
+ int i = floorMid(lowerLeft, lowerRight);
+ if (comparator.compareValueToMin(i) < 0) {
+ lowerRight = upperRight = i - 1;
+ } else if (comparator.compareValueToMax(i) > 0) {
+ lowerLeft = upperLeft = i + 1;
+ } else {
+ lowerRight = upperLeft = i;
+ }
+ } while (lowerLeft != lowerRight);
+ do {
+ if (upperLeft > upperRight) {
+ return null;
+ }
+ int i = ceilingMid(upperLeft, upperRight);
+ if (comparator.compareValueToMin(i) < 0) {
+ upperRight = i - 1;
+ } else if (comparator.compareValueToMax(i) > 0) {
+ upperLeft = i + 1;
+ } else {
+ upperLeft = i;
+ }
+ } while (upperLeft != upperRight);
+ return new Bounds(lowerLeft, upperRight);
+ }
+ },
+ DESCENDING {
+ @Override
+ OfInt eq(ColumnIndexBase<?>.ValueComparator comparator) {
+ Bounds bounds = findBounds(comparator);
+ if (bounds == null) {
+ return IndexIterator.EMPTY;
+ }
+ return IndexIterator.rangeTranslate(bounds.lower, bounds.upper, comparator::translate);
+ }
+
+ @Override
+ OfInt gt(ColumnIndexBase<?>.ValueComparator comparator) {
+ int length = comparator.arrayLength();
+ int left = -1;
+ int right = length - 1;
+ do {
+ int i = ceilingMid(left, right);
+ if (comparator.compareValueToMax(i) >= 0) {
+ right = i - 1;
+ } else {
+ left = i;
+ }
+ } while (left < right);
+ return IndexIterator.rangeTranslate(0, left, comparator::translate);
+ }
+
+ @Override
+ OfInt gtEq(ColumnIndexBase<?>.ValueComparator comparator) {
+ int length = comparator.arrayLength();
+ int left = -1;
+ int right = length - 1;
+ do {
+ int i = ceilingMid(left, right);
+ if (comparator.compareValueToMax(i) > 0) {
+ right = i - 1;
+ } else {
+ left = i;
+ }
+ } while (left < right);
+ return IndexIterator.rangeTranslate(0, left, comparator::translate);
+ }
+
+ @Override
+ OfInt lt(ColumnIndexBase<?>.ValueComparator comparator) {
+ int length = comparator.arrayLength();
+ int left = 0;
+ int right = length;
+ do {
+ int i = floorMid(left, right);
+ if (comparator.compareValueToMin(i) <= 0) {
+ left = i + 1;
+ } else {
+ right = i;
+ }
+ } while (left < right);
+ return IndexIterator.rangeTranslate(right, length - 1, comparator::translate);
+ }
+
+ @Override
+ OfInt ltEq(ColumnIndexBase<?>.ValueComparator comparator) {
+ int length = comparator.arrayLength();
+ int left = 0;
+ int right = length;
+ do {
+ int i = floorMid(left, right);
+ if (comparator.compareValueToMin(i) < 0) {
+ left = i + 1;
+ } else {
+ right = i;
+ }
+ } while (left < right);
+ return IndexIterator.rangeTranslate(right, length - 1, comparator::translate);
+ }
+
+ @Override
+ OfInt notEq(ColumnIndexBase<?>.ValueComparator comparator) {
+ Bounds bounds = findBounds(comparator);
+ int length = comparator.arrayLength();
+ if (bounds == null) {
+ return IndexIterator.all(comparator);
+ }
+ return IndexIterator.filterTranslate(
+ length,
+ i -> i < bounds.lower || i > bounds.upper || comparator.compareValueToMin(i) != 0
+ || comparator.compareValueToMax(i) != 0,
+ comparator::translate);
+ }
+
+ private Bounds findBounds(ColumnIndexBase<?>.ValueComparator comparator) {
+ int length = comparator.arrayLength();
+ int lowerLeft = 0;
+ int upperLeft = 0;
+ int lowerRight = length - 1;
+ int upperRight = length - 1;
+ do {
+ if (lowerLeft > lowerRight) {
+ return null;
+ }
+ int i = floorMid(lowerLeft, lowerRight);
+ if (comparator.compareValueToMax(i) > 0) {
+ lowerRight = upperRight = i - 1;
+ } else if (comparator.compareValueToMin(i) < 0) {
+ lowerLeft = upperLeft = i + 1;
+ } else {
+ lowerRight = upperLeft = i;
+ }
+ } while (lowerLeft != lowerRight);
+ do {
+ if (upperLeft > upperRight) {
+ return null;
+ }
+ int i = ceilingMid(upperLeft, upperRight);
+ if (comparator.compareValueToMax(i) > 0) {
+ upperRight = i - 1;
+ } else if (comparator.compareValueToMin(i) < 0) {
+ upperLeft = i + 1;
+ } else {
+ upperLeft = i;
+ }
+ } while (upperLeft != upperRight);
+ return new Bounds(lowerLeft, upperRight);
+ }
+ };
+
+ private static class Bounds {
+ final int lower, upper;
+
+ Bounds(int lower, int upper) {
+ assert lower <= upper;
+ this.lower = lower;
+ this.upper = upper;
+ }
+ }
+
+ private static int floorMid(int left, int right) {
+ // Avoid the possible overflow might happen in case of (left + right) / 2
+ return left + ((right - left) / 2);
+ }
+
+ private static int ceilingMid(int left, int right) {
+ // Avoid the possible overflow might happen in case of (left + right + 1) / 2
+ return left + ((right - left + 1) / 2);
+ }
+
+ abstract PrimitiveIterator.OfInt eq(ColumnIndexBase<?>.ValueComparator comparator);
+
+ abstract PrimitiveIterator.OfInt gt(ColumnIndexBase<?>.ValueComparator comparator);
+
+ abstract PrimitiveIterator.OfInt gtEq(ColumnIndexBase<?>.ValueComparator comparator);
+
+ abstract PrimitiveIterator.OfInt lt(ColumnIndexBase<?>.ValueComparator comparator);
+
+ abstract PrimitiveIterator.OfInt ltEq(ColumnIndexBase<?>.ValueComparator comparator);
+
+ abstract PrimitiveIterator.OfInt notEq(ColumnIndexBase<?>.ValueComparator comparator);
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndex.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndex.java
new file mode 100644
index 000000000..b91a5c0d9
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndex.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.PrimitiveIterator;
+
+import org.apache.parquet.filter2.predicate.FilterPredicate.Visitor;
+import org.apache.parquet.internal.filter2.columnindex.ColumnIndexFilter;
+
+/**
+ * Column index containing min/max and null count values for the pages in a column chunk. It also implements methods of
+ * {@link Visitor} to return the indexes of the matching pages. They are used by {@link ColumnIndexFilter}.
+ *
+ * @see org.apache.parquet.format.ColumnIndex
+ */
+public interface ColumnIndex extends Visitor<PrimitiveIterator.OfInt> {
+ /**
+ * @return the boundary order of the min/max values; used for converting to the related thrift object
+ */
+ public BoundaryOrder getBoundaryOrder();
+
+ /**
+ * @return the unmodifiable list of null counts; used for converting to the related thrift object
+ */
+ public List<Long> getNullCounts();
+
+ /**
+ * @return the unmodifiable list of null pages; used for converting to the related thrift object
+ */
+ public List<Boolean> getNullPages();
+
+ /**
+ * @return the list of the min values as {@link ByteBuffer}s; used for converting to the related thrift object
+ */
+ public List<ByteBuffer> getMinValues();
+
+ /**
+ * @return the list of the max values as {@link ByteBuffer}s; used for converting to the related thrift object
+ */
+ public List<ByteBuffer> getMaxValues();
+
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java
new file mode 100644
index 000000000..b28fddee4
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java
@@ -0,0 +1,636 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import static java.util.Objects.requireNonNull;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.Formatter;
+import java.util.List;
+import java.util.Map;
+import java.util.PrimitiveIterator;
+import java.util.function.IntPredicate;
+
+import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
+import org.apache.parquet.filter2.predicate.Operators.And;
+import org.apache.parquet.filter2.predicate.Operators.Eq;
+import org.apache.parquet.filter2.predicate.Operators.Gt;
+import org.apache.parquet.filter2.predicate.Operators.GtEq;
+import org.apache.parquet.filter2.predicate.Operators.LogicalNotUserDefined;
+import org.apache.parquet.filter2.predicate.Operators.Lt;
+import org.apache.parquet.filter2.predicate.Operators.LtEq;
+import org.apache.parquet.filter2.predicate.Operators.Not;
+import org.apache.parquet.filter2.predicate.Operators.NotEq;
+import org.apache.parquet.filter2.predicate.Operators.Or;
+import org.apache.parquet.filter2.predicate.Operators.UserDefined;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.PrimitiveComparator;
+import org.apache.parquet.schema.PrimitiveStringifier;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+
+import it.unimi.dsi.fastutil.booleans.BooleanArrayList;
+import it.unimi.dsi.fastutil.booleans.BooleanList;
+import it.unimi.dsi.fastutil.booleans.BooleanLists;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
+import it.unimi.dsi.fastutil.ints.IntSet;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongList;
+import it.unimi.dsi.fastutil.longs.LongLists;
+
+/**
+ * Builder implementation to create {@link ColumnIndex} objects.
+ */
+public abstract class ColumnIndexBuilder {
+
+ static abstract class ColumnIndexBase<C> implements ColumnIndex {
+ /*
+ * A class containing the value to be compared to the min/max values. This way we only need to do the deboxing once
+ * per predicate execution instead for every comparison.
+ */
+ abstract class ValueComparator {
+ abstract int compareValueToMin(int arrayIndex);
+
+ abstract int compareValueToMax(int arrayIndex);
+
+ int arrayLength() {
+ return pageIndexes.length;
+ }
+
+ int translate(int arrayIndex) {
+ return pageIndexes[arrayIndex];
+ }
+ }
+
+ private static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.allocate(0);
+ private static final int MAX_VALUE_LENGTH_FOR_TOSTRING = 40;
+ private static final String TOSTRING_TRUNCATION_MARKER = "(...)";
+ private static final int TOSTRING_TRUNCATION_START_POS = (MAX_VALUE_LENGTH_FOR_TOSTRING
+ - TOSTRING_TRUNCATION_MARKER.length()) / 2;
+ private static final int TOSTRING_TRUNCATION_END_POS = MAX_VALUE_LENGTH_FOR_TOSTRING
+ - TOSTRING_TRUNCATION_MARKER.length() - TOSTRING_TRUNCATION_START_POS;
+ private static final String TOSTRING_MISSING_VALUE_MARKER = "<none>";
+
+ final PrimitiveStringifier stringifier;
+ final PrimitiveComparator<C> comparator;
+ private boolean[] nullPages;
+ private BoundaryOrder boundaryOrder;
+ // Storing the page index for each array index (min/max values are not stored for null-pages)
+ private int[] pageIndexes;
+ // might be null
+ private long[] nullCounts;
+
+ static String truncate(String str) {
+ if (str.length() <= MAX_VALUE_LENGTH_FOR_TOSTRING) {
+ return str;
+ }
+ return str.substring(0, TOSTRING_TRUNCATION_START_POS) + TOSTRING_TRUNCATION_MARKER
+ + str.substring(str.length() - TOSTRING_TRUNCATION_END_POS);
+ }
+
+ ColumnIndexBase(PrimitiveType type) {
+ comparator = type.comparator();
+ stringifier = type.stringifier();
+ }
+
+ @Override
+ public BoundaryOrder getBoundaryOrder() {
+ return boundaryOrder;
+ }
+
+ @Override
+ public List<Long> getNullCounts() {
+ if (nullCounts == null) {
+ return null;
+ }
+ return LongLists.unmodifiable(LongArrayList.wrap(nullCounts));
+ }
+
+ @Override
+ public List<Boolean> getNullPages() {
+ return BooleanLists.unmodifiable(BooleanArrayList.wrap(nullPages));
+ }
+
+ @Override
+ public List<ByteBuffer> getMinValues() {
+ List<ByteBuffer> list = new ArrayList<>(getPageCount());
+ int arrayIndex = 0;
+ for (int i = 0, n = getPageCount(); i < n; ++i) {
+ if (isNullPage(i)) {
+ list.add(EMPTY_BYTE_BUFFER);
+ } else {
+ list.add(getMinValueAsBytes(arrayIndex++));
+ }
+ }
+ return list;
+ }
+
+ @Override
+ public List<ByteBuffer> getMaxValues() {
+ List<ByteBuffer> list = new ArrayList<>(getPageCount());
+ int arrayIndex = 0;
+ for (int i = 0, n = getPageCount(); i < n; ++i) {
+ if (isNullPage(i)) {
+ list.add(EMPTY_BYTE_BUFFER);
+ } else {
+ list.add(getMaxValueAsBytes(arrayIndex++));
+ }
+ }
+ return list;
+ }
+
+ @Override
+ public String toString() {
+ try (Formatter formatter = new Formatter()) {
+ formatter.format("Boudary order: %s\n", boundaryOrder);
+ String minMaxPart = " %-" + MAX_VALUE_LENGTH_FOR_TOSTRING + "s %-" + MAX_VALUE_LENGTH_FOR_TOSTRING + "s\n";
+ formatter.format("%-10s %20s" + minMaxPart, "", "null count", "min", "max");
+ String format = "page-%-5d %20s" + minMaxPart;
+ int arrayIndex = 0;
+ for (int i = 0, n = nullPages.length; i < n; ++i) {
+ String nullCount = nullCounts == null ? TOSTRING_MISSING_VALUE_MARKER : Long.toString(nullCounts[i]);
+ String min, max;
+ if (nullPages[i]) {
+ min = max = TOSTRING_MISSING_VALUE_MARKER;
+ } else {
+ min = truncate(getMinValueAsString(arrayIndex));
+ max = truncate(getMaxValueAsString(arrayIndex++));
+ }
+ formatter.format(format, i, nullCount, min, max);
+ }
+ return formatter.toString();
+ }
+ }
+
+ int getPageCount() {
+ return nullPages.length;
+ }
+
+ boolean isNullPage(int pageIndex) {
+ return nullPages[pageIndex];
+ }
+
+ /*
+ * Returns the min value for arrayIndex as a ByteBuffer. (Min values are not stored for null-pages so arrayIndex
+ * might not equal to pageIndex.)
+ */
+ abstract ByteBuffer getMinValueAsBytes(int arrayIndex);
+
+ /*
+ * Returns the max value for arrayIndex as a ByteBuffer. (Max values are not stored for null-pages so arrayIndex
+ * might not equal to pageIndex.)
+ */
+ abstract ByteBuffer getMaxValueAsBytes(int arrayIndex);
+
+ /*
+ * Returns the min value for arrayIndex as a String. (Min values are not stored for null-pages so arrayIndex might
+ * not equal to pageIndex.)
+ */
+ abstract String getMinValueAsString(int arrayIndex);
+
+ /*
+ * Returns the max value for arrayIndex as a String. (Max values are not stored for null-pages so arrayIndex might
+ * not equal to pageIndex.)
+ */
+ abstract String getMaxValueAsString(int arrayIndex);
+
+ /* Creates a Statistics object for filtering. Used for user defined predicates. */
+ abstract <T extends Comparable<T>> org.apache.parquet.filter2.predicate.Statistics<T> createStats(int arrayIndex);
+
+ /* Creates a ValueComparator object containing the specified value to be compared for min/max values */
+ abstract ValueComparator createValueComparator(Object value);
+
+ @Override
+ public PrimitiveIterator.OfInt visit(And and) {
+ throw new UnsupportedOperationException("AND shall not be used on column index directly");
+ }
+
+ @Override
+ public PrimitiveIterator.OfInt visit(Not not) {
+ throw new UnsupportedOperationException("NOT shall not be used on column index directly");
+ }
+
+ @Override
+ public PrimitiveIterator.OfInt visit(Or or) {
+ throw new UnsupportedOperationException("OR shall not be used on column index directly");
+ }
+
+ @Override
+ public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(Eq<T> eq) {
+ T value = eq.getValue();
+ if (value == null) {
+ if (nullCounts == null) {
+ // Searching for nulls so if we don't have null related statistics we have to return all pages
+ return IndexIterator.all(getPageCount());
+ } else {
+ return IndexIterator.filter(getPageCount(), pageIndex -> nullCounts[pageIndex] > 0);
+ }
+ }
+ return getBoundaryOrder().eq(createValueComparator(value));
+ }
+
+ @Override
+ public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(Gt<T> gt) {
+ return getBoundaryOrder().gt(createValueComparator(gt.getValue()));
+ }
+
+ @Override
+ public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(GtEq<T> gtEq) {
+ return getBoundaryOrder().gtEq(createValueComparator(gtEq.getValue()));
+ }
+
+ @Override
+ public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(Lt<T> lt) {
+ return getBoundaryOrder().lt(createValueComparator(lt.getValue()));
+ }
+
+ @Override
+ public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(LtEq<T> ltEq) {
+ return getBoundaryOrder().ltEq(createValueComparator(ltEq.getValue()));
+ }
+
+ @Override
+ public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(NotEq<T> notEq) {
+ T value = notEq.getValue();
+ if (value == null) {
+ return IndexIterator.filter(getPageCount(), pageIndex -> !nullPages[pageIndex]);
+ }
+
+ if (nullCounts == null) {
+ // Nulls match so if we don't have null related statistics we have to return all pages
+ return IndexIterator.all(getPageCount());
+ }
+
+ // Merging value filtering with pages containing nulls
+ IntSet matchingIndexes = new IntOpenHashSet();
+ getBoundaryOrder().notEq(createValueComparator(value))
+ .forEachRemaining((int index) -> matchingIndexes.add(index));
+ return IndexIterator.filter(getPageCount(),
+ pageIndex -> nullCounts[pageIndex] > 0 || matchingIndexes.contains(pageIndex));
+ }
+
+ @Override
+ public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> PrimitiveIterator.OfInt visit(
+ UserDefined<T, U> udp) {
+ final UserDefinedPredicate<T> predicate = udp.getUserDefinedPredicate();
+ final boolean acceptNulls = predicate.keep(null);
+
+ if (acceptNulls && nullCounts == null) {
+ // Nulls match so if we don't have null related statistics we have to return all pages
+ return IndexIterator.all(getPageCount());
+ }
+
+ return IndexIterator.filter(getPageCount(), new IntPredicate() {
+ private int arrayIndex = -1;
+
+ @Override
+ public boolean test(int pageIndex) {
+ if (isNullPage(pageIndex)) {
+ return acceptNulls;
+ } else {
+ ++arrayIndex;
+ if (acceptNulls && nullCounts[pageIndex] > 0) {
+ return true;
+ }
+ org.apache.parquet.filter2.predicate.Statistics<T> stats = createStats(arrayIndex);
+ return !predicate.canDrop(stats);
+ }
+ }
+ });
+ }
+
+ @Override
+ public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> PrimitiveIterator.OfInt visit(
+ LogicalNotUserDefined<T, U> udp) {
+ final UserDefinedPredicate<T> inversePredicate = udp.getUserDefined().getUserDefinedPredicate();
+ final boolean acceptNulls = !inversePredicate.keep(null);
+
+ if (acceptNulls && nullCounts == null) {
+ // Nulls match so if we don't have null related statistics we have to return all pages
+ return IndexIterator.all(getPageCount());
+ }
+
+ return IndexIterator.filter(getPageCount(), new IntPredicate() {
+ private int arrayIndex = -1;
+
+ @Override
+ public boolean test(int pageIndex) {
+ if (isNullPage(pageIndex)) {
+ return acceptNulls;
+ } else {
+ ++arrayIndex;
+ if (acceptNulls && nullCounts[pageIndex] > 0) {
+ return true;
+ }
+ org.apache.parquet.filter2.predicate.Statistics<T> stats = createStats(arrayIndex);
+ return !inversePredicate.inverseCanDrop(stats);
+ }
+ }
+ });
+ }
+ }
+
+ private static final ColumnIndexBuilder NO_OP_BUILDER = new ColumnIndexBuilder() {
+ @Override
+ public ColumnIndex build() {
+ return null;
+ }
+
+ @Override
+ public void add(Statistics<?> stats) {
+ }
+
+ @Override
+ void addMinMax(Object min, Object max) {
+ }
+
+ @Override
+ ColumnIndexBase<?> createColumnIndex(PrimitiveType type) {
+ return null;
+ }
+
+ @Override
+ void clearMinMax() {
+ }
+
+ @Override
+ void addMinMaxFromBytes(ByteBuffer min, ByteBuffer max) {
+ }
+
+ @Override
+ int compareMinValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return 0;
+ }
+
+ @Override
+ int compareMaxValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return 0;
+ }
+
+ @Override
+ int sizeOf(Object value) {
+ return 0;
+ }
+ };
+
+ private static final Map<PrimitiveTypeName, ColumnIndexBuilder> BUILDERS = new EnumMap<>(PrimitiveTypeName.class);
+
+ private PrimitiveType type;
+ private final BooleanList nullPages = new BooleanArrayList();
+ private final LongList nullCounts = new LongArrayList();
+ private long minMaxSize;
+ private final IntList pageIndexes = new IntArrayList();
+ private int nextPageIndex;
+
+ /**
+ * @return a no-op builder that does not collect statistics objects and therefore returns {@code null} at
+ * {@link #build()}.
+ */
+ public static ColumnIndexBuilder getNoOpBuilder() {
+ return NO_OP_BUILDER;
+ }
+
+ /**
+ * @param type
+ * the type this builder is to be created for
+ * @param truncateLength
+ * the length to be used for truncating binary values if possible
+ * @return a {@link ColumnIndexBuilder} instance to be used for creating {@link ColumnIndex} objects
+ */
+ public static ColumnIndexBuilder getBuilder(PrimitiveType type, int truncateLength) {
+ ColumnIndexBuilder builder = createNewBuilder(type, truncateLength);
+ builder.type = type;
+ return builder;
+ }
+
+ private static ColumnIndexBuilder createNewBuilder(PrimitiveType type, int truncateLength) {
+ switch (type.getPrimitiveTypeName()) {
+ case BINARY:
+ case FIXED_LEN_BYTE_ARRAY:
+ case INT96:
+ return new BinaryColumnIndexBuilder(type, truncateLength);
+ case BOOLEAN:
+ return new BooleanColumnIndexBuilder();
+ case DOUBLE:
+ return new DoubleColumnIndexBuilder();
+ case FLOAT:
+ return new FloatColumnIndexBuilder();
+ case INT32:
+ return new IntColumnIndexBuilder();
+ case INT64:
+ return new LongColumnIndexBuilder();
+ default:
+ throw new IllegalArgumentException("Unsupported type for column index: " + type);
+ }
+ }
+
+ /**
+ * @param type
+ * the primitive type
+ * @param boundaryOrder
+ * the boundary order of the min/max values
+ * @param nullPages
+ * the null pages (one boolean value for each page that signifies whether the page consists of nulls
+ * entirely)
+ * @param nullCounts
+ * the number of null values for each page
+ * @param minValues
+ * the min values for each page
+ * @param maxValues
+ * the max values for each page
+ * @return the newly created {@link ColumnIndex} object based on the specified arguments
+ */
+ public static ColumnIndex build(
+ PrimitiveType type,
+ BoundaryOrder boundaryOrder,
+ List<Boolean> nullPages,
+ List<Long> nullCounts,
+ List<ByteBuffer> minValues,
+ List<ByteBuffer> maxValues) {
+
+ PrimitiveTypeName typeName = type.getPrimitiveTypeName();
+ ColumnIndexBuilder builder = BUILDERS.get(typeName);
+ if (builder == null) {
+ builder = createNewBuilder(type, Integer.MAX_VALUE);
+ BUILDERS.put(typeName, builder);
+ }
+
+ builder.fill(nullPages, nullCounts, minValues, maxValues);
+ ColumnIndexBase<?> columnIndex = builder.build(type);
+ columnIndex.boundaryOrder = requireNonNull(boundaryOrder);
+ return columnIndex;
+ }
+
+ ColumnIndexBuilder() {
+ // Shall be able to be created inside this package only
+ }
+
+ /**
+ * Adds the data from the specified statistics to this builder
+ *
+ * @param stats
+ * the statistics to be added
+ */
+ public void add(Statistics<?> stats) {
+ if (stats.hasNonNullValue()) {
+ nullPages.add(false);
+ Object min = stats.genericGetMin();
+ Object max = stats.genericGetMax();
+ addMinMax(min, max);
+ pageIndexes.add(nextPageIndex);
+ minMaxSize += sizeOf(min);
+ minMaxSize += sizeOf(max);
+ } else {
+ nullPages.add(true);
+ }
+ nullCounts.add(stats.getNumNulls());
+ ++nextPageIndex;
+ }
+
+ abstract void addMinMaxFromBytes(ByteBuffer min, ByteBuffer max);
+
+ abstract void addMinMax(Object min, Object max);
+
+ private void fill(List<Boolean> nullPages, List<Long> nullCounts, List<ByteBuffer> minValues,
+ List<ByteBuffer> maxValues) {
+ clear();
+ int pageCount = nullPages.size();
+ if ((nullCounts != null && nullCounts.size() != pageCount) || minValues.size() != pageCount
+ || maxValues.size() != pageCount) {
+ throw new IllegalArgumentException(
+ String.format("Not all sizes are equal (nullPages:%d, nullCounts:%s, minValues:%d, maxValues:%d",
+ nullPages.size(), nullCounts == null ? "null" : nullCounts.size(), minValues.size(), maxValues.size()));
+ }
+ this.nullPages.addAll(nullPages);
+ // Nullcounts is optional in the format
+ if (nullCounts != null) {
+ this.nullCounts.addAll(nullCounts);
+ }
+
+ for (int i = 0; i < pageCount; ++i) {
+ if (!nullPages.get(i)) {
+ ByteBuffer min = minValues.get(i);
+ ByteBuffer max = maxValues.get(i);
+ addMinMaxFromBytes(min, max);
+ pageIndexes.add(i);
+ minMaxSize += min.remaining();
+ minMaxSize += max.remaining();
+ }
+ }
+ }
+
+ /**
+ * @return the newly created column index or {@code null} if the {@link ColumnIndex} would be empty
+ */
+ public ColumnIndex build() {
+ ColumnIndexBase<?> columnIndex = build(type);
+ if (columnIndex == null) {
+ return null;
+ }
+ columnIndex.boundaryOrder = calculateBoundaryOrder(type.comparator());
+ return columnIndex;
+ }
+
+ private ColumnIndexBase<?> build(PrimitiveType type) {
+ if (nullPages.isEmpty()) {
+ return null;
+ }
+ ColumnIndexBase<?> columnIndex = createColumnIndex(type);
+ if (columnIndex == null) {
+ // Might happen if the specialized builder discovers invalid min/max values
+ return null;
+ }
+ columnIndex.nullPages = nullPages.toBooleanArray();
+ // Null counts is optional so keep it null if the builder has no values
+ if (!nullCounts.isEmpty()) {
+ columnIndex.nullCounts = nullCounts.toLongArray();
+ }
+ columnIndex.pageIndexes = pageIndexes.toIntArray();
+
+ return columnIndex;
+ }
+
+ private BoundaryOrder calculateBoundaryOrder(PrimitiveComparator<Binary> comparator) {
+ if (isAscending(comparator)) {
+ return BoundaryOrder.ASCENDING;
+ } else if (isDescending(comparator)) {
+ return BoundaryOrder.DESCENDING;
+ } else {
+ return BoundaryOrder.UNORDERED;
+ }
+ }
+
+ // min[i] <= min[i+1] && max[i] <= max[i+1]
+ private boolean isAscending(PrimitiveComparator<Binary> comparator) {
+ for (int i = 1, n = pageIndexes.size(); i < n; ++i) {
+ if (compareMinValues(comparator, i - 1, i) > 0 || compareMaxValues(comparator, i - 1, i) > 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ // min[i] >= min[i+1] && max[i] >= max[i+1]
+ private boolean isDescending(PrimitiveComparator<Binary> comparator) {
+ for (int i = 1, n = pageIndexes.size(); i < n; ++i) {
+ if (compareMinValues(comparator, i - 1, i) < 0 || compareMaxValues(comparator, i - 1, i) < 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ abstract int compareMinValues(PrimitiveComparator<Binary> comparator, int index1, int index2);
+
+ abstract int compareMaxValues(PrimitiveComparator<Binary> comparator, int index1, int index2);
+
+ private void clear() {
+ nullPages.clear();
+ nullCounts.clear();
+ clearMinMax();
+ minMaxSize = 0;
+ nextPageIndex = 0;
+ pageIndexes.clear();
+ }
+
+ abstract void clearMinMax();
+
+ abstract ColumnIndexBase<?> createColumnIndex(PrimitiveType type);
+
+ abstract int sizeOf(Object value);
+
+ /**
+ * @return the number of pages added so far to this builder
+ */
+ public int getPageCount() {
+ return nullPages.size();
+ }
+
+ /**
+ * @return the sum of size in bytes of the min/max values added so far to this builder
+ */
+ public long getMinMaxSize() {
+ return minMaxSize;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/DoubleColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/DoubleColumnIndexBuilder.java
new file mode 100644
index 000000000..074d02573
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/DoubleColumnIndexBuilder.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+
+import java.nio.ByteBuffer;
+
+import org.apache.parquet.filter2.predicate.Statistics;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.PrimitiveComparator;
+import org.apache.parquet.schema.PrimitiveType;
+
+import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
+import it.unimi.dsi.fastutil.doubles.DoubleList;
+
+class DoubleColumnIndexBuilder extends ColumnIndexBuilder {
+ private static class DoubleColumnIndex extends ColumnIndexBase<Double> {
+ private double[] minValues;
+ private double[] maxValues;
+
+ private DoubleColumnIndex(PrimitiveType type) {
+ super(type);
+ }
+
+ @Override
+ ByteBuffer getMinValueAsBytes(int pageIndex) {
+ return convert(minValues[pageIndex]);
+ }
+
+ @Override
+ ByteBuffer getMaxValueAsBytes(int pageIndex) {
+ return convert(maxValues[pageIndex]);
+ }
+
+ @Override
+ String getMinValueAsString(int pageIndex) {
+ return stringifier.stringify(minValues[pageIndex]);
+ }
+
+ @Override
+ String getMaxValueAsString(int pageIndex) {
+ return stringifier.stringify(maxValues[pageIndex]);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ <T extends Comparable<T>> Statistics<T> createStats(int arrayIndex) {
+ return (Statistics<T>) new Statistics<Double>(minValues[arrayIndex], maxValues[arrayIndex], comparator);
+ }
+
+ @Override
+ ValueComparator createValueComparator(Object value) {
+ final double v = (double) value;
+ return new ValueComparator() {
+ @Override
+ int compareValueToMin(int arrayIndex) {
+ return comparator.compare(v, minValues[arrayIndex]);
+ }
+
+ @Override
+ int compareValueToMax(int arrayIndex) {
+ return comparator.compare(v, maxValues[arrayIndex]);
+ }
+ };
+ }
+ }
+
+ private final DoubleList minValues = new DoubleArrayList();
+ private final DoubleList maxValues = new DoubleArrayList();
+ private boolean invalid;
+
+ private static double convert(ByteBuffer buffer) {
+ return buffer.order(LITTLE_ENDIAN).getDouble(0);
+ }
+
+ private static ByteBuffer convert(double value) {
+ return ByteBuffer.allocate(Double.BYTES).order(LITTLE_ENDIAN).putDouble(0, value);
+ }
+
+ @Override
+ void addMinMaxFromBytes(ByteBuffer min, ByteBuffer max) {
+ minValues.add(convert(min));
+ maxValues.add(convert(max));
+ }
+
+ @Override
+ void addMinMax(Object min, Object max) {
+ double dMin = (double) min;
+ double dMax = (double) max;
+ if (Double.isNaN(dMin) || Double.isNaN(dMax)) {
+ // Invalidate this column index in case of NaN as the sorting order of values is undefined for this case
+ invalid = true;
+ }
+
+ // Sorting order is undefined for -0.0 so let min = -0.0 and max = +0.0 to ensure that no 0.0 values are skipped
+ if (Double.compare(dMin, +0.0) == 0) {
+ dMin = -0.0;
+ }
+ if (Double.compare(dMax, -0.0) == 0) {
+ dMax = +0.0;
+ }
+
+ minValues.add(dMin);
+ maxValues.add(dMax);
+ }
+
+ @Override
+ ColumnIndexBase<Double> createColumnIndex(PrimitiveType type) {
+ if (invalid) {
+ return null;
+ }
+ DoubleColumnIndex columnIndex = new DoubleColumnIndex(type);
+ columnIndex.minValues = minValues.toDoubleArray();
+ columnIndex.maxValues = maxValues.toDoubleArray();
+ return columnIndex;
+ }
+
+ @Override
+ void clearMinMax() {
+ minValues.clear();
+ maxValues.clear();
+ }
+
+ @Override
+ int compareMinValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(minValues.get(index1), minValues.get(index2));
+ }
+
+ @Override
+ int compareMaxValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(maxValues.get(index1), maxValues.get(index2));
+ }
+
+ @Override
+ int sizeOf(Object value) {
+ return Double.BYTES;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java
new file mode 100644
index 000000000..cbcdf949d
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+
+import java.nio.ByteBuffer;
+
+import org.apache.parquet.filter2.predicate.Statistics;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.PrimitiveComparator;
+import org.apache.parquet.schema.PrimitiveType;
+
+import it.unimi.dsi.fastutil.floats.FloatArrayList;
+import it.unimi.dsi.fastutil.floats.FloatList;
+
+class FloatColumnIndexBuilder extends ColumnIndexBuilder {
+ private static class FloatColumnIndex extends ColumnIndexBase<Float> {
+ private float[] minValues;
+ private float[] maxValues;
+
+ private FloatColumnIndex(PrimitiveType type) {
+ super(type);
+ }
+
+ @Override
+ ByteBuffer getMinValueAsBytes(int pageIndex) {
+ return convert(minValues[pageIndex]);
+ }
+
+ @Override
+ ByteBuffer getMaxValueAsBytes(int pageIndex) {
+ return convert(maxValues[pageIndex]);
+ }
+
+ @Override
+ String getMinValueAsString(int pageIndex) {
+ return stringifier.stringify(minValues[pageIndex]);
+ }
+
+ @Override
+ String getMaxValueAsString(int pageIndex) {
+ return stringifier.stringify(maxValues[pageIndex]);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ <T extends Comparable<T>> Statistics<T> createStats(int arrayIndex) {
+ return (Statistics<T>) new Statistics<Float>(minValues[arrayIndex], maxValues[arrayIndex], comparator);
+ }
+
+ @Override
+ ValueComparator createValueComparator(Object value) {
+ final float v = (float) value;
+ return new ValueComparator() {
+ @Override
+ int compareValueToMin(int arrayIndex) {
+ return comparator.compare(v, minValues[arrayIndex]);
+ }
+
+ @Override
+ int compareValueToMax(int arrayIndex) {
+ return comparator.compare(v, maxValues[arrayIndex]);
+ }
+ };
+ }
+ }
+
+ private final FloatList minValues = new FloatArrayList();
+ private final FloatList maxValues = new FloatArrayList();
+ private boolean invalid;
+
+ private static float convert(ByteBuffer buffer) {
+ return buffer.order(LITTLE_ENDIAN).getFloat(0);
+ }
+
+ private static ByteBuffer convert(float value) {
+ return ByteBuffer.allocate(Float.BYTES).order(LITTLE_ENDIAN).putFloat(0, value);
+ }
+
+ @Override
+ void addMinMaxFromBytes(ByteBuffer min, ByteBuffer max) {
+ minValues.add(convert(min));
+ maxValues.add(convert(max));
+ }
+
+ @Override
+ void addMinMax(Object min, Object max) {
+ float fMin = (float) min;
+ float fMax = (float) max;
+ if (Float.isNaN(fMin) || Float.isNaN(fMax)) {
+ // Invalidate this column index in case of NaN as the sorting order of values is undefined for this case
+ invalid = true;
+ }
+
+ // Sorting order is undefined for -0.0 so let min = -0.0 and max = +0.0 to ensure that no 0.0 values are skipped
+ if (Float.compare(fMin, +0.0f) == 0) {
+ fMin = -0.0f;
+ }
+ if (Float.compare(fMax, -0.0f) == 0) {
+ fMax = +0.0f;
+ }
+
+ minValues.add(fMin);
+ maxValues.add(fMax);
+ }
+
+ @Override
+ ColumnIndexBase<Float> createColumnIndex(PrimitiveType type) {
+ if (invalid) {
+ return null;
+ }
+ FloatColumnIndex columnIndex = new FloatColumnIndex(type);
+ columnIndex.minValues = minValues.toFloatArray();
+ columnIndex.maxValues = maxValues.toFloatArray();
+ return columnIndex;
+ }
+
+ @Override
+ void clearMinMax() {
+ minValues.clear();
+ maxValues.clear();
+ }
+
+ @Override
+ int compareMinValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(minValues.get(index1), minValues.get(index2));
+ }
+
+ @Override
+ int compareMaxValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(maxValues.get(index1), maxValues.get(index2));
+ }
+
+ @Override
+ int sizeOf(Object value) {
+ return Float.BYTES;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/IndexIterator.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/IndexIterator.java
new file mode 100644
index 000000000..9eab65e5b
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/IndexIterator.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import java.util.NoSuchElementException;
+import java.util.PrimitiveIterator;
+import java.util.function.IntPredicate;
+import java.util.function.IntUnaryOperator;
+
+import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder.ColumnIndexBase;
+
+/**
+ * Iterator implementation for page indexes.
+ */
+class IndexIterator implements PrimitiveIterator.OfInt {
+ public static final PrimitiveIterator.OfInt EMPTY = new OfInt() {
+ @Override
+ public boolean hasNext() {
+ return false;
+ }
+
+ @Override
+ public int nextInt() {
+ throw new NoSuchElementException();
+ }
+ };
+ private int index;
+ private final int endIndex;
+ private final IntPredicate filter;
+ private final IntUnaryOperator translator;
+
+ static PrimitiveIterator.OfInt all(int pageCount) {
+ return new IndexIterator(0, pageCount, i -> true, i -> i);
+ }
+
+ static PrimitiveIterator.OfInt all(ColumnIndexBase<?>.ValueComparator comparator) {
+ return new IndexIterator(0, comparator.arrayLength(), i -> true, comparator::translate);
+ }
+
+ static PrimitiveIterator.OfInt filter(int pageCount, IntPredicate filter) {
+ return new IndexIterator(0, pageCount, filter, i -> i);
+ }
+
+ static PrimitiveIterator.OfInt filterTranslate(int arrayLength, IntPredicate filter, IntUnaryOperator translator) {
+ return new IndexIterator(0, arrayLength, filter, translator);
+ }
+
+ static PrimitiveIterator.OfInt rangeTranslate(int from, int to, IntUnaryOperator translator) {
+ return new IndexIterator(from, to + 1, i -> true, translator);
+ }
+
+ private IndexIterator(int startIndex, int endIndex, IntPredicate filter, IntUnaryOperator translator) {
+ this.endIndex = endIndex;
+ this.filter = filter;
+ this.translator = translator;
+ index = nextPageIndex(startIndex);
+ }
+
+ private int nextPageIndex(int startIndex) {
+ for (int i = startIndex; i < endIndex; ++i) {
+ if (filter.test(i)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return index >= 0;
+ }
+
+ @Override
+ public int nextInt() {
+ if (hasNext()) {
+ int ret = index;
+ index = nextPageIndex(index + 1);
+ return translator.applyAsInt(ret);
+ }
+ throw new NoSuchElementException();
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/IntColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/IntColumnIndexBuilder.java
new file mode 100644
index 000000000..2d19d270f
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/IntColumnIndexBuilder.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+
+import java.nio.ByteBuffer;
+
+import org.apache.parquet.filter2.predicate.Statistics;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.PrimitiveComparator;
+import org.apache.parquet.schema.PrimitiveType;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+
+class IntColumnIndexBuilder extends ColumnIndexBuilder {
+ private static class IntColumnIndex extends ColumnIndexBase<Integer> {
+ private int[] minValues;
+ private int[] maxValues;
+
+ private IntColumnIndex(PrimitiveType type) {
+ super(type);
+ }
+
+ @Override
+ ByteBuffer getMinValueAsBytes(int pageIndex) {
+ return convert(minValues[pageIndex]);
+ }
+
+ @Override
+ ByteBuffer getMaxValueAsBytes(int pageIndex) {
+ return convert(maxValues[pageIndex]);
+ }
+
+ @Override
+ String getMinValueAsString(int pageIndex) {
+ return stringifier.stringify(minValues[pageIndex]);
+ }
+
+ @Override
+ String getMaxValueAsString(int pageIndex) {
+ return stringifier.stringify(maxValues[pageIndex]);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ <T extends Comparable<T>> Statistics<T> createStats(int arrayIndex) {
+ return (Statistics<T>) new Statistics<Integer>(minValues[arrayIndex], maxValues[arrayIndex], comparator);
+ }
+
+ @Override
+ ValueComparator createValueComparator(Object value) {
+ final int v = (int) value;
+ return new ValueComparator() {
+ @Override
+ int compareValueToMin(int arrayIndex) {
+ return comparator.compare(v, minValues[arrayIndex]);
+ }
+
+ @Override
+ int compareValueToMax(int arrayIndex) {
+ return comparator.compare(v, maxValues[arrayIndex]);
+ }
+ };
+ }
+ }
+
+ private final IntList minValues = new IntArrayList();
+ private final IntList maxValues = new IntArrayList();
+
+ private static int convert(ByteBuffer buffer) {
+ return buffer.order(LITTLE_ENDIAN).getInt(0);
+ }
+
+ private static ByteBuffer convert(int value) {
+ return ByteBuffer.allocate(Integer.BYTES).order(LITTLE_ENDIAN).putInt(0, value);
+ }
+
+ @Override
+ void addMinMaxFromBytes(ByteBuffer min, ByteBuffer max) {
+ minValues.add(convert(min));
+ maxValues.add(convert(max));
+ }
+
+ @Override
+ void addMinMax(Object min, Object max) {
+ minValues.add((int) min);
+ maxValues.add((int) max);
+ }
+
+ @Override
+ ColumnIndexBase<Integer> createColumnIndex(PrimitiveType type) {
+ IntColumnIndex columnIndex = new IntColumnIndex(type);
+ columnIndex.minValues = minValues.toIntArray();
+ columnIndex.maxValues = maxValues.toIntArray();
+ return columnIndex;
+ }
+
+ @Override
+ void clearMinMax() {
+ minValues.clear();
+ maxValues.clear();
+ }
+
+ @Override
+ int compareMinValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(minValues.get(index1), minValues.get(index2));
+ }
+
+ @Override
+ int compareMaxValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(maxValues.get(index1), maxValues.get(index2));
+ }
+
+ @Override
+ int sizeOf(Object value) {
+ return Integer.BYTES;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/LongColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/LongColumnIndexBuilder.java
new file mode 100644
index 000000000..b0189b709
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/LongColumnIndexBuilder.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import static java.nio.ByteOrder.LITTLE_ENDIAN;
+
+import java.nio.ByteBuffer;
+
+import org.apache.parquet.filter2.predicate.Statistics;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.PrimitiveComparator;
+import org.apache.parquet.schema.PrimitiveType;
+
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongList;
+
+class LongColumnIndexBuilder extends ColumnIndexBuilder {
+ private static class LongColumnIndex extends ColumnIndexBase<Long> {
+ private long[] minValues;
+ private long[] maxValues;
+
+ private LongColumnIndex(PrimitiveType type) {
+ super(type);
+ }
+
+ @Override
+ ByteBuffer getMinValueAsBytes(int pageIndex) {
+ return convert(minValues[pageIndex]);
+ }
+
+ @Override
+ ByteBuffer getMaxValueAsBytes(int pageIndex) {
+ return convert(maxValues[pageIndex]);
+ }
+
+ @Override
+ String getMinValueAsString(int pageIndex) {
+ return stringifier.stringify(minValues[pageIndex]);
+ }
+
+ @Override
+ String getMaxValueAsString(int pageIndex) {
+ return stringifier.stringify(maxValues[pageIndex]);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ <T extends Comparable<T>> Statistics<T> createStats(int arrayIndex) {
+ return (Statistics<T>) new Statistics<Long>(minValues[arrayIndex], maxValues[arrayIndex], comparator);
+ }
+
+ @Override
+ ValueComparator createValueComparator(Object value) {
+ final long v = (long) value;
+ return new ValueComparator() {
+ @Override
+ int compareValueToMin(int arrayIndex) {
+ return comparator.compare(v, minValues[arrayIndex]);
+ }
+
+ @Override
+ int compareValueToMax(int arrayIndex) {
+ return comparator.compare(v, maxValues[arrayIndex]);
+ }
+ };
+ }
+ }
+
+ private final LongList minValues = new LongArrayList();
+ private final LongList maxValues = new LongArrayList();
+
+ private static long convert(ByteBuffer buffer) {
+ return buffer.order(LITTLE_ENDIAN).getLong(0);
+ }
+
+ private static ByteBuffer convert(long value) {
+ return ByteBuffer.allocate(Long.BYTES).order(LITTLE_ENDIAN).putLong(0, value);
+ }
+
+ @Override
+ void addMinMaxFromBytes(ByteBuffer min, ByteBuffer max) {
+ minValues.add(convert(min));
+ maxValues.add(convert(max));
+ }
+
+ @Override
+ void addMinMax(Object min, Object max) {
+ minValues.add((long) min);
+ maxValues.add((long) max);
+ }
+
+ @Override
+ ColumnIndexBase<Long> createColumnIndex(PrimitiveType type) {
+ LongColumnIndex columnIndex = new LongColumnIndex(type);
+ columnIndex.minValues = minValues.toLongArray();
+ columnIndex.maxValues = maxValues.toLongArray();
+ return columnIndex;
+ }
+
+ @Override
+ void clearMinMax() {
+ minValues.clear();
+ maxValues.clear();
+ }
+
+ @Override
+ int compareMinValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(minValues.get(index1), minValues.get(index2));
+ }
+
+ @Override
+ int compareMaxValues(PrimitiveComparator<Binary> comparator, int index1, int index2) {
+ return comparator.compare(maxValues.get(index1), maxValues.get(index2));
+ }
+
+ @Override
+ int sizeOf(Object value) {
+ return Long.BYTES;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/OffsetIndex.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/OffsetIndex.java
new file mode 100644
index 000000000..ba984ebc7
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/OffsetIndex.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+/**
+ * Offset index containing the offset and size of the page and the index of the first row in the page.
+ *
+ * @see org.apache.parquet.format.OffsetIndex
+ */
+public interface OffsetIndex {
+ /**
+ * @return the number of pages
+ */
+ public int getPageCount();
+
+ /**
+ * @param pageIndex
+ * the index of the page
+ * @return the offset of the page in the file
+ */
+ public long getOffset(int pageIndex);
+
+ /**
+ * @param pageIndex
+ * the index of the page
+ * @return the compressed size of the page (including page header)
+ */
+ public int getCompressedPageSize(int pageIndex);
+
+ /**
+ * @param pageIndex
+ * the index of the page
+ * @return the index of the first row in the page
+ */
+ public long getFirstRowIndex(int pageIndex);
+
+ /**
+ * @param pageIndex
+ * the index of the page
+ * @param rowGroupRowCount
+ * the total number of rows in the row-group
+ * @return the calculated index of the last row of the given page
+ */
+ public default long getLastRowIndex(int pageIndex, long rowGroupRowCount) {
+ int nextPageIndex = pageIndex + 1;
+ return (nextPageIndex >= getPageCount() ? rowGroupRowCount : getFirstRowIndex(nextPageIndex)) - 1;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/OffsetIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/OffsetIndexBuilder.java
new file mode 100644
index 000000000..e4907b548
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/OffsetIndexBuilder.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import java.util.Formatter;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongList;
+
+/**
+ * Builder implementation to create {@link OffsetIndex} objects during writing a parquet file.
+ */
+public class OffsetIndexBuilder {
+
+ private static class OffsetIndexImpl implements OffsetIndex {
+ private long[] offsets;
+ private int[] compressedPageSizes;
+ private long[] firstRowIndexes;
+
+ @Override
+ public String toString() {
+ try (Formatter formatter = new Formatter()) {
+ formatter.format("%-10s %20s %16s %20s\n", "", "offset", "compressed size", "first row index");
+ for (int i = 0, n = offsets.length; i < n; ++i) {
+ formatter.format("page-%-5d %20d %16d %20d\n", i, offsets[i], compressedPageSizes[i], firstRowIndexes[i]);
+ }
+ return formatter.toString();
+ }
+ }
+
+ @Override
+ public int getPageCount() {
+ return offsets.length;
+ }
+
+ @Override
+ public long getOffset(int pageIndex) {
+ return offsets[pageIndex];
+ }
+
+ @Override
+ public int getCompressedPageSize(int pageIndex) {
+ return compressedPageSizes[pageIndex];
+ }
+
+ @Override
+ public long getFirstRowIndex(int pageIndex) {
+ return firstRowIndexes[pageIndex];
+ }
+ }
+
+ private static final OffsetIndexBuilder NO_OP_BUILDER = new OffsetIndexBuilder() {
+ @Override
+ public void add(int compressedPageSize, long rowCount) {
+ }
+
+ @Override
+ public void add(long offset, int compressedPageSize, long rowCount) {
+ }
+ };
+
+ private final LongList offsets = new LongArrayList();
+ private final IntList compressedPageSizes = new IntArrayList();
+ private final LongList firstRowIndexes = new LongArrayList();
+ private long previousOffset;
+ private int previousPageSize;
+ private long previousRowIndex;
+ private long previousRowCount;
+
+ /**
+ * @return a no-op builder that does not collect values and therefore returns {@code null} at {@link #build(long)}
+ */
+ public static OffsetIndexBuilder getNoOpBuilder() {
+ return NO_OP_BUILDER;
+ }
+
+ /**
+ * @return an {@link OffsetIndexBuilder} instance to build an {@link OffsetIndex} object
+ */
+ public static OffsetIndexBuilder getBuilder() {
+ return new OffsetIndexBuilder();
+ }
+
+ private OffsetIndexBuilder() {
+ }
+
+ /**
+ * Adds the specified parameters to this builder. Used by the writers to building up {@link OffsetIndex} objects to be
+ * written to the Parquet file.
+ *
+ * @param compressedPageSize
+ * the size of the page (including header)
+ * @param rowCount
+ * the number of rows in the page
+ */
+ public void add(int compressedPageSize, long rowCount) {
+ add(previousOffset + previousPageSize, compressedPageSize, previousRowIndex + previousRowCount);
+ previousRowCount = rowCount;
+ }
+
+ /**
+ * Adds the specified parameters to this builder. Used by the metadata converter to building up {@link OffsetIndex}
+ * objects read from the Parquet file.
+ *
+ * @param offset
+ * the offset of the page in the file
+ * @param compressedPageSize
+ * the size of the page (including header)
+ * @param firstRowIndex
+ * the index of the first row in the page (within the row group)
+ */
+ public void add(long offset, int compressedPageSize, long firstRowIndex) {
+ previousOffset = offset;
+ offsets.add(offset);
+ previousPageSize = compressedPageSize;
+ compressedPageSizes.add(compressedPageSize);
+ previousRowIndex = firstRowIndex;
+ firstRowIndexes.add(firstRowIndex);
+ }
+
+ /**
+ * Builds the offset index. Used by the metadata converter to building up {@link OffsetIndex}
+ * objects read from the Parquet file.
+ *
+ * @return the newly created offset index or {@code null} if the {@link OffsetIndex} object would be empty
+ */
+ public OffsetIndex build() {
+ return build(0);
+ }
+
+ /**
+ * Builds the offset index. Used by the writers to building up {@link OffsetIndex} objects to be
+ * written to the Parquet file.
+ *
+ * @param firstPageOffset
+ * the actual offset in the file to be used to translate all the collected offsets
+ * @return the newly created offset index or {@code null} if the {@link OffsetIndex} object would be empty
+ */
+ public OffsetIndex build(long firstPageOffset) {
+ if (compressedPageSizes.isEmpty()) {
+ return null;
+ }
+ long[] offsets = this.offsets.toLongArray();
+ if (firstPageOffset != 0) {
+ for (int i = 0, n = offsets.length; i < n; ++i) {
+ offsets[i] += firstPageOffset;
+ }
+ }
+ OffsetIndexImpl offsetIndex = new OffsetIndexImpl();
+ offsetIndex.offsets = offsets;
+ offsetIndex.compressedPageSizes = compressedPageSizes.toIntArray();
+ offsetIndex.firstRowIndexes = firstRowIndexes.toLongArray();
+
+ return offsetIndex;
+ }
+
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/filter2/columnindex/ColumnIndexFilter.java b/parquet-column/src/main/java/org/apache/parquet/internal/filter2/columnindex/ColumnIndexFilter.java
new file mode 100644
index 000000000..fb3077e87
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/filter2/columnindex/ColumnIndexFilter.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.filter2.columnindex;
+
+import java.util.PrimitiveIterator;
+import java.util.Set;
+import java.util.function.Function;
+
+import org.apache.parquet.filter2.compat.FilterCompat;
+import org.apache.parquet.filter2.compat.FilterCompat.FilterPredicateCompat;
+import org.apache.parquet.filter2.compat.FilterCompat.NoOpFilter;
+import org.apache.parquet.filter2.compat.FilterCompat.UnboundRecordFilterCompat;
+import org.apache.parquet.filter2.predicate.FilterPredicate.Visitor;
+import org.apache.parquet.filter2.predicate.Operators.And;
+import org.apache.parquet.filter2.predicate.Operators.Column;
+import org.apache.parquet.filter2.predicate.Operators.Eq;
+import org.apache.parquet.filter2.predicate.Operators.Gt;
+import org.apache.parquet.filter2.predicate.Operators.GtEq;
+import org.apache.parquet.filter2.predicate.Operators.LogicalNotUserDefined;
+import org.apache.parquet.filter2.predicate.Operators.Lt;
+import org.apache.parquet.filter2.predicate.Operators.LtEq;
+import org.apache.parquet.filter2.predicate.Operators.Not;
+import org.apache.parquet.filter2.predicate.Operators.NotEq;
+import org.apache.parquet.filter2.predicate.Operators.Or;
+import org.apache.parquet.filter2.predicate.Operators.UserDefined;
+import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.internal.filter2.columnindex.ColumnIndexStore.MissingOffsetIndexException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Filter implementation based on column indexes.
+ * No filtering will be applied for columns where no column index is available.
+ * Offset index is required for all the columns in the projection, therefore a {@link MissingOffsetIndexException} will
+ * be thrown from any {@code visit} methods if any of the required offset indexes is missing.
+ */
+public class ColumnIndexFilter implements Visitor<RowRanges> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ColumnIndexFilter.class);
+ private final ColumnIndexStore columnIndexStore;
+ private final Set<ColumnPath> columns;
+ private final long rowCount;
+ private RowRanges allRows;
+
+ /**
+ * Calculates the row ranges containing the indexes of the rows might match the specified filter.
+ *
+ * @param filter
+ * to be used for filtering the rows
+ * @param columnIndexStore
+ * the store for providing column/offset indexes
+ * @param paths
+ * the paths of the columns used in the actual projection; a column not being part of the projection will be
+ * handled as containing {@code null} values only even if the column has values written in the file
+ * @param rowCount
+ * the total number of rows in the row-group
+ * @return the ranges of the possible matching row indexes; the returned ranges will contain all the rows if any of
+ * the required offset index is missing
+ */
+ public static RowRanges calculateRowRanges(FilterCompat.Filter filter, ColumnIndexStore columnIndexStore,
+ Set<ColumnPath> paths, long rowCount) {
+ return filter.accept(new FilterCompat.Visitor<RowRanges>() {
+ @Override
+ public RowRanges visit(FilterPredicateCompat filterPredicateCompat) {
+ try {
+ return filterPredicateCompat.getFilterPredicate()
+ .accept(new ColumnIndexFilter(columnIndexStore, paths, rowCount));
+ } catch (MissingOffsetIndexException e) {
+ LOGGER.warn("Unable to do filtering", e);
+ return RowRanges.createSingle(rowCount);
+ }
+ }
+
+ @Override
+ public RowRanges visit(UnboundRecordFilterCompat unboundRecordFilterCompat) {
+ return RowRanges.createSingle(rowCount);
+ }
+
+ @Override
+ public RowRanges visit(NoOpFilter noOpFilter) {
+ return RowRanges.createSingle(rowCount);
+ }
+ });
+ }
+
+ private ColumnIndexFilter(ColumnIndexStore columnIndexStore, Set<ColumnPath> paths, long rowCount) {
+ this.columnIndexStore = columnIndexStore;
+ this.columns = paths;
+ this.rowCount = rowCount;
+ }
+
+ private RowRanges allRows() {
+ if (allRows == null) {
+ allRows = RowRanges.createSingle(rowCount);
+ }
+ return allRows;
+ }
+
+ @Override
+ public <T extends Comparable<T>> RowRanges visit(Eq<T> eq) {
+ return applyPredicate(eq.getColumn(), ci -> ci.visit(eq), eq.getValue() == null ? allRows() : RowRanges.EMPTY);
+ }
+
+ @Override
+ public <T extends Comparable<T>> RowRanges visit(NotEq<T> notEq) {
+ return applyPredicate(notEq.getColumn(), ci -> ci.visit(notEq),
+ notEq.getValue() == null ? RowRanges.EMPTY : allRows());
+ }
+
+ @Override
+ public <T extends Comparable<T>> RowRanges visit(Lt<T> lt) {
+ return applyPredicate(lt.getColumn(), ci -> ci.visit(lt), RowRanges.EMPTY);
+ }
+
+ @Override
+ public <T extends Comparable<T>> RowRanges visit(LtEq<T> ltEq) {
+ return applyPredicate(ltEq.getColumn(), ci -> ci.visit(ltEq), RowRanges.EMPTY);
+ }
+
+ @Override
+ public <T extends Comparable<T>> RowRanges visit(Gt<T> gt) {
+ return applyPredicate(gt.getColumn(), ci -> ci.visit(gt), RowRanges.EMPTY);
+ }
+
+ @Override
+ public <T extends Comparable<T>> RowRanges visit(GtEq<T> gtEq) {
+ return applyPredicate(gtEq.getColumn(), ci -> ci.visit(gtEq), RowRanges.EMPTY);
+ }
+
+ @Override
+ public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> RowRanges visit(UserDefined<T, U> udp) {
+ return applyPredicate(udp.getColumn(), ci -> ci.visit(udp),
+ udp.getUserDefinedPredicate().keep(null) ? allRows() : RowRanges.EMPTY);
+ }
+
+ @Override
+ public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> RowRanges visit(
+ LogicalNotUserDefined<T, U> udp) {
+ return applyPredicate(udp.getUserDefined().getColumn(), ci -> ci.visit(udp),
+ udp.getUserDefined().getUserDefinedPredicate().keep(null) ? RowRanges.EMPTY : allRows());
+ }
+
+ private RowRanges applyPredicate(Column<?> column, Function<ColumnIndex, PrimitiveIterator.OfInt> func,
+ RowRanges rangesForMissingColumns) {
+ ColumnPath columnPath = column.getColumnPath();
+ if (!columns.contains(columnPath)) {
+ return rangesForMissingColumns;
+ }
+
+ OffsetIndex oi = columnIndexStore.getOffsetIndex(columnPath);
+ ColumnIndex ci = columnIndexStore.getColumnIndex(columnPath);
+ if (ci == null) {
+ LOGGER.warn("No column index for column {} is available; Unable to filter on this column", columnPath);
+ return allRows();
+ }
+
+ return RowRanges.create(rowCount, func.apply(ci), oi);
+ }
+
+ @Override
+ public RowRanges visit(And and) {
+ return RowRanges.intersection(and.getLeft().accept(this), and.getRight().accept(this));
+ }
+
+ @Override
+ public RowRanges visit(Or or) {
+ return RowRanges.union(or.getLeft().accept(this), or.getRight().accept(this));
+ }
+
+ @Override
+ public RowRanges visit(Not not) {
+ throw new IllegalArgumentException(
+ "Predicates containing a NOT must be run through LogicalInverseRewriter. " + not);
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/filter2/columnindex/ColumnIndexStore.java b/parquet-column/src/main/java/org/apache/parquet/internal/filter2/columnindex/ColumnIndexStore.java
new file mode 100644
index 000000000..c82861ac2
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/filter2/columnindex/ColumnIndexStore.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.filter2.columnindex;
+
+import org.apache.parquet.ParquetRuntimeException;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+
+/**
+ * Provides the {@link ColumnIndex} and {@link OffsetIndex} objects for a row-group.
+ */
+public interface ColumnIndexStore {
+
+ /**
+ * Exception thrown in case of an offset index is missing for any of the columns.
+ */
+ public static class MissingOffsetIndexException extends ParquetRuntimeException {
+ public MissingOffsetIndexException(ColumnPath path) {
+ super("No offset index for column " + path.toDotString() + " is available; Unable to do filtering");
+ }
+ }
+
+ /**
+ * @param column
+ * the path of the column
+ * @return the column index for the column-chunk in the row-group or {@code null} if no column index is available
+ */
+ ColumnIndex getColumnIndex(ColumnPath column);
+
+ /**
+ * @param column
+ * the path of the column
+ * @return the offset index for the column-chunk in the row-group
+ * @throws MissingOffsetIndexException
+ * if the related offset index is missing
+ */
+ OffsetIndex getOffsetIndex(ColumnPath column) throws MissingOffsetIndexException;
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/filter2/columnindex/RowRanges.java b/parquet-column/src/main/java/org/apache/parquet/internal/filter2/columnindex/RowRanges.java
new file mode 100644
index 000000000..775350790
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/filter2/columnindex/RowRanges.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.filter2.columnindex;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.PrimitiveIterator;
+import java.util.Set;
+
+import org.apache.parquet.filter2.compat.FilterCompat.Filter;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+
+/**
+ * Class representing row ranges in a row-group. These row ranges are calculated as a result of the column index based
+ * filtering. To be used iterate over the matching row indexes to be read from a row-group, retrieve the count of the
+ * matching rows or check overlapping of a row index range.
+ *
+ * @see ColumnIndexFilter#calculateRowRanges(Filter, ColumnIndexStore, Set, long)
+ */
+public class RowRanges {
+ private static class Range {
+
+ // Returns the union of the two ranges or null if there are elements between them.
+ private static Range union(Range left, Range right) {
+ if (left.from <= right.from) {
+ if (left.to + 1 >= right.from) {
+ return new Range(left.from, Math.max(left.to, right.to));
+ }
+ } else if (right.to + 1 >= left.from) {
+ return new Range(right.from, Math.max(left.to, right.to));
+ }
+ return null;
+ }
+
+ // Returns the intersection of the two ranges of null if they are not overlapped.
+ private static Range intersection(Range left, Range right) {
+ if (left.from <= right.from) {
+ if (left.to >= right.from) {
+ return new Range(right.from, Math.min(left.to, right.to));
+ }
+ } else if (right.to >= left.from) {
+ return new Range(left.from, Math.min(left.to, right.to));
+ }
+ return null;
+ }
+
+ final long from;
+ final long to;
+
+ // Creates a range of [from, to] (from and to are inclusive; empty ranges are not valid)
+ Range(long from, long to) {
+ assert from <= to;
+ this.from = from;
+ this.to = to;
+ }
+
+ long count() {
+ return to - from + 1;
+ }
+
+ boolean isBefore(Range other) {
+ return to < other.from;
+ }
+
+ boolean isAfter(Range other) {
+ return from > other.to;
+ }
+
+ @Override
+ public String toString() {
+ return "[" + from + ", " + to + ']';
+ }
+ }
+
+ static final RowRanges EMPTY = new RowRanges();
+
+ /*
+ * Creates a new RowRanges object with the single range [0, rowCount - 1].
+ */
+ static RowRanges createSingle(long rowCount) {
+ RowRanges ranges = new RowRanges();
+ ranges.add(new Range(0, rowCount - 1));
+ return ranges;
+ }
+
+ /*
+ * Creates a new RowRanges object with the following ranges.
+ * [firstRowIndex[0], lastRowIndex[0]],
+ * [firstRowIndex[1], lastRowIndex[1]],
+ * ...,
+ * [firstRowIndex[n], lastRowIndex[n]]
+ * (See OffsetIndex.getFirstRowIndex and OffsetIndex.getLastRowIndex for details.)
+ *
+ * The union of the ranges are calculated so the result ranges always contain the disjunct ranges. See union for
+ * details.
+ */
+ static RowRanges create(long rowCount, PrimitiveIterator.OfInt pageIndexes, OffsetIndex offsetIndex) {
+ RowRanges ranges = new RowRanges();
+ while (pageIndexes.hasNext()) {
+ int pageIndex = pageIndexes.nextInt();
+ ranges.add(new Range(offsetIndex.getFirstRowIndex(pageIndex), offsetIndex.getLastRowIndex(pageIndex, rowCount)));
+ }
+ return ranges;
+ }
+
+ /*
+ * Calculates the union of the two specified RowRanges object. The union of two range is calculated if there are no
+ * elements between them. Otherwise, the two disjunct ranges are stored separately.
+ * For example:
+ * [113, 241] ∪ [221, 340] = [113, 330]
+ * [113, 230] ∪ [231, 340] = [113, 340]
+ * while
+ * [113, 230] ∪ [232, 340] = [113, 230], [232, 340]
+ *
+ * The result RowRanges object will contain all the row indexes that were contained in one of the specified objects.
+ */
+ static RowRanges union(RowRanges left, RowRanges right) {
+ RowRanges result = new RowRanges();
+ Iterator<Range> it1 = left.ranges.iterator();
+ Iterator<Range> it2 = right.ranges.iterator();
+ if (it2.hasNext()) {
+ Range range2 = it2.next();
+ while (it1.hasNext()) {
+ Range range1 = it1.next();
+ if (range1.isAfter(range2)) {
+ result.add(range2);
+ range2 = range1;
+ Iterator<Range> tmp = it1;
+ it1 = it2;
+ it2 = tmp;
+ } else {
+ result.add(range1);
+ }
+ }
+ result.add(range2);
+ } else {
+ it2 = it1;
+ }
+ while (it2.hasNext()) {
+ result.add(it2.next());
+ }
+
+ return result;
+ }
+
+ /*
+ * Calculates the intersection of the two specified RowRanges object. Two ranges intersect if they have common
+ * elements otherwise the result is empty.
+ * For example:
+ * [113, 241] ∩ [221, 340] = [221, 241]
+ * while
+ * [113, 230] ∩ [231, 340] = <EMPTY>
+ *
+ * The result RowRanges object will contain all the row indexes there were contained in both of the specified objects
+ */
+ static RowRanges intersection(RowRanges left, RowRanges right) {
+ RowRanges result = new RowRanges();
+
+ int rightIndex = 0;
+ for (Range l : left.ranges) {
+ for (int i = rightIndex, n = right.ranges.size(); i < n; ++i) {
+ Range r = right.ranges.get(i);
+ if (l.isBefore(r)) {
+ break;
+ } else if (l.isAfter(r)) {
+ rightIndex = i + 1;
+ continue;
+ }
+ result.add(Range.intersection(l, r));
+ }
+ }
+
+ return result;
+ }
+
+ private final List<Range> ranges = new ArrayList<>();
+
+ private RowRanges() {
+ }
+
+ /*
+ * Adds a range to the end of the list of ranges. It maintains the disjunct ascending order(*) of the ranges by
+ * trying to union the specified range to the last ranges in the list. The specified range shall be larger(*) than
+ * the last one or might be overlapped with some of the last ones.
+ * (*) [a, b] < [c, d] if b < c
+ */
+ private void add(Range range) {
+ Range rangeToAdd = range;
+ for (int i = ranges.size() - 1; i >= 0; --i) {
+ Range last = ranges.get(i);
+ assert !last.isAfter(range);
+ Range u = Range.union(last, rangeToAdd);
+ if (u == null) {
+ break;
+ }
+ rangeToAdd = u;
+ ranges.remove(i);
+ }
+ ranges.add(rangeToAdd);
+ }
+
+ /**
+ * @return the number of rows in the ranges
+ */
+ public long rowCount() {
+ long cnt = 0;
+ for (Range range : ranges) {
+ cnt += range.count();
+ }
+ return cnt;
+ }
+
+ /**
+ * @return the ascending iterator of the row indexes contained in the ranges
+ */
+ public PrimitiveIterator.OfLong iterator() {
+ return new PrimitiveIterator.OfLong() {
+ private int currentRangeIndex = -1;
+ private Range currentRange;
+ private long next = findNext();
+
+ private long findNext() {
+ if (currentRange == null || next + 1 > currentRange.to) {
+ if (currentRangeIndex + 1 < ranges.size()) {
+ currentRange = ranges.get(++currentRangeIndex);
+ next = currentRange.from;
+ } else {
+ return -1;
+ }
+ } else {
+ ++next;
+ }
+ return next;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return next >= 0;
+ }
+
+ @Override
+ public long nextLong() {
+ long ret = next;
+ if (ret < 0) {
+ throw new NoSuchElementException();
+ }
+ next = findNext();
+ return ret;
+ }
+ };
+ }
+
+ /**
+ * @param from
+ * the first row of the range to be checked for connection
+ * @param to
+ * the last row of the range to be checked for connection
+ * @return {@code true} if the specified range is overlapping (have common elements) with one of the ranges
+ */
+ public boolean isOverlapping(long from, long to) {
+ return Collections.binarySearch(ranges, new Range(from, to),
+ (r1, r2) -> r1.isBefore(r2) ? -1 : r1.isAfter(r2) ? 1 : 0) >= 0;
+ }
+
+ @Override
+ public String toString() {
+ return ranges.toString();
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java b/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java
index 6046a3931..c1b7d99fd 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java
@@ -32,6 +32,16 @@
import static java.util.Optional.empty;
import static org.apache.parquet.schema.ColumnOrder.ColumnOrderName.TYPE_DEFINED_ORDER;
import static org.apache.parquet.schema.ColumnOrder.ColumnOrderName.UNDEFINED;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_MICROS_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_MICROS_UTC_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_MILLIS_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_MILLIS_UTC_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_NANOS_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_NANOS_UTC_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIME_NANOS_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIME_NANOS_UTC_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIME_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIME_UTC_STRINGIFIER;
public abstract class LogicalTypeAnnotation {
enum LogicalTypeToken {
@@ -520,7 +530,8 @@ PrimitiveStringifier valueStringifier(PrimitiveType primitiveType) {
public enum TimeUnit {
MILLIS,
- MICROS
+ MICROS,
+ NANOS
}
public static class TimeLogicalTypeAnnotation extends LogicalTypeAnnotation {
@@ -540,7 +551,7 @@ public OriginalType toOriginalType() {
case MICROS:
return OriginalType.TIME_MICROS;
default:
- throw new RuntimeException("Unknown original type for " + unit);
+ return null;
}
}
@@ -589,7 +600,15 @@ public int hashCode() {
@Override
PrimitiveStringifier valueStringifier(PrimitiveType primitiveType) {
- return PrimitiveStringifier.TIME_STRINGIFIER;
+ switch (unit) {
+ case MICROS:
+ case MILLIS:
+ return isAdjustedToUTC ? TIME_UTC_STRINGIFIER : TIME_STRINGIFIER;
+ case NANOS:
+ return isAdjustedToUTC ? TIME_NANOS_UTC_STRINGIFIER : TIME_NANOS_STRINGIFIER;
+ default:
+ return super.valueStringifier(primitiveType);
+ }
}
}
@@ -610,7 +629,7 @@ public OriginalType toOriginalType() {
case MICROS:
return OriginalType.TIMESTAMP_MICROS;
default:
- throw new RuntimeException("Unknown original type for " + unit);
+ return null;
}
}
@@ -661,9 +680,11 @@ public int hashCode() {
PrimitiveStringifier valueStringifier(PrimitiveType primitiveType) {
switch (unit) {
case MICROS:
- return PrimitiveStringifier.TIMESTAMP_MICROS_STRINGIFIER;
+ return isAdjustedToUTC ? TIMESTAMP_MICROS_UTC_STRINGIFIER : TIMESTAMP_MICROS_STRINGIFIER;
case MILLIS:
- return PrimitiveStringifier.TIMESTAMP_MILLIS_STRINGIFIER;
+ return isAdjustedToUTC ? TIMESTAMP_MILLIS_UTC_STRINGIFIER : TIMESTAMP_MILLIS_STRINGIFIER;
+ case NANOS:
+ return isAdjustedToUTC ? TIMESTAMP_NANOS_UTC_STRINGIFIER : TIMESTAMP_NANOS_STRINGIFIER;
default:
return super.valueStringifier(primitiveType);
}
@@ -697,7 +718,7 @@ public OriginalType toOriginalType() {
case 64:
return isSigned ? OriginalType.INT_64 : OriginalType.UINT_64;
default:
- throw new RuntimeException("Unknown original type " + toOriginalType());
+ return null;
}
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
index c1a9b582f..4705ad94e 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
@@ -22,14 +22,16 @@
import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-import java.text.SimpleDateFormat;
-import java.util.TimeZone;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.concurrent.TimeUnit;
import javax.naming.OperationNotSupportedException;
@@ -242,71 +244,120 @@ String stringifyNotNull(Binary value) {
};
private static class DateStringifier extends PrimitiveStringifier {
- private final SimpleDateFormat formatter;
- private static final TimeZone UTC = TimeZone.getTimeZone("utc");
+ private final DateTimeFormatter formatter;
private DateStringifier(String name, String format) {
super(name);
- formatter = new SimpleDateFormat(format);
- formatter.setTimeZone(UTC);
+ formatter = DateTimeFormatter.ofPattern(format).withZone(ZoneOffset.UTC);
}
@Override
public String stringify(int value) {
- return toFormattedString(toMillis(value));
+ return toFormattedString(getInstant(value));
}
@Override
public String stringify(long value) {
- return toFormattedString(toMillis(value));
+ return toFormattedString(getInstant(value));
}
- private String toFormattedString(long millis) {
- return formatter.format(millis);
+ private String toFormattedString(Instant instant) {
+ return formatter.format(instant);
}
- long toMillis(int value) {
+ Instant getInstant(int value) {
// throw the related unsupported exception
super.stringify(value);
- return 0;
+ return null;
}
- long toMillis(long value) {
+ Instant getInstant(long value) {
// throw the related unsupported exception
super.stringify(value);
- return 0;
+ return null;
}
}
static final PrimitiveStringifier DATE_STRINGIFIER = new DateStringifier("DATE_STRINGIFIER", "yyyy-MM-dd") {
@Override
- long toMillis(int value) {
- return TimeUnit.DAYS.toMillis(value);
+ Instant getInstant(int value) {
+ return Instant.ofEpochMilli(TimeUnit.DAYS.toMillis(value));
};
};
static final PrimitiveStringifier TIMESTAMP_MILLIS_STRINGIFIER = new DateStringifier(
"TIMESTAMP_MILLIS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSS") {
@Override
- long toMillis(long value) {
- return value;
+ Instant getInstant(long value) {
+ return Instant.ofEpochMilli(value);
}
};
static final PrimitiveStringifier TIMESTAMP_MICROS_STRINGIFIER = new DateStringifier(
- "TIMESTAMP_MICROS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSS") {
+ "TIMESTAMP_MICROS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS") {
@Override
- public String stringify(long value) {
- return super.stringify(value) + String.format("%03d", Math.abs(value % 1000));
+ Instant getInstant(long value) {
+ return Instant.ofEpochSecond(MICROSECONDS.toSeconds(value), MICROSECONDS.toNanos(value % SECONDS.toMicros(1)));
+ }
+ };
+
+ static final PrimitiveStringifier TIMESTAMP_NANOS_STRINGIFIER = new DateStringifier(
+ "TIMESTAMP_NANOS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS") {
+ @Override
+ Instant getInstant(long value) {
+ return Instant.ofEpochSecond(NANOSECONDS.toSeconds(value), NANOSECONDS.toNanos(value % SECONDS.toNanos(1)));
}
+ };
+ static final PrimitiveStringifier TIMESTAMP_MILLIS_UTC_STRINGIFIER = new DateStringifier(
+ "TIMESTAMP_MILLIS_UTC_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSZ") {
@Override
- long toMillis(long value) {
- return value / 1000;
+ Instant getInstant(long value) {
+ return Instant.ofEpochMilli(value);
}
};
- static final PrimitiveStringifier TIME_STRINGIFIER = new PrimitiveStringifier("TIME_STRINGIFIER") {
+ static final PrimitiveStringifier TIMESTAMP_MICROS_UTC_STRINGIFIER = new DateStringifier(
+ "TIMESTAMP_MICROS_UTC_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ") {
+ @Override
+ Instant getInstant(long value) {
+ return Instant.ofEpochSecond(MICROSECONDS.toSeconds(value), MICROSECONDS.toNanos(value % SECONDS.toMicros(1)));
+ }
+ };
+
+ static final PrimitiveStringifier TIMESTAMP_NANOS_UTC_STRINGIFIER = new DateStringifier(
+ "TIMESTAMP_NANOS_UTC_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSZ") {
+ @Override
+ Instant getInstant(long value) {
+ return Instant.ofEpochSecond(NANOSECONDS.toSeconds(value), NANOSECONDS.toNanos(value % SECONDS.toNanos(1)));
+ }
+ };
+
+ private abstract static class TimeStringifier extends PrimitiveStringifier {
+ private final boolean withZone;
+
+ TimeStringifier(String name, boolean withZone) {
+ super(name);
+ this.withZone = withZone;
+ }
+
+ protected String toTimeString(long duration, TimeUnit unit) {
+ String additionalFormat = (unit == MILLISECONDS ? "3d" : unit == MICROSECONDS ? "6d" : "9d");
+ String timeZone = withZone ? "+0000" : "";
+ String format = "%02d:%02d:%02d.%0" + additionalFormat + timeZone;
+ return String.format(format,
+ unit.toHours(duration),
+ convert(duration, unit, MINUTES, HOURS),
+ convert(duration, unit, SECONDS, MINUTES),
+ convert(duration, unit, unit, SECONDS));
+ }
+
+ protected long convert(long duration, TimeUnit from, TimeUnit to, TimeUnit higher) {
+ return Math.abs(to.convert(duration, from) % to.convert(1, higher));
+ }
+ }
+
+ static final PrimitiveStringifier TIME_STRINGIFIER = new TimeStringifier("TIME_STRINGIFIER", false) {
@Override
public String stringify(int millis) {
return toTimeString(millis, MILLISECONDS);
@@ -316,18 +367,31 @@ public String stringify(int millis) {
public String stringify(long micros) {
return toTimeString(micros, MICROSECONDS);
}
+ };
- private String toTimeString(long duration, TimeUnit unit) {
- String format = "%02d:%02d:%02d.%0" + (unit == MILLISECONDS ? "3d" : "6d");
- return String.format(format,
- unit.toHours(duration),
- convert(duration, unit, MINUTES, HOURS),
- convert(duration, unit, SECONDS, MINUTES),
- convert(duration, unit, unit, SECONDS));
+ static final PrimitiveStringifier TIME_NANOS_STRINGIFIER = new TimeStringifier("TIME_NANOS_STRINGIFIER", false) {
+ @Override
+ public String stringify(long nanos) {
+ return toTimeString(nanos, NANOSECONDS);
}
+ };
- private long convert(long duration, TimeUnit from, TimeUnit to, TimeUnit higher) {
- return Math.abs(to.convert(duration, from) % to.convert(1, higher));
+ static final PrimitiveStringifier TIME_UTC_STRINGIFIER = new TimeStringifier("TIME_UTC_STRINGIFIER", true) {
+ @Override
+ public String stringify(int millis) {
+ return toTimeString(millis, MILLISECONDS);
+ }
+
+ @Override
+ public String stringify(long micros) {
+ return toTimeString(micros, MICROSECONDS);
+ }
+ };
+
+ static final PrimitiveStringifier TIME_NANOS_UTC_STRINGIFIER = new TimeStringifier("TIME_NANOS_UTC_STRINGIFIER", true) {
+ @Override
+ public String stringify(long nanos) {
+ return toTimeString(nanos, NANOSECONDS);
}
};
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
index 378d6653e..a1cd73658 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
@@ -507,6 +507,7 @@ protected PrimitiveType build(String name) {
checkInt32PrimitiveType(timeLogicalType);
break;
case MICROS:
+ case NANOS:
checkInt64PrimitiveType(timeLogicalType);
break;
default:
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/impl/TestColumnReaderImpl.java b/parquet-column/src/test/java/org/apache/parquet/column/impl/TestColumnReaderImpl.java
index d2d78c43d..35fddaf0b 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/impl/TestColumnReaderImpl.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/impl/TestColumnReaderImpl.java
@@ -65,10 +65,10 @@ public void test() throws Exception {
for (int i = 0; i < rows; i++) {
columnWriterV2.write(Binary.fromString("bar" + i % 10), 0, 0);
if ((i + 1) % 1000 == 0) {
- columnWriterV2.writePage(i);
+ columnWriterV2.writePage();
}
}
- columnWriterV2.writePage(rows);
+ columnWriterV2.writePage();
columnWriterV2.finalizeColumnChunk();
List<DataPage> pages = pageWriter.getPages();
int valueCount = 0;
@@ -103,10 +103,10 @@ public void testOptional() throws Exception {
for (int i = 0; i < rows; i++) {
columnWriterV2.writeNull(0, 0);
if ((i + 1) % 1000 == 0) {
- columnWriterV2.writePage(i);
+ columnWriterV2.writePage();
}
}
- columnWriterV2.writePage(rows);
+ columnWriterV2.writePage();
columnWriterV2.finalizeColumnChunk();
List<DataPage> pages = pageWriter.getPages();
int valueCount = 0;
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/mem/TestMemColumn.java b/parquet-column/src/test/java/org/apache/parquet/column/mem/TestMemColumn.java
index c855339c5..e5db38c94 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/mem/TestMemColumn.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/mem/TestMemColumn.java
@@ -20,12 +20,10 @@
import static org.junit.Assert.assertEquals;
-import org.apache.parquet.column.ParquetProperties;
-import org.junit.Test;
-
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.ColumnReader;
import org.apache.parquet.column.ColumnWriter;
+import org.apache.parquet.column.ParquetProperties;
import org.apache.parquet.column.impl.ColumnReadStoreImpl;
import org.apache.parquet.column.impl.ColumnWriteStoreV1;
import org.apache.parquet.column.page.mem.MemPageStore;
@@ -33,6 +31,7 @@
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,6 +46,7 @@ public void testMemColumn() throws Exception {
ColumnWriteStoreV1 memColumnsStore = newColumnWriteStoreImpl(memPageStore);
ColumnWriter columnWriter = memColumnsStore.getColumnWriter(path);
columnWriter.write(42l, 0, 0);
+ memColumnsStore.endRecord();
memColumnsStore.flush();
ColumnReader columnReader = getColumnReader(memPageStore, path, schema);
@@ -85,6 +85,7 @@ public void testMemColumnBinary() throws Exception {
ColumnWriter columnWriter = memColumnsStore.getColumnWriter(path);
columnWriter.write(Binary.fromString("42"), 0, 0);
+ memColumnsStore.endRecord();
memColumnsStore.flush();
ColumnReader columnReader = getColumnReader(memPageStore, path, mt);
@@ -108,6 +109,7 @@ public void testMemColumnSeveralPages() throws Exception {
ColumnWriter columnWriter = memColumnsStore.getColumnWriter(path);
for (int i = 0; i < 2000; i++) {
columnWriter.write(42l, 0, 0);
+ memColumnsStore.endRecord();
}
memColumnsStore.flush();
@@ -136,12 +138,16 @@ public void testMemColumnSeveralPagesRepeated() throws Exception {
int r = rs[i % rs.length];
int d = ds[i % ds.length];
LOG.debug("write i: {}", i);
+ if (i != 0 && r == 0) {
+ memColumnsStore.endRecord();
+ }
if (d == 2) {
columnWriter.write((long)i, r, d);
} else {
columnWriter.writeNull(r, d);
}
}
+ memColumnsStore.endRecord();
memColumnsStore.flush();
ColumnReader columnReader = getColumnReader(memPageStore, path, mt);
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageWriter.java b/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageWriter.java
index be3a0f9cb..706b00110 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageWriter.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/page/mem/MemPageWriter.java
@@ -56,6 +56,12 @@ public void writePage(BytesInput bytesInput, int valueCount, Statistics statisti
LOG.debug("page written for {} bytes and {} records", bytesInput.size(), valueCount);
}
+ @Override
+ public void writePage(BytesInput bytesInput, int valueCount, int rowCount, Statistics<?> statistics,
+ Encoding rlEncoding, Encoding dlEncoding, Encoding valuesEncoding) throws IOException {
+ writePage(bytesInput, valueCount, statistics, rlEncoding, dlEncoding, valuesEncoding);
+ }
+
@Override
public void writePageV2(int rowCount, int nullCount, int valueCount,
BytesInput repetitionLevels, BytesInput definitionLevels,
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/bitpacking/TestBitPackingColumn.java b/parquet-column/src/test/java/org/apache/parquet/column/values/bitpacking/TestBitPackingColumn.java
index 867af2876..3ca3d0898 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/bitpacking/TestBitPackingColumn.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/bitpacking/TestBitPackingColumn.java
@@ -183,6 +183,22 @@ private void validateEncodeDecode(int bitLength, int[] vals, String expected) th
}
LOG.debug("result: {}", TestBitPacking.toString(result));
assertArrayEquals(type + " result: " + TestBitPacking.toString(result), vals, result);
+
+ // Test skipping
+ r.initFromPage(vals.length, ByteBufferInputStream.wrap(ByteBuffer.wrap(bytes)));
+ for (int i = 0; i < vals.length; i += 2) {
+ assertEquals(vals[i], r.readInteger());
+ r.skip();
+ }
+
+ // Test n-skipping
+ r.initFromPage(vals.length, ByteBufferInputStream.wrap(ByteBuffer.wrap(bytes)));
+ int skipCount;
+ for (int i = 0; i < vals.length; i += skipCount + 1) {
+ skipCount = (vals.length - i) / 2;
+ assertEquals(vals[i], r.readInteger());
+ r.skip(skipCount);
+ }
}
}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesWriterForIntegerTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesWriterForIntegerTest.java
index df99e3c74..c69e0ff9c 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesWriterForIntegerTest.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesWriterForIntegerTest.java
@@ -212,6 +212,23 @@ public void shouldSkip() throws IOException {
}
}
+ @Test
+ public void shouldSkipN() throws IOException {
+ int[] data = new int[5 * blockSize + 1];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = i * 32;
+ }
+ writeData(data);
+ reader = new DeltaBinaryPackingValuesReader();
+ reader.initFromPage(100, writer.getBytes().toInputStream());
+ int skipCount;
+ for (int i = 0; i < data.length; i += skipCount + 1) {
+ skipCount = (data.length - i) / 2;
+ assertEquals(i * 32, reader.readInteger());
+ reader.skip(skipCount);
+ }
+ }
+
@Test
public void shouldReset() throws IOException {
shouldReadWriteWhenDataIsNotAlignedWithBlock();
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesWriterForLongTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesWriterForLongTest.java
index 65ac819e8..ca12bbdb8 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesWriterForLongTest.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/delta/DeltaBinaryPackingValuesWriterForLongTest.java
@@ -211,6 +211,23 @@ public void shouldSkip() throws IOException {
}
}
+ @Test
+ public void shouldSkipN() throws IOException {
+ long[] data = new long[5 * blockSize + 1];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = i * 32;
+ }
+ writeData(data);
+ reader = new DeltaBinaryPackingValuesReader();
+ reader.initFromPage(100, writer.getBytes().toInputStream());
+ int skipCount;
+ for (int i = 0; i < data.length; i += skipCount + 1) {
+ skipCount = (data.length - i) / 2;
+ assertEquals(i * 32, reader.readLong());
+ reader.skip(skipCount);
+ }
+ }
+
@Test
public void shouldReset() throws IOException {
shouldReadWriteWhenDataIsNotAlignedWithBlock();
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/deltalengthbytearray/TestDeltaLengthByteArray.java b/parquet-column/src/test/java/org/apache/parquet/column/values/deltalengthbytearray/TestDeltaLengthByteArray.java
index d214a8898..6c974307b 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/deltalengthbytearray/TestDeltaLengthByteArray.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/deltalengthbytearray/TestDeltaLengthByteArray.java
@@ -64,6 +64,30 @@ public void testRandomStrings() throws IOException {
}
}
+ @Test
+ public void testSkipWithRandomStrings() throws IOException {
+ DeltaLengthByteArrayValuesWriter writer = getDeltaLengthByteArrayValuesWriter();
+ DeltaLengthByteArrayValuesReader reader = new DeltaLengthByteArrayValuesReader();
+
+ String[] values = Utils.getRandomStringSamples(1000, 32);
+ Utils.writeData(writer, values);
+
+ reader.initFromPage(values.length, writer.getBytes().toInputStream());
+ for (int i = 0; i < values.length; i += 2) {
+ Assert.assertEquals(Binary.fromString(values[i]), reader.readBytes());
+ reader.skip();
+ }
+
+ reader = new DeltaLengthByteArrayValuesReader();
+ reader.initFromPage(values.length, writer.getBytes().toInputStream());
+ int skipCount;
+ for (int i = 0; i < values.length; i += skipCount + 1) {
+ skipCount = (values.length - i) / 2;
+ Assert.assertEquals(Binary.fromString(values[i]), reader.readBytes());
+ reader.skip(skipCount);
+ }
+ }
+
@Test
public void testLengths() throws IOException {
DeltaLengthByteArrayValuesWriter writer = getDeltaLengthByteArrayValuesWriter();
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
index c13a3a2b8..a5a22a8db 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/deltastrings/TestDeltaByteArray.java
@@ -58,6 +58,13 @@ public void testRandomStringsWithSkip() throws Exception {
assertReadWriteWithSkip(writer, reader, randvalues);
}
+ @Test
+ public void testRandomStringsWithSkipN() throws Exception {
+ DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());
+ DeltaByteArrayReader reader = new DeltaByteArrayReader();
+ assertReadWriteWithSkipN(writer, reader, randvalues);
+ }
+
@Test
public void testLengths() throws IOException {
DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());
@@ -99,6 +106,18 @@ private void assertReadWriteWithSkip(DeltaByteArrayWriter writer, DeltaByteArray
}
}
+ private void assertReadWriteWithSkipN(DeltaByteArrayWriter writer, DeltaByteArrayReader reader, String[] vals) throws Exception {
+ Utils.writeData(writer, vals);
+
+ reader.initFromPage(vals.length, writer.getBytes().toInputStream());
+ int skipCount;
+ for (int i = 0; i < vals.length; i += skipCount + 1) {
+ skipCount = (vals.length - i) / 2;
+ Assert.assertEquals(Binary.fromString(vals[i]), reader.readBytes());
+ reader.skip(skipCount);
+ }
+ }
+
@Test
public void testWriterReset() throws Exception {
DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/dictionary/TestDictionary.java b/parquet-column/src/test/java/org/apache/parquet/column/values/dictionary/TestDictionary.java
index ef2b7215d..ba3f9034a 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/dictionary/TestDictionary.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/dictionary/TestDictionary.java
@@ -99,6 +99,47 @@ public void testBinaryDictionary() throws IOException {
checkDistinct(COUNT, bytes3, cr2, "c");
}
+ @Test
+ public void testSkipInBinaryDictionary() throws Exception {
+ ValuesWriter cw = newPlainBinaryDictionaryValuesWriter(1000, 10000);
+ writeRepeated(100, cw, "a");
+ writeDistinct(100, cw, "b");
+ assertEquals(PLAIN_DICTIONARY, cw.getEncoding());
+
+ // Test skip and skip-n with dictionary encoding
+ ByteBufferInputStream stream = cw.getBytes().toInputStream();
+ DictionaryValuesReader cr = initDicReader(cw, BINARY);
+ cr.initFromPage(200, stream);
+ for (int i = 0; i < 100; i += 2) {
+ assertEquals(Binary.fromString("a" + i % 10), cr.readBytes());
+ cr.skip();
+ }
+ int skipCount;
+ for (int i = 0; i < 100; i += skipCount + 1) {
+ skipCount = (100 - i) / 2;
+ assertEquals(Binary.fromString("b" + i), cr.readBytes());
+ cr.skip(skipCount);
+ }
+
+ // Ensure fallback
+ writeDistinct(1000, cw, "c");
+ assertEquals(PLAIN, cw.getEncoding());
+
+ // Test skip and skip-n with plain encoding (after fallback)
+ ValuesReader plainReader = new BinaryPlainValuesReader();
+ plainReader.initFromPage(1200, cw.getBytes().toInputStream());
+ plainReader.skip(200);
+ for (int i = 0; i < 100; i += 2) {
+ assertEquals("c" + i, plainReader.readBytes().toStringUsingUTF8());
+ plainReader.skip();
+ }
+ for (int i = 100; i < 1000; i += skipCount + 1) {
+ skipCount = (1000 - i) / 2;
+ assertEquals(Binary.fromString("c" + i), plainReader.readBytes());
+ plainReader.skip(skipCount);
+ }
+ }
+
@Test
public void testBinaryDictionaryFallBack() throws IOException {
int slabSize = 100;
@@ -234,6 +275,22 @@ private void roundTripLong(FallbackValuesWriter<PlainLongDictionaryValuesWriter,
for (long i = 0; i < 100; i++) {
assertEquals(i, reader.readLong());
}
+
+ // Test skip with plain encoding
+ reader.initFromPage(100, cw.getBytes().toInputStream());
+ for (int i = 0; i < 100; i += 2) {
+ assertEquals(i, reader.readLong());
+ reader.skip();
+ }
+
+ // Test skip-n with plain encoding
+ reader.initFromPage(100, cw.getBytes().toInputStream());
+ int skipCount;
+ for (int i = 0; i < 100; i += skipCount + 1) {
+ skipCount = (100 - i) / 2;
+ assertEquals(i, reader.readLong());
+ reader.skip(skipCount);
+ }
}
@Test
@@ -305,6 +362,22 @@ private void roundTripDouble(FallbackValuesWriter<PlainDoubleDictionaryValuesWri
for (double i = 0; i < 100; i++) {
assertEquals(i, reader.readDouble(), 0.00001);
}
+
+ // Test skip with plain encoding
+ reader.initFromPage(100, cw.getBytes().toInputStream());
+ for (int i = 0; i < 100; i += 2) {
+ assertEquals(i, reader.readDouble(), 0.0);
+ reader.skip();
+ }
+
+ // Test skip-n with plain encoding
+ reader.initFromPage(100, cw.getBytes().toInputStream());
+ int skipCount;
+ for (int i = 0; i < 100; i += skipCount + 1) {
+ skipCount = (100 - i) / 2;
+ assertEquals(i, reader.readDouble(), 0.0);
+ reader.skip(skipCount);
+ }
}
@Test
@@ -376,6 +449,22 @@ private void roundTripInt(FallbackValuesWriter<PlainIntegerDictionaryValuesWrite
for (int i = 0; i < 100; i++) {
assertEquals(i, reader.readInteger());
}
+
+ // Test skip with plain encoding
+ reader.initFromPage(100, cw.getBytes().toInputStream());
+ for (int i = 0; i < 100; i += 2) {
+ assertEquals(i, reader.readInteger());
+ reader.skip();
+ }
+
+ // Test skip-n with plain encoding
+ reader.initFromPage(100, cw.getBytes().toInputStream());
+ int skipCount;
+ for (int i = 0; i < 100; i += skipCount + 1) {
+ skipCount = (100 - i) / 2;
+ assertEquals(i, reader.readInteger());
+ reader.skip(skipCount);
+ }
}
@Test
@@ -447,6 +536,22 @@ private void roundTripFloat(FallbackValuesWriter<PlainFloatDictionaryValuesWrite
for (float i = 0; i < 100; i++) {
assertEquals(i, reader.readFloat(), 0.00001);
}
+
+ // Test skip with plain encoding
+ reader.initFromPage(100, cw.getBytes().toInputStream());
+ for (int i = 0; i < 100; i += 2) {
+ assertEquals(i, reader.readFloat(), 0.0f);
+ reader.skip();
+ }
+
+ // Test skip-n with plain encoding
+ reader.initFromPage(100, cw.getBytes().toInputStream());
+ int skipCount;
+ for (int i = 0; i < 100; i += skipCount + 1) {
+ skipCount = (100 - i) / 2;
+ assertEquals(i, reader.readFloat(), 0.0f);
+ reader.skip(skipCount);
+ }
}
@Test
diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java
new file mode 100644
index 000000000..c3e3d8574
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBinaryTruncator.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import static org.apache.parquet.schema.OriginalType.BSON;
+import static org.apache.parquet.schema.OriginalType.DECIMAL;
+import static org.apache.parquet.schema.OriginalType.ENUM;
+import static org.apache.parquet.schema.OriginalType.INTERVAL;
+import static org.apache.parquet.schema.OriginalType.JSON;
+import static org.apache.parquet.schema.OriginalType.UTF8;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT96;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+import java.util.Comparator;
+import java.util.Random;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.PrimitiveStringifier;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Types;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tests for {@link BinaryTruncator}
+ */
+public class TestBinaryTruncator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestBinaryTruncator.class);
+ private static final PrimitiveStringifier HEXA_STRINGIFIER = Types.required(BINARY)
+ .named("dummy_type").stringifier();
+ private static final Random RANDOM = new Random(42);
+ private static final CharsetDecoder UTF8_DECODER = StandardCharsets.UTF_8.newDecoder();
+ static {
+ UTF8_DECODER.onMalformedInput(CodingErrorAction.REPORT);
+ UTF8_DECODER.onUnmappableCharacter(CodingErrorAction.REPORT);
+ }
+
+ // The maximum values in UTF-8 for the 1, 2, 3 and 4 bytes representations
+ private static final String UTF8_1BYTE_MAX_CHAR = "\u007F";
+ private static final String UTF8_2BYTES_MAX_CHAR = "\u07FF";
+ private static final String UTF8_3BYTES_MAX_CHAR = "\uFFFF";
+ private static final String UTF8_4BYTES_MAX_CHAR = "\uDBFF\uDFFF";
+
+ @Test
+ public void testNonStringTruncate() {
+ BinaryTruncator truncator = BinaryTruncator
+ .getTruncator(Types.required(BINARY).as(DECIMAL).precision(10).scale(2).named("test_binary_decimal"));
+ assertEquals(binary(0xFF, 0xFE, 0xFD, 0xFC, 0xFB, 0xFA),
+ truncator.truncateMin(binary(0xFF, 0xFE, 0xFD, 0xFC, 0xFB, 0xFA), 2));
+ assertEquals(binary(0x01, 0x02, 0x03, 0x04, 0x05, 0x06),
+ truncator.truncateMax(binary(0x01, 0x02, 0x03, 0x04, 0x05, 0x06), 2));
+ }
+
+ @Test
+ public void testContractNonStringTypes() {
+ testTruncator(
+ Types.required(FIXED_LEN_BYTE_ARRAY).length(8).as(DECIMAL).precision(18).scale(4).named("test_fixed_decimal"),
+ false);
+ testTruncator(Types.required(FIXED_LEN_BYTE_ARRAY).length(12).as(INTERVAL).named("test_fixed_interval"), false);
+ testTruncator(Types.required(BINARY).as(DECIMAL).precision(10).scale(2).named("test_binary_decimal"), false);
+ testTruncator(Types.required(INT96).named("test_int96"), false);
+ }
+
+ @Test
+ public void testStringTruncate() {
+ BinaryTruncator truncator = BinaryTruncator.getTruncator(Types.required(BINARY).as(UTF8).named("test_utf8"));
+
+ // Truncate 1 byte characters
+ assertEquals(Binary.fromString("abc"), truncator.truncateMin(Binary.fromString("abcdef"), 3));
+ assertEquals(Binary.fromString("abd"), truncator.truncateMax(Binary.fromString("abcdef"), 3));
+
+ // Truncate 1-2 bytes characters; the target length is "inside" a UTF-8 character
+ assertEquals(Binary.fromString("árvízt"), truncator.truncateMin(Binary.fromString("árvíztűrő"), 9));
+ assertEquals(Binary.fromString("árvízu"), truncator.truncateMax(Binary.fromString("árvíztűrő"), 9));
+
+ // Truncate highest UTF-8 values -> unable to increment
+ assertEquals(
+ Binary.fromString(
+ UTF8_1BYTE_MAX_CHAR
+ + UTF8_2BYTES_MAX_CHAR),
+ truncator.truncateMin(Binary.fromString(
+ UTF8_1BYTE_MAX_CHAR
+ + UTF8_2BYTES_MAX_CHAR
+ + UTF8_3BYTES_MAX_CHAR
+ + UTF8_4BYTES_MAX_CHAR),
+ 5));
+ assertEquals(
+ Binary.fromString(
+ UTF8_1BYTE_MAX_CHAR
+ + UTF8_2BYTES_MAX_CHAR
+ + UTF8_3BYTES_MAX_CHAR
+ + UTF8_4BYTES_MAX_CHAR),
+ truncator.truncateMax(Binary.fromString(
+ UTF8_1BYTE_MAX_CHAR
+ + UTF8_2BYTES_MAX_CHAR
+ + UTF8_3BYTES_MAX_CHAR
+ + UTF8_4BYTES_MAX_CHAR),
+ 5));
+
+ // Truncate highest UTF-8 values at the end -> increment the first possible character
+ assertEquals(
+ Binary.fromString(
+ UTF8_1BYTE_MAX_CHAR
+ + UTF8_2BYTES_MAX_CHAR
+ + "b"
+ + UTF8_3BYTES_MAX_CHAR),
+ truncator.truncateMax(Binary.fromString(
+ UTF8_1BYTE_MAX_CHAR
+ + UTF8_2BYTES_MAX_CHAR
+ + "a"
+ + UTF8_3BYTES_MAX_CHAR
+ + UTF8_4BYTES_MAX_CHAR),
+ 10));
+
+ // Truncate invalid UTF-8 values -> truncate without validity check
+ assertEquals(binary(0xFF, 0xFE, 0xFD), truncator.truncateMin(binary(0xFF, 0xFE, 0xFD, 0xFC, 0xFB, 0xFA), 3));
+ assertEquals(binary(0xFF, 0xFE, 0xFE), truncator.truncateMax(binary(0xFF, 0xFE, 0xFD, 0xFC, 0xFB, 0xFA), 3));
+ assertEquals(binary(0xFF, 0xFE, 0xFE, 0x00, 0x00), truncator.truncateMax(binary(0xFF, 0xFE, 0xFD, 0xFF, 0xFF, 0xFF), 5));
+ }
+
+ @Test
+ public void testContractStringTypes() {
+ testTruncator(Types.required(BINARY).named("test_binary"), true);
+ testTruncator(Types.required(BINARY).as(UTF8).named("test_utf8"), true);
+ testTruncator(Types.required(BINARY).as(ENUM).named("test_enum"), true);
+ testTruncator(Types.required(BINARY).as(JSON).named("test_json"), true);
+ testTruncator(Types.required(BINARY).as(BSON).named("test_bson"), true);
+ testTruncator(Types.required(FIXED_LEN_BYTE_ARRAY).length(5).named("test_fixed"), true);
+ }
+
+ private void testTruncator(PrimitiveType type, boolean strict) {
+ BinaryTruncator truncator = BinaryTruncator.getTruncator(type);
+ Comparator<Binary> comparator = type.comparator();
+
+ checkContract(truncator, comparator, Binary.fromString("aaaaaaaaaa"), strict, strict);
+ checkContract(truncator, comparator, Binary.fromString("árvíztűrő tükörfúrógép"), strict, strict);
+ checkContract(truncator, comparator, Binary.fromString("aaaaaaaaaa" + UTF8_3BYTES_MAX_CHAR), strict, strict);
+ checkContract(truncator, comparator, Binary.fromString("a" + UTF8_3BYTES_MAX_CHAR + UTF8_1BYTE_MAX_CHAR), strict,
+ strict);
+
+ checkContract(truncator, comparator,
+ Binary.fromConstantByteArray(new byte[] { (byte) 0xFE, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, (byte) 0xFF }), strict,
+ strict);
+
+ // Edge case: zero length -> unable to truncate
+ checkContract(truncator, comparator, Binary.fromString(""), false, false);
+ // Edge case: containing only UTF-8 max characters -> unable to truncate for max
+ checkContract(truncator, comparator, Binary.fromString(
+ UTF8_1BYTE_MAX_CHAR +
+ UTF8_4BYTES_MAX_CHAR +
+ UTF8_3BYTES_MAX_CHAR +
+ UTF8_4BYTES_MAX_CHAR +
+ UTF8_2BYTES_MAX_CHAR +
+ UTF8_3BYTES_MAX_CHAR +
+ UTF8_3BYTES_MAX_CHAR +
+ UTF8_1BYTE_MAX_CHAR +
+ UTF8_2BYTES_MAX_CHAR +
+ UTF8_3BYTES_MAX_CHAR +
+ UTF8_4BYTES_MAX_CHAR),
+ strict, false);
+ // Edge case: non-UTF-8; max bytes -> unable to truncate for max
+ checkContract(
+ truncator, comparator,
+ binary(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF),
+ strict, false);
+ }
+
+ // Checks the contract of truncator
+ // strict means actual truncation is required and the truncated value is a valid UTF-8 string
+ private void checkContract(BinaryTruncator truncator, Comparator<Binary> comparator, Binary value, boolean strictMin,
+ boolean strictMax) {
+ int length = value.length();
+
+ // Edge cases: returning the original value if no truncation is required
+ assertSame(value, truncator.truncateMin(value, length));
+ assertSame(value, truncator.truncateMax(value, length));
+ assertSame(value, truncator.truncateMin(value, random(length + 1, length * 2 + 1)));
+ assertSame(value, truncator.truncateMax(value, random(length + 1, length * 2 + 1)));
+
+ if (length > 1) {
+ checkMinContract(truncator, comparator, value, length - 1, strictMin);
+ checkMaxContract(truncator, comparator, value, length - 1, strictMax);
+ checkMinContract(truncator, comparator, value, random(1, length - 1), strictMin);
+ checkMaxContract(truncator, comparator, value, random(1, length - 1), strictMax);
+ }
+
+ // Edge case: possible to truncate min value to 0 length if original value is not empty
+ checkMinContract(truncator, comparator, value, 0, strictMin);
+ // Edge case: impossible to truncate max value to 0 length -> returning the original value
+ assertSame(value, truncator.truncateMax(value, 0));
+ }
+
+ private void checkMinContract(BinaryTruncator truncator, Comparator<Binary> comparator, Binary value, int length,
+ boolean strict) {
+ Binary truncated = truncator.truncateMin(value, length);
+ LOG.debug("\"{}\" --truncMin({})--> \"{}\" [{}]", value.toStringUsingUTF8(), length, truncated.toStringUsingUTF8(),
+ HEXA_STRINGIFIER.stringify(truncated));
+ assertTrue("truncatedMin(value) should be <= than value", comparator.compare(truncated, value) <= 0);
+ assertFalse("length of truncateMin(value) should not be > than the length of value",
+ truncated.length() > value.length());
+ if (isValidUtf8(value)) {
+ checkValidUtf8(truncated);
+ }
+ if (strict) {
+ assertTrue("length of truncateMin(value) ahould be < than the length of value",
+ truncated.length() < value.length());
+ }
+ }
+
+ private void checkMaxContract(BinaryTruncator truncator, Comparator<Binary> comparator, Binary value, int length,
+ boolean strict) {
+ Binary truncated = truncator.truncateMax(value, length);
+ LOG.debug("\"{}\" --truncMax({})--> \"{}\" [{}]", value.toStringUsingUTF8(), length, truncated.toStringUsingUTF8(),
+ HEXA_STRINGIFIER.stringify(truncated));
+ assertTrue("truncatedMax(value) should be >= than value", comparator.compare(truncated, value) >= 0);
+ assertFalse("length of truncateMax(value) should not be > than the length of value",
+ truncated.length() > value.length());
+ if (isValidUtf8(value)) {
+ checkValidUtf8(truncated);
+ }
+ if (strict) {
+ assertTrue("length of truncateMax(value) ahould be < than the length of value",
+ truncated.length() < value.length());
+ }
+ }
+
+ private static boolean isValidUtf8(Binary binary) {
+ try {
+ UTF8_DECODER.decode(binary.toByteBuffer());
+ return true;
+ } catch (CharacterCodingException e) {
+ return false;
+ }
+ }
+
+ private static void checkValidUtf8(Binary binary) {
+ try {
+ UTF8_DECODER.decode(binary.toByteBuffer());
+ } catch (CharacterCodingException e) {
+ throw new AssertionError("Truncated value should be a valid UTF-8 string", e);
+ }
+ }
+
+ private static int random(int min, int max) {
+ return RANDOM.nextInt(max - min + 1) + min;
+ }
+
+ private static Binary binary(int... unsignedBytes) {
+ byte[] byteArray = new byte[unsignedBytes.length];
+ for (int i = 0, n = byteArray.length; i < n; ++i) {
+ int b = unsignedBytes[i];
+ assert (0xFFFFFF00 & b) == 0;
+ byteArray[i] = (byte) b;
+ }
+ return Binary.fromConstantByteArray(byteArray);
+ }
+
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBoundaryOrder.java b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBoundaryOrder.java
new file mode 100644
index 000000000..3d2a92421
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestBoundaryOrder.java
@@ -0,0 +1,487 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.PrimitiveIterator;
+import java.util.Random;
+import java.util.function.Function;
+import java.util.stream.IntStream;
+
+import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder.ColumnIndexBase;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+
+/**
+ * Tests the operator implementations in {@link BoundaryOrder}.
+ */
+public class TestBoundaryOrder {
+ private static class SpyValueComparatorBuilder extends ColumnIndexBase<Integer> {
+ class SpyValueComparator extends ValueComparator {
+ private final ColumnIndexBase<?>.ValueComparator delegate;
+ private int compareCount;
+
+ SpyValueComparator(ColumnIndexBase<?>.ValueComparator delegate) {
+ this.delegate = delegate;
+ }
+
+ int getCompareCount() {
+ return compareCount;
+ }
+
+ @Override
+ int arrayLength() {
+ return delegate.arrayLength();
+ }
+
+ @Override
+ int translate(int arrayIndex) {
+ return delegate.translate(arrayIndex);
+ }
+
+ @Override
+ int compareValueToMin(int arrayIndex) {
+ ++compareCount;
+ return delegate.compareValueToMin(arrayIndex);
+ }
+
+ @Override
+ int compareValueToMax(int arrayIndex) {
+ ++compareCount;
+ return delegate.compareValueToMax(arrayIndex);
+ }
+ }
+
+ private SpyValueComparatorBuilder() {
+ super(TYPE);
+ }
+
+ SpyValueComparator build(ColumnIndexBase<?>.ValueComparator comparator) {
+ return new SpyValueComparator(comparator);
+ }
+
+ @Override
+ ByteBuffer getMinValueAsBytes(int arrayIndex) {
+ throw new Error("Shall never be invoked");
+ }
+
+ @Override
+ ByteBuffer getMaxValueAsBytes(int arrayIndex) {
+ throw new Error("Shall never be invoked");
+ }
+
+ @Override
+ String getMinValueAsString(int arrayIndex) {
+ throw new Error("Shall never be invoked");
+ }
+
+ @Override
+ String getMaxValueAsString(int arrayIndex) {
+ throw new Error("Shall never be invoked");
+ }
+
+ @Override
+ <T extends Comparable<T>> org.apache.parquet.filter2.predicate.Statistics<T> createStats(int arrayIndex) {
+ throw new Error("Shall never be invoked");
+ }
+
+ @Override
+ ColumnIndexBase<Integer>.ValueComparator createValueComparator(Object value) {
+ throw new Error("Shall never be invoked");
+ }
+ }
+
+ private static class ExecStats {
+ private long linearTime;
+ private long binaryTime;
+ private int linearCompareCount;
+ private int binaryCompareCount;
+ private int execCount;
+
+ IntList measureLinear(Function<ColumnIndexBase<?>.ValueComparator, PrimitiveIterator.OfInt> op,
+ ColumnIndexBase<?>.ValueComparator comparator) {
+ IntList list = new IntArrayList(comparator.arrayLength());
+ SpyValueComparatorBuilder.SpyValueComparator spyComparator = SPY_COMPARATOR_BUILDER.build(comparator);
+ long start = System.nanoTime();
+ op.apply(spyComparator).forEachRemaining((int value) -> list.add(value));
+ linearTime = System.nanoTime() - start;
+ linearCompareCount += spyComparator.getCompareCount();
+ return list;
+ }
+
+ IntList measureBinary(Function<ColumnIndexBase<?>.ValueComparator, PrimitiveIterator.OfInt> op,
+ ColumnIndexBase<?>.ValueComparator comparator) {
+ IntList list = new IntArrayList(comparator.arrayLength());
+ SpyValueComparatorBuilder.SpyValueComparator spyComparator = SPY_COMPARATOR_BUILDER.build(comparator);
+ long start = System.nanoTime();
+ op.apply(spyComparator).forEachRemaining((int value) -> list.add(value));
+ binaryTime = System.nanoTime() - start;
+ binaryCompareCount += spyComparator.getCompareCount();
+ return list;
+ }
+
+ void add(ExecStats stats) {
+ linearTime += stats.linearTime;
+ linearCompareCount += stats.linearCompareCount;
+ binaryTime += stats.binaryTime;
+ binaryCompareCount += stats.binaryCompareCount;
+ ++execCount;
+ }
+
+ @Override
+ public String toString() {
+ double linearMs = linearTime / 1_000_000.0;
+ double binaryMs = binaryTime / 1_000_000.0;
+ return String.format(
+ "Linear search: %.2fms (avg: %.6fms); number of compares: %d (avg: %d) [100.00%%]%n"
+ + "Binary search: %.2fms (avg: %.6fms); number of compares: %d (avg: %d) [%.2f%%]",
+ linearMs, linearMs / execCount, linearCompareCount, linearCompareCount / execCount,
+ binaryMs, binaryMs / execCount, binaryCompareCount, binaryCompareCount / execCount,
+ 100.0 * binaryCompareCount / linearCompareCount);
+ }
+ }
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(TestBoundaryOrder.class);
+ private static final PrimitiveType TYPE = Types.required(PrimitiveTypeName.INT32).named("test_int32");
+ private static final int FROM = -15;
+ private static final int TO = 15;
+ private static final ColumnIndexBase<?> ASCENDING;
+ private static final ColumnIndexBase<?> DESCENDING;
+ private static final int SINGLE_FROM = -1;
+ private static final int SINGLE_TO = 1;
+ private static final ColumnIndexBase<?> SINGLE;
+ private static final Random RANDOM = new Random(42);
+ private static final int RAND_FROM = -2000;
+ private static final int RAND_TO = 2000;
+ private static final int RAND_COUNT = 2000;
+ private static final ColumnIndexBase<?> RAND_ASCENDING;
+ private static final ColumnIndexBase<?> RAND_DESCENDING;
+ private static final SpyValueComparatorBuilder SPY_COMPARATOR_BUILDER = new SpyValueComparatorBuilder();
+ static {
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(TYPE, Integer.MAX_VALUE);
+ builder.add(stats(FROM, -12));
+ builder.add(stats(-10, -8));
+ builder.add(stats(-8, -4));
+ builder.add(stats(-6, -4));
+ builder.add(stats(-6, -3));
+ builder.add(stats(-6, -3));
+ builder.add(stats(-6, -3));
+ builder.add(stats(0, 3));
+ builder.add(stats(3, 5));
+ builder.add(stats(3, 5));
+ builder.add(stats(5, 8));
+ builder.add(stats(10, TO));
+ ASCENDING = (ColumnIndexBase<?>) builder.build();
+
+ builder = ColumnIndexBuilder.getBuilder(TYPE, Integer.MAX_VALUE);
+ builder.add(stats(10, TO));
+ builder.add(stats(5, 8));
+ builder.add(stats(3, 5));
+ builder.add(stats(3, 5));
+ builder.add(stats(0, 3));
+ builder.add(stats(-6, -3));
+ builder.add(stats(-6, -3));
+ builder.add(stats(-6, -3));
+ builder.add(stats(-6, -4));
+ builder.add(stats(-8, -4));
+ builder.add(stats(-10, -8));
+ builder.add(stats(FROM, -12));
+ DESCENDING = (ColumnIndexBase<?>) builder.build();
+
+ builder = ColumnIndexBuilder.getBuilder(TYPE, Integer.MAX_VALUE);
+ builder.add(stats(SINGLE_FROM, SINGLE_TO));
+ SINGLE = (ColumnIndexBase<?>) builder.build();
+
+ builder = ColumnIndexBuilder.getBuilder(TYPE, Integer.MAX_VALUE);
+ for (PrimitiveIterator.OfInt it = IntStream.generate(() -> RANDOM.nextInt(RAND_TO - RAND_FROM + 1) + RAND_FROM)
+ .limit(RAND_COUNT * 2).sorted().iterator(); it.hasNext();) {
+ builder.add(stats(it.nextInt(), it.nextInt()));
+ }
+ RAND_ASCENDING = (ColumnIndexBase<?>) builder.build();
+
+ builder = ColumnIndexBuilder.getBuilder(TYPE, Integer.MAX_VALUE);
+ for (Iterator<Integer> it = IntStream.generate(() -> RANDOM.nextInt(RAND_TO - RAND_FROM + 1) + RAND_FROM)
+ .limit(RAND_COUNT * 2).mapToObj(Integer::valueOf).sorted(Collections.reverseOrder()).iterator(); it
+ .hasNext();) {
+ builder.add(stats(it.next(), it.next()));
+ }
+ RAND_DESCENDING = (ColumnIndexBase<?>) builder.build();
+ }
+
+ private static Statistics<?> stats(int min, int max) {
+ Statistics<?> stats = Statistics.createStats(TYPE);
+ stats.updateStats(min);
+ stats.updateStats(max);
+ return stats;
+ }
+
+ private static ExecStats validateOperator(String msg,
+ Function<ColumnIndexBase<?>.ValueComparator, PrimitiveIterator.OfInt> validatorOp,
+ Function<ColumnIndexBase<?>.ValueComparator, PrimitiveIterator.OfInt> actualOp,
+ ColumnIndexBase<?>.ValueComparator comparator) {
+ ExecStats stats = new ExecStats();
+
+ IntList expected = stats.measureLinear(validatorOp, comparator);
+ IntList actual = stats.measureBinary(actualOp, comparator);
+
+ Assert.assertEquals(msg, expected, actual);
+
+ return stats;
+ }
+
+ @Test
+ public void testEq() {
+ for (int i = FROM - 1; i <= TO + 1; ++i) {
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::eq,
+ BoundaryOrder.ASCENDING::eq,
+ ASCENDING.createValueComparator(i));
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::eq,
+ BoundaryOrder.DESCENDING::eq,
+ DESCENDING.createValueComparator(i));
+ }
+ for (int i = SINGLE_FROM - 1; i <= SINGLE_TO + 1; ++i) {
+ ColumnIndexBase<?>.ValueComparator singleComparator = SINGLE.createValueComparator(i);
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::eq,
+ BoundaryOrder.ASCENDING::eq,
+ singleComparator);
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::eq,
+ BoundaryOrder.DESCENDING::eq,
+ singleComparator);
+ }
+ ExecStats stats = new ExecStats();
+ for (int i = RAND_FROM - 1; i <= RAND_TO + 1; ++i) {
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::eq,
+ BoundaryOrder.ASCENDING::eq,
+ RAND_ASCENDING.createValueComparator(i)));
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::eq,
+ BoundaryOrder.DESCENDING::eq,
+ RAND_DESCENDING.createValueComparator(i)));
+ }
+ LOGGER.info("Executed eq on random data (page count: {}, values searched: {}):\n{}", RAND_COUNT,
+ RAND_TO - RAND_FROM + 2, stats);
+ }
+
+ @Test
+ public void testGt() {
+ for (int i = FROM - 1; i <= TO + 1; ++i) {
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::gt,
+ BoundaryOrder.ASCENDING::gt,
+ ASCENDING.createValueComparator(i));
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::gt,
+ BoundaryOrder.DESCENDING::gt,
+ DESCENDING.createValueComparator(i));
+ }
+ for (int i = SINGLE_FROM - 1; i <= SINGLE_TO + 1; ++i) {
+ ColumnIndexBase<?>.ValueComparator singleComparator = SINGLE.createValueComparator(i);
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::gt,
+ BoundaryOrder.ASCENDING::gt,
+ singleComparator);
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::gt,
+ BoundaryOrder.DESCENDING::gt,
+ singleComparator);
+ }
+ ExecStats stats = new ExecStats();
+ for (int i = RAND_FROM - 1; i <= RAND_TO + 1; ++i) {
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::gt,
+ BoundaryOrder.ASCENDING::gt,
+ RAND_ASCENDING.createValueComparator(i)));
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::gt,
+ BoundaryOrder.DESCENDING::gt,
+ RAND_DESCENDING.createValueComparator(i)));
+ }
+ LOGGER.info("Executed gt on random data (page count: {}, values searched: {}):\n{}", RAND_COUNT,
+ RAND_TO - RAND_FROM + 2, stats);
+ }
+
+ @Test
+ public void testGtEq() {
+ for (int i = FROM - 1; i <= TO + 1; ++i) {
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::gtEq,
+ BoundaryOrder.ASCENDING::gtEq,
+ ASCENDING.createValueComparator(i));
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::gtEq,
+ BoundaryOrder.DESCENDING::gtEq,
+ DESCENDING.createValueComparator(i));
+ }
+ for (int i = SINGLE_FROM - 1; i <= SINGLE_TO + 1; ++i) {
+ ColumnIndexBase<?>.ValueComparator singleComparator = SINGLE.createValueComparator(i);
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::gtEq,
+ BoundaryOrder.ASCENDING::gtEq,
+ singleComparator);
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::gtEq,
+ BoundaryOrder.DESCENDING::gtEq,
+ singleComparator);
+ }
+ ExecStats stats = new ExecStats();
+ for (int i = RAND_FROM - 1; i <= RAND_TO + 1; ++i) {
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::gtEq,
+ BoundaryOrder.ASCENDING::gtEq,
+ RAND_ASCENDING.createValueComparator(i)));
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::gtEq,
+ BoundaryOrder.DESCENDING::gtEq,
+ RAND_DESCENDING.createValueComparator(i)));
+ }
+ LOGGER.info("Executed gtEq on random data (page count: {}, values searched: {}):\n{}", RAND_COUNT,
+ RAND_TO - RAND_FROM + 2, stats);
+ }
+
+ @Test
+ public void testLt() {
+ for (int i = FROM - 1; i <= TO + 1; ++i) {
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::lt,
+ BoundaryOrder.ASCENDING::lt,
+ ASCENDING.createValueComparator(i));
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::lt,
+ BoundaryOrder.DESCENDING::lt,
+ DESCENDING.createValueComparator(i));
+ }
+ for (int i = SINGLE_FROM - 1; i <= SINGLE_TO + 1; ++i) {
+ ColumnIndexBase<?>.ValueComparator singleComparator = SINGLE.createValueComparator(i);
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::lt,
+ BoundaryOrder.ASCENDING::lt,
+ singleComparator);
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::lt,
+ BoundaryOrder.DESCENDING::lt,
+ singleComparator);
+ }
+ ExecStats stats = new ExecStats();
+ for (int i = RAND_FROM - 1; i <= RAND_TO + 1; ++i) {
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::lt,
+ BoundaryOrder.ASCENDING::lt,
+ RAND_ASCENDING.createValueComparator(i)));
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::lt,
+ BoundaryOrder.DESCENDING::lt,
+ RAND_DESCENDING.createValueComparator(i)));
+ }
+ LOGGER.info("Executed lt on random data (page count: {}, values searched: {}):\n{}", RAND_COUNT,
+ RAND_TO - RAND_FROM + 2, stats);
+ }
+
+ @Test
+ public void testLtEq() {
+ for (int i = FROM - 1; i <= TO + 1; ++i) {
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::ltEq,
+ BoundaryOrder.ASCENDING::ltEq,
+ ASCENDING.createValueComparator(i));
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::ltEq,
+ BoundaryOrder.DESCENDING::ltEq,
+ DESCENDING.createValueComparator(i));
+ }
+ for (int i = SINGLE_FROM - 1; i <= SINGLE_TO + 1; ++i) {
+ ColumnIndexBase<?>.ValueComparator singleComparator = SINGLE.createValueComparator(i);
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::ltEq,
+ BoundaryOrder.ASCENDING::ltEq,
+ singleComparator);
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::ltEq,
+ BoundaryOrder.DESCENDING::ltEq,
+ singleComparator);
+ }
+ ExecStats stats = new ExecStats();
+ for (int i = RAND_FROM - 1; i <= RAND_TO + 1; ++i) {
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::ltEq,
+ BoundaryOrder.ASCENDING::ltEq,
+ RAND_ASCENDING.createValueComparator(i)));
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::ltEq,
+ BoundaryOrder.DESCENDING::ltEq,
+ RAND_DESCENDING.createValueComparator(i)));
+ }
+ LOGGER.info("Executed ltEq on random data (page count: {}, values searched: {}):\n{}", RAND_COUNT,
+ RAND_TO - RAND_FROM + 2, stats);
+ }
+
+ @Test
+ public void testNotEq() {
+ for (int i = -16; i <= 16; ++i) {
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::notEq,
+ BoundaryOrder.ASCENDING::notEq,
+ ASCENDING.createValueComparator(i));
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::notEq,
+ BoundaryOrder.DESCENDING::notEq,
+ DESCENDING.createValueComparator(i));
+ }
+ for (int i = FROM - 1; i <= TO + 1; ++i) {
+ ColumnIndexBase<?>.ValueComparator singleComparator = SINGLE.createValueComparator(i);
+ validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::notEq,
+ BoundaryOrder.ASCENDING::notEq,
+ singleComparator);
+ validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::notEq,
+ BoundaryOrder.DESCENDING::notEq,
+ singleComparator);
+ }
+ ExecStats stats = new ExecStats();
+ for (int i = RAND_FROM - 1; i <= RAND_TO + 1; ++i) {
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with ASCENDING order",
+ BoundaryOrder.UNORDERED::notEq,
+ BoundaryOrder.ASCENDING::notEq,
+ RAND_ASCENDING.createValueComparator(i)));
+ stats.add(validateOperator("Mismatching page indexes for value " + i + " with DESCENDING order",
+ BoundaryOrder.UNORDERED::notEq,
+ BoundaryOrder.DESCENDING::notEq,
+ RAND_DESCENDING.createValueComparator(i)));
+ }
+ LOGGER.info("Executed notEq on random data (page count: {}, values searched: {}):\n{}", RAND_COUNT,
+ RAND_TO - RAND_FROM + 2, stats);
+ }
+
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java
new file mode 100644
index 000000000..5a3947c98
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilder.java
@@ -0,0 +1,1546 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import static java.util.Arrays.asList;
+import static org.apache.parquet.filter2.predicate.FilterApi.binaryColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.booleanColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.doubleColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.eq;
+import static org.apache.parquet.filter2.predicate.FilterApi.floatColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.gt;
+import static org.apache.parquet.filter2.predicate.FilterApi.gtEq;
+import static org.apache.parquet.filter2.predicate.FilterApi.intColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.longColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.lt;
+import static org.apache.parquet.filter2.predicate.FilterApi.ltEq;
+import static org.apache.parquet.filter2.predicate.FilterApi.notEq;
+import static org.apache.parquet.filter2.predicate.FilterApi.userDefined;
+import static org.apache.parquet.filter2.predicate.LogicalInverter.invert;
+import static org.apache.parquet.schema.OriginalType.DECIMAL;
+import static org.apache.parquet.schema.OriginalType.UINT_8;
+import static org.apache.parquet.schema.OriginalType.UTF8;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.filter2.predicate.Operators.BinaryColumn;
+import org.apache.parquet.filter2.predicate.Operators.BooleanColumn;
+import org.apache.parquet.filter2.predicate.Operators.DoubleColumn;
+import org.apache.parquet.filter2.predicate.Operators.FloatColumn;
+import org.apache.parquet.filter2.predicate.Operators.IntColumn;
+import org.apache.parquet.filter2.predicate.Operators.LongColumn;
+import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Types;
+import org.junit.Test;
+
+/**
+ * Tests for {@link ColumnIndexBuilder}.
+ */
+public class TestColumnIndexBuilder {
+
+ public static class BinaryDecimalIsNullOrZeroUdp extends UserDefinedPredicate<Binary> {
+ private static final Binary ZERO = decimalBinary("0.0");
+
+ @Override
+ public boolean keep(Binary value) {
+ return value == null || value.equals(ZERO);
+ }
+
+ @Override
+ public boolean canDrop(org.apache.parquet.filter2.predicate.Statistics<Binary> statistics) {
+ Comparator<Binary> cmp = statistics.getComparator();
+ return cmp.compare(statistics.getMin(), ZERO) > 0 || cmp.compare(statistics.getMax(), ZERO) < 0;
+ }
+
+ @Override
+ public boolean inverseCanDrop(org.apache.parquet.filter2.predicate.Statistics<Binary> statistics) {
+ Comparator<Binary> cmp = statistics.getComparator();
+ return cmp.compare(statistics.getMin(), ZERO) == 0 && cmp.compare(statistics.getMax(), ZERO) == 0;
+ }
+ }
+
+ public static class BinaryUtf8StartsWithB extends UserDefinedPredicate<Binary> {
+ private static final Binary B = stringBinary("B");
+ private static final Binary C = stringBinary("C");
+
+ @Override
+ public boolean keep(Binary value) {
+ return value != null && value.length() > 0 && value.getBytesUnsafe()[0] == 'B';
+ }
+
+ @Override
+ public boolean canDrop(org.apache.parquet.filter2.predicate.Statistics<Binary> statistics) {
+ Comparator<Binary> cmp = statistics.getComparator();
+ return cmp.compare(statistics.getMin(), C) >= 0 || cmp.compare(statistics.getMax(), B) < 0;
+ }
+
+ @Override
+ public boolean inverseCanDrop(org.apache.parquet.filter2.predicate.Statistics<Binary> statistics) {
+ Comparator<Binary> cmp = statistics.getComparator();
+ return cmp.compare(statistics.getMin(), B) >= 0 && cmp.compare(statistics.getMax(), C) < 0;
+ }
+ }
+
+ public static class BooleanIsTrueOrNull extends UserDefinedPredicate<Boolean> {
+ @Override
+ public boolean keep(Boolean value) {
+ return value == null || value;
+ }
+
+ @Override
+ public boolean canDrop(org.apache.parquet.filter2.predicate.Statistics<Boolean> statistics) {
+ return statistics.getComparator().compare(statistics.getMax(), true) != 0;
+ }
+
+ @Override
+ public boolean inverseCanDrop(org.apache.parquet.filter2.predicate.Statistics<Boolean> statistics) {
+ return statistics.getComparator().compare(statistics.getMin(), true) == 0;
+ }
+ }
+
+ public static class DoubleIsInteger extends UserDefinedPredicate<Double> {
+ @Override
+ public boolean keep(Double value) {
+ return value != null && Math.floor(value) == value;
+ }
+
+ @Override
+ public boolean canDrop(org.apache.parquet.filter2.predicate.Statistics<Double> statistics) {
+ double min = statistics.getMin();
+ double max = statistics.getMax();
+ Comparator<Double> cmp = statistics.getComparator();
+ return cmp.compare(Math.floor(min), Math.floor(max)) == 0 && cmp.compare(Math.floor(min), min) != 0
+ && cmp.compare(Math.floor(max), max) != 0;
+ }
+
+ @Override
+ public boolean inverseCanDrop(org.apache.parquet.filter2.predicate.Statistics<Double> statistics) {
+ double min = statistics.getMin();
+ double max = statistics.getMax();
+ Comparator<Double> cmp = statistics.getComparator();
+ return cmp.compare(min, max) == 0 && cmp.compare(Math.floor(min), min) == 0;
+ }
+ }
+
+ public static class FloatIsInteger extends UserDefinedPredicate<Float> {
+ private static float floor(float value) {
+ return (float) Math.floor(value);
+ }
+
+ @Override
+ public boolean keep(Float value) {
+ return value != null && Math.floor(value) == value;
+ }
+
+ @Override
+ public boolean canDrop(org.apache.parquet.filter2.predicate.Statistics<Float> statistics) {
+ float min = statistics.getMin();
+ float max = statistics.getMax();
+ Comparator<Float> cmp = statistics.getComparator();
+ return cmp.compare(floor(min), floor(max)) == 0 && cmp.compare(floor(min), min) != 0
+ && cmp.compare(floor(max), max) != 0;
+ }
+
+ @Override
+ public boolean inverseCanDrop(org.apache.parquet.filter2.predicate.Statistics<Float> statistics) {
+ float min = statistics.getMin();
+ float max = statistics.getMax();
+ Comparator<Float> cmp = statistics.getComparator();
+ return cmp.compare(min, max) == 0 && cmp.compare(floor(min), min) == 0;
+ }
+ }
+
+ public static class IntegerIsDivisableWith3 extends UserDefinedPredicate<Integer> {
+ @Override
+ public boolean keep(Integer value) {
+ return value != null && value % 3 == 0;
+ }
+
+ @Override
+ public boolean canDrop(org.apache.parquet.filter2.predicate.Statistics<Integer> statistics) {
+ int min = statistics.getMin();
+ int max = statistics.getMax();
+ return min % 3 != 0 && max % 3 != 0 && max - min < 3;
+ }
+
+ @Override
+ public boolean inverseCanDrop(org.apache.parquet.filter2.predicate.Statistics<Integer> statistics) {
+ int min = statistics.getMin();
+ int max = statistics.getMax();
+ return min == max && min % 3 == 0;
+ }
+ }
+
+ public static class LongIsDivisableWith3 extends UserDefinedPredicate<Long> {
+ @Override
+ public boolean keep(Long value) {
+ return value != null && value % 3 == 0;
+ }
+
+ @Override
+ public boolean canDrop(org.apache.parquet.filter2.predicate.Statistics<Long> statistics) {
+ long min = statistics.getMin();
+ long max = statistics.getMax();
+ return min % 3 != 0 && max % 3 != 0 && max - min < 3;
+ }
+
+ @Override
+ public boolean inverseCanDrop(org.apache.parquet.filter2.predicate.Statistics<Long> statistics) {
+ long min = statistics.getMin();
+ long max = statistics.getMax();
+ return min == max && min % 3 == 0;
+ }
+ }
+
+ @Test
+ public void testBuildBinaryDecimal() {
+ PrimitiveType type = Types.required(BINARY).as(DECIMAL).precision(12).scale(2).named("test_binary_decimal");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ assertThat(builder, instanceOf(BinaryColumnIndexBuilder.class));
+ assertNull(builder.build());
+ BinaryColumn col = binaryColumn("test_col");
+
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, decimalBinary("-0.17"), decimalBinary("1234567890.12")));
+ builder.add(sb.stats(type, decimalBinary("-234.23"), null, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, decimalBinary("-9999293.23"), decimalBinary("2348978.45")));
+ builder.add(sb.stats(type, null, null, null, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, decimalBinary("87656273")));
+ assertEquals(8, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ ColumnIndex columnIndex = builder.build();
+ assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 0, 3, 3, 0, 4, 2, 0);
+ assertCorrectNullPages(columnIndex, true, false, false, true, false, true, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(),
+ null,
+ decimalBinary("1234567890.12"),
+ decimalBinary("-234.23"),
+ null,
+ decimalBinary("2348978.45"),
+ null,
+ null,
+ decimalBinary("87656273"));
+ assertCorrectValues(columnIndex.getMinValues(),
+ null,
+ decimalBinary("-0.17"),
+ decimalBinary("-234.23"),
+ null,
+ decimalBinary("-9999293.23"),
+ null,
+ null,
+ decimalBinary("87656273"));
+ assertCorrectFiltering(columnIndex, eq(col, decimalBinary("0.0")), 1, 4);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 5, 6);
+ assertCorrectFiltering(columnIndex, notEq(col, decimalBinary("87656273")), 0, 1, 2, 3, 4, 5, 6);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 2, 4, 7);
+ assertCorrectFiltering(columnIndex, gt(col, decimalBinary("2348978.45")), 1);
+ assertCorrectFiltering(columnIndex, gtEq(col, decimalBinary("2348978.45")), 1, 4);
+ assertCorrectFiltering(columnIndex, lt(col, decimalBinary("-234.23")), 4);
+ assertCorrectFiltering(columnIndex, ltEq(col, decimalBinary("-234.23")), 2, 4);
+ assertCorrectFiltering(columnIndex, userDefined(col, BinaryDecimalIsNullOrZeroUdp.class), 0, 1, 2, 3, 4, 5, 6);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, BinaryDecimalIsNullOrZeroUdp.class)), 1, 2, 4, 7);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null, null, null));
+ builder.add(sb.stats(type, decimalBinary("-9999293.23"), decimalBinary("-234.23")));
+ builder.add(sb.stats(type, decimalBinary("-0.17"), decimalBinary("87656273")));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, decimalBinary("87656273")));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, decimalBinary("1234567890.12"), null, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ assertEquals(8, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 4, 0, 0, 2, 0, 2, 3, 3);
+ assertCorrectNullPages(columnIndex, true, false, false, true, false, true, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(),
+ null,
+ decimalBinary("-234.23"),
+ decimalBinary("87656273"),
+ null,
+ decimalBinary("87656273"),
+ null,
+ decimalBinary("1234567890.12"),
+ null);
+ assertCorrectValues(columnIndex.getMinValues(),
+ null,
+ decimalBinary("-9999293.23"),
+ decimalBinary("-0.17"),
+ null,
+ decimalBinary("87656273"),
+ null,
+ decimalBinary("1234567890.12"),
+ null);
+ assertCorrectFiltering(columnIndex, eq(col, decimalBinary("87656273")), 2, 4);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 3, 5, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, decimalBinary("87656273")), 0, 1, 2, 3, 5, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 2, 4, 6);
+ assertCorrectFiltering(columnIndex, gt(col, decimalBinary("87656273")), 6);
+ assertCorrectFiltering(columnIndex, gtEq(col, decimalBinary("87656273")), 2, 4, 6);
+ assertCorrectFiltering(columnIndex, lt(col, decimalBinary("-0.17")), 1);
+ assertCorrectFiltering(columnIndex, ltEq(col, decimalBinary("-0.17")), 1, 2);
+ assertCorrectFiltering(columnIndex, userDefined(col, BinaryDecimalIsNullOrZeroUdp.class), 0, 2, 3, 5, 6, 7);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, BinaryDecimalIsNullOrZeroUdp.class)), 1, 2, 4, 6);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, decimalBinary("1234567890.12"), null, null, null));
+ builder.add(sb.stats(type, null, null, null, null));
+ builder.add(sb.stats(type, decimalBinary("1234567890.12"), decimalBinary("87656273")));
+ builder.add(sb.stats(type, decimalBinary("987656273"), decimalBinary("-0.17")));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, decimalBinary("-234.23"), decimalBinary("-9999293.23")));
+ assertEquals(8, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 3, 2, 3, 4, 0, 0, 2, 0);
+ assertCorrectNullPages(columnIndex, true, true, false, true, false, false, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(),
+ null,
+ null,
+ decimalBinary("1234567890.12"),
+ null,
+ decimalBinary("1234567890.12"),
+ decimalBinary("987656273"),
+ null,
+ decimalBinary("-234.23"));
+ assertCorrectValues(columnIndex.getMinValues(),
+ null,
+ null,
+ decimalBinary("1234567890.12"),
+ null,
+ decimalBinary("87656273"),
+ decimalBinary("-0.17"),
+ null,
+ decimalBinary("-9999293.23"));
+ assertCorrectFiltering(columnIndex, eq(col, decimalBinary("1234567890.12")), 2, 4);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 6);
+ assertCorrectFiltering(columnIndex, notEq(col, decimalBinary("0.0")), 0, 1, 2, 3, 4, 5, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 2, 4, 5, 7);
+ assertCorrectFiltering(columnIndex, gt(col, decimalBinary("1234567890.12")));
+ assertCorrectFiltering(columnIndex, gtEq(col, decimalBinary("1234567890.12")), 2, 4);
+ assertCorrectFiltering(columnIndex, lt(col, decimalBinary("-0.17")), 7);
+ assertCorrectFiltering(columnIndex, ltEq(col, decimalBinary("-0.17")), 5, 7);
+ assertCorrectFiltering(columnIndex, userDefined(col, BinaryDecimalIsNullOrZeroUdp.class), 0, 1, 2, 3, 5, 6);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, BinaryDecimalIsNullOrZeroUdp.class)), 2, 4, 5, 7);
+ }
+
+ @Test
+ public void testBuildBinaryUtf8() {
+ PrimitiveType type = Types.required(BINARY).as(UTF8).named("test_binary_utf8");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ assertThat(builder, instanceOf(BinaryColumnIndexBuilder.class));
+ assertNull(builder.build());
+ BinaryColumn col = binaryColumn("test_col");
+
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, stringBinary("Jeltz"), stringBinary("Slartibartfast"), null, null));
+ builder.add(sb.stats(type, null, null, null, null, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, stringBinary("Beeblebrox"), stringBinary("Prefect")));
+ builder.add(sb.stats(type, stringBinary("Dent"), stringBinary("Trilian"), null));
+ builder.add(sb.stats(type, stringBinary("Beeblebrox")));
+ builder.add(sb.stats(type, null, null));
+ assertEquals(8, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ ColumnIndex columnIndex = builder.build();
+ assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 2, 5, 2, 0, 1, 0, 2);
+ assertCorrectNullPages(columnIndex, true, false, true, true, false, false, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(),
+ null,
+ stringBinary("Slartibartfast"),
+ null,
+ null,
+ stringBinary("Prefect"),
+ stringBinary("Trilian"),
+ stringBinary("Beeblebrox"),
+ null);
+ assertCorrectValues(columnIndex.getMinValues(),
+ null,
+ stringBinary("Jeltz"),
+ null,
+ null,
+ stringBinary("Beeblebrox"),
+ stringBinary("Dent"),
+ stringBinary("Beeblebrox"),
+ null);
+ assertCorrectFiltering(columnIndex, eq(col, stringBinary("Marvin")), 1, 4, 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 5, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, stringBinary("Beeblebrox")), 0, 1, 2, 3, 4, 5, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 4, 5, 6);
+ assertCorrectFiltering(columnIndex, gt(col, stringBinary("Prefect")), 1, 5);
+ assertCorrectFiltering(columnIndex, gtEq(col, stringBinary("Prefect")), 1, 4, 5);
+ assertCorrectFiltering(columnIndex, lt(col, stringBinary("Dent")), 4, 6);
+ assertCorrectFiltering(columnIndex, ltEq(col, stringBinary("Dent")), 4, 5, 6);
+ assertCorrectFiltering(columnIndex, userDefined(col, BinaryUtf8StartsWithB.class), 4, 6);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, BinaryUtf8StartsWithB.class)), 0, 1, 2, 3, 4, 5, 7);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, stringBinary("Beeblebrox"), stringBinary("Dent"), null, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null, null, null, null));
+ builder.add(sb.stats(type, stringBinary("Dent"), stringBinary("Jeltz")));
+ builder.add(sb.stats(type, stringBinary("Dent"), stringBinary("Prefect"), null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, stringBinary("Slartibartfast")));
+ builder.add(sb.stats(type, null, null));
+ assertEquals(8, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 2, 5, 0, 1, 2, 0, 2);
+ assertCorrectNullPages(columnIndex, false, true, true, false, false, true, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(),
+ stringBinary("Dent"),
+ null,
+ null,
+ stringBinary("Jeltz"),
+ stringBinary("Prefect"),
+ null,
+ stringBinary("Slartibartfast"),
+ null);
+ assertCorrectValues(columnIndex.getMinValues(),
+ stringBinary("Beeblebrox"),
+ null,
+ null,
+ stringBinary("Dent"),
+ stringBinary("Dent"),
+ null,
+ stringBinary("Slartibartfast"),
+ null);
+ assertCorrectFiltering(columnIndex, eq(col, stringBinary("Jeltz")), 3, 4);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 4, 5, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, stringBinary("Slartibartfast")), 0, 1, 2, 3, 4, 5, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 0, 3, 4, 6);
+ assertCorrectFiltering(columnIndex, gt(col, stringBinary("Marvin")), 4, 6);
+ assertCorrectFiltering(columnIndex, gtEq(col, stringBinary("Marvin")), 4, 6);
+ assertCorrectFiltering(columnIndex, lt(col, stringBinary("Dent")), 0);
+ assertCorrectFiltering(columnIndex, ltEq(col, stringBinary("Dent")), 0, 3, 4);
+ assertCorrectFiltering(columnIndex, userDefined(col, BinaryUtf8StartsWithB.class), 0);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, BinaryUtf8StartsWithB.class)), 0, 1, 2, 3, 4, 5, 6, 7);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, stringBinary("Slartibartfast")));
+ builder.add(sb.stats(type, null, null, null, null, null));
+ builder.add(sb.stats(type, stringBinary("Prefect"), stringBinary("Jeltz"), null));
+ builder.add(sb.stats(type, stringBinary("Dent"), stringBinary("Dent")));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, stringBinary("Dent"), stringBinary("Beeblebrox"), null, null));
+ assertEquals(8, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 0, 5, 1, 0, 2, 2, 2);
+ assertCorrectNullPages(columnIndex, true, false, true, false, false, true, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(),
+ null,
+ stringBinary("Slartibartfast"),
+ null,
+ stringBinary("Prefect"),
+ stringBinary("Dent"),
+ null,
+ null,
+ stringBinary("Dent"));
+ assertCorrectValues(columnIndex.getMinValues(),
+ null,
+ stringBinary("Slartibartfast"),
+ null,
+ stringBinary("Jeltz"),
+ stringBinary("Dent"),
+ null,
+ null,
+ stringBinary("Beeblebrox"));
+ assertCorrectFiltering(columnIndex, eq(col, stringBinary("Marvin")), 3);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 5, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, stringBinary("Dent")), 0, 1, 2, 3, 5, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 3, 4, 7);
+ assertCorrectFiltering(columnIndex, gt(col, stringBinary("Prefect")), 1);
+ assertCorrectFiltering(columnIndex, gtEq(col, stringBinary("Prefect")), 1, 3);
+ assertCorrectFiltering(columnIndex, lt(col, stringBinary("Marvin")), 3, 4, 7);
+ assertCorrectFiltering(columnIndex, ltEq(col, stringBinary("Marvin")), 3, 4, 7);
+ assertCorrectFiltering(columnIndex, userDefined(col, BinaryUtf8StartsWithB.class), 7);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, BinaryUtf8StartsWithB.class)), 0, 1, 2, 3, 4, 5, 6, 7);
+ }
+
+ @Test
+ public void testStaticBuildBinary() {
+ ColumnIndex columnIndex = ColumnIndexBuilder.build(
+ Types.required(BINARY).as(UTF8).named("test_binary_utf8"),
+ BoundaryOrder.ASCENDING,
+ asList(true, true, false, false, true, false, true, false),
+ asList(1l, 2l, 3l, 4l, 5l, 6l, 7l, 8l),
+ toBBList(
+ null,
+ null,
+ stringBinary("Beeblebrox"),
+ stringBinary("Dent"),
+ null,
+ stringBinary("Jeltz"),
+ null,
+ stringBinary("Slartibartfast")),
+ toBBList(
+ null,
+ null,
+ stringBinary("Dent"),
+ stringBinary("Dent"),
+ null,
+ stringBinary("Prefect"),
+ null,
+ stringBinary("Slartibartfast")));
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 1, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectNullPages(columnIndex, true, true, false, false, true, false, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(),
+ null,
+ null,
+ stringBinary("Dent"),
+ stringBinary("Dent"),
+ null,
+ stringBinary("Prefect"),
+ null,
+ stringBinary("Slartibartfast"));
+ assertCorrectValues(columnIndex.getMinValues(),
+ null,
+ null,
+ stringBinary("Beeblebrox"),
+ stringBinary("Dent"),
+ null,
+ stringBinary("Jeltz"),
+ null,
+ stringBinary("Slartibartfast"));
+ }
+
+ @Test
+ public void testFilterWithoutNullCounts() {
+ ColumnIndex columnIndex = ColumnIndexBuilder.build(
+ Types.required(BINARY).as(UTF8).named("test_binary_utf8"),
+ BoundaryOrder.ASCENDING,
+ asList(true, true, false, false, true, false, true, false),
+ null,
+ toBBList(
+ null,
+ null,
+ stringBinary("Beeblebrox"),
+ stringBinary("Dent"),
+ null,
+ stringBinary("Jeltz"),
+ null,
+ stringBinary("Slartibartfast")),
+ toBBList(
+ null,
+ null,
+ stringBinary("Dent"),
+ stringBinary("Dent"),
+ null,
+ stringBinary("Prefect"),
+ null,
+ stringBinary("Slartibartfast")));
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertNull(columnIndex.getNullCounts());
+ assertCorrectNullPages(columnIndex, true, true, false, false, true, false, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(),
+ null,
+ null,
+ stringBinary("Dent"),
+ stringBinary("Dent"),
+ null,
+ stringBinary("Prefect"),
+ null,
+ stringBinary("Slartibartfast"));
+ assertCorrectValues(columnIndex.getMinValues(),
+ null,
+ null,
+ stringBinary("Beeblebrox"),
+ stringBinary("Dent"),
+ null,
+ stringBinary("Jeltz"),
+ null,
+ stringBinary("Slartibartfast"));
+
+ BinaryColumn col = binaryColumn("test_col");
+ assertCorrectFiltering(columnIndex, eq(col, stringBinary("Dent")), 2, 3);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 4, 5, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, stringBinary("Dent")), 0, 1, 2, 3, 4, 5, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 2, 3, 5, 7);
+ assertCorrectFiltering(columnIndex, userDefined(col, BinaryDecimalIsNullOrZeroUdp.class), 0, 1, 2, 3, 4, 5, 6, 7);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, BinaryDecimalIsNullOrZeroUdp.class)), 2, 3, 5, 7);
+ }
+
+ @Test
+ public void testBuildBoolean() {
+ PrimitiveType type = Types.required(BOOLEAN).named("test_boolean");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ assertThat(builder, instanceOf(BooleanColumnIndexBuilder.class));
+ assertNull(builder.build());
+ BooleanColumn col = booleanColumn("test_col");
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(type, false, true));
+ builder.add(sb.stats(type, true, false, null));
+ builder.add(sb.stats(type, true, true, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, false, false));
+ assertEquals(5, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ ColumnIndex columnIndex = builder.build();
+ assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0);
+ assertCorrectNullPages(columnIndex, false, false, false, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(), true, true, true, null, false);
+ assertCorrectValues(columnIndex.getMinValues(), false, false, true, null, false);
+ assertCorrectFiltering(columnIndex, eq(col, true), 0, 1, 2);
+ assertCorrectFiltering(columnIndex, eq(col, null), 1, 2, 3);
+ assertCorrectFiltering(columnIndex, notEq(col, true), 0, 1, 2, 3, 4);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 0, 1, 2, 4);
+ assertCorrectFiltering(columnIndex, userDefined(col, BooleanIsTrueOrNull.class), 0, 1, 2, 3);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, BooleanIsTrueOrNull.class)), 0, 1, 4);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, false, false));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, null, null, null, null));
+ builder.add(sb.stats(type, false, true, null));
+ builder.add(sb.stats(type, false, true, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ assertEquals(7, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 0, 3, 4, 1, 2, 3);
+ assertCorrectNullPages(columnIndex, true, false, true, true, false, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(), null, false, null, null, true, true, null);
+ assertCorrectValues(columnIndex.getMinValues(), null, false, null, null, false, false, null);
+ assertCorrectFiltering(columnIndex, eq(col, true), 4, 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 4, 5, 6);
+ assertCorrectFiltering(columnIndex, notEq(col, true), 0, 1, 2, 3, 4, 5, 6);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 4, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, BooleanIsTrueOrNull.class), 0, 2, 3, 4, 5, 6);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, BooleanIsTrueOrNull.class)), 1, 4, 5);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, true, true));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, null, null, null, null));
+ builder.add(sb.stats(type, true, false, null));
+ builder.add(sb.stats(type, false, false, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ assertEquals(7, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 0, 3, 4, 1, 2, 3);
+ assertCorrectNullPages(columnIndex, true, false, true, true, false, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(), null, true, null, null, true, false, null);
+ assertCorrectValues(columnIndex.getMinValues(), null, true, null, null, false, false, null);
+ assertCorrectFiltering(columnIndex, eq(col, true), 1, 4);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 4, 5, 6);
+ assertCorrectFiltering(columnIndex, notEq(col, true), 0, 2, 3, 4, 5, 6);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 4, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, BooleanIsTrueOrNull.class), 0, 1, 2, 3, 4, 5, 6);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, BooleanIsTrueOrNull.class)), 4, 5);
+ }
+
+ @Test
+ public void testStaticBuildBoolean() {
+ ColumnIndex columnIndex = ColumnIndexBuilder.build(
+ Types.required(BOOLEAN).named("test_boolean"),
+ BoundaryOrder.DESCENDING,
+ asList(false, true, false, true, false, true),
+ asList(9l, 8l, 7l, 6l, 5l, 0l),
+ toBBList(false, null, false, null, true, null),
+ toBBList(true, null, false, null, true, null));
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 9, 8, 7, 6, 5, 0);
+ assertCorrectNullPages(columnIndex, false, true, false, true, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(), true, null, false, null, true, null);
+ assertCorrectValues(columnIndex.getMinValues(), false, null, false, null, true, null);
+ }
+
+ @Test
+ public void testBuildDouble() {
+ PrimitiveType type = Types.required(DOUBLE).named("test_double");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ assertThat(builder, instanceOf(DoubleColumnIndexBuilder.class));
+ assertNull(builder.build());
+ DoubleColumn col = doubleColumn("test_col");
+
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(type, -4.2, -4.1));
+ builder.add(sb.stats(type, -11.7, 7.0, null));
+ builder.add(sb.stats(type, 2.2, 2.2, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 1.9, 2.32));
+ builder.add(sb.stats(type, -21.0, 8.1));
+ assertEquals(6, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ ColumnIndex columnIndex = builder.build();
+ assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0, 0);
+ assertCorrectNullPages(columnIndex, false, false, false, true, false, false);
+ assertCorrectValues(columnIndex.getMaxValues(), -4.1, 7.0, 2.2, null, 2.32, 8.1);
+ assertCorrectValues(columnIndex.getMinValues(), -4.2, -11.7, 2.2, null, 1.9, -21.0);
+ assertCorrectFiltering(columnIndex, eq(col, 0.0), 1, 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 1, 2, 3);
+ assertCorrectFiltering(columnIndex, notEq(col, 2.2), 0, 1, 2, 3, 4, 5);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, gt(col, 2.2), 1, 4, 5);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2.2), 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, lt(col, -4.2), 1, 5);
+ assertCorrectFiltering(columnIndex, ltEq(col, -4.2), 0, 1, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, DoubleIsInteger.class), 1, 4, 5);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, DoubleIsInteger.class)), 0, 1, 2, 3, 4, 5);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, -532.3, -345.2, null, null));
+ builder.add(sb.stats(type, -234.7, -234.6, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, -234.6, 2.99999));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 3.0, 42.83));
+ builder.add(sb.stats(type, null, null));
+ assertEquals(9, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2);
+ assertCorrectNullPages(columnIndex, true, false, false, true, true, false, true, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(), null, -345.2, -234.6, null, null, 2.99999, null, 42.83, null);
+ assertCorrectValues(columnIndex.getMinValues(), null, -532.3, -234.7, null, null, -234.6, null, 3.0, null);
+ assertCorrectFiltering(columnIndex, eq(col, 0.0), 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 4, 6, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, 0.0), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 2, 5, 7);
+ assertCorrectFiltering(columnIndex, gt(col, 2.99999), 7);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2.99999), 5, 7);
+ assertCorrectFiltering(columnIndex, lt(col, -234.6), 1, 2);
+ assertCorrectFiltering(columnIndex, ltEq(col, -234.6), 1, 2, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, DoubleIsInteger.class), 1, 5, 7);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, DoubleIsInteger.class)), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null, null, null, null));
+ builder.add(sb.stats(type, 532.3, 345.2));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 234.7, 234.6, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 234.69, -2.99999));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, -3.0, -42.83));
+ assertEquals(9, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0);
+ assertCorrectNullPages(columnIndex, true, false, true, false, true, false, true, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(), null, 532.3, null, 234.7, null, 234.69, null, null, -3.0);
+ assertCorrectValues(columnIndex.getMinValues(), null, 345.2, null, 234.6, null, -2.99999, null, null, -42.83);
+ assertCorrectFiltering(columnIndex, eq(col, 234.6), 3, 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 4, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, 2.2), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 3, 5, 8);
+ assertCorrectFiltering(columnIndex, gt(col, 2.2), 1, 3, 5);
+ assertCorrectFiltering(columnIndex, gtEq(col, 234.69), 1, 3, 5);
+ assertCorrectFiltering(columnIndex, lt(col, -2.99999), 8);
+ assertCorrectFiltering(columnIndex, ltEq(col, -2.99999), 5, 8);
+ assertCorrectFiltering(columnIndex, userDefined(col, DoubleIsInteger.class), 1, 5, 8);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, DoubleIsInteger.class)), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ }
+
+ @Test
+ public void testBuildDoubleZeroNaN() {
+ PrimitiveType type = Types.required(DOUBLE).named("test_double");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(type, -1.0, -0.0));
+ builder.add(sb.stats(type, 0.0, 1.0));
+ builder.add(sb.stats(type, 1.0, 100.0));
+ ColumnIndex columnIndex = builder.build();
+ assertCorrectValues(columnIndex.getMinValues(), -1.0, -0.0, 1.0);
+ assertCorrectValues(columnIndex.getMaxValues(), 0.0, 1.0, 100.0);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ builder.add(sb.stats(type, -1.0, -0.0));
+ builder.add(sb.stats(type, 0.0, Double.NaN));
+ builder.add(sb.stats(type, 1.0, 100.0));
+ assertNull(builder.build());
+ }
+
+ @Test
+ public void testStaticBuildDouble() {
+ ColumnIndex columnIndex = ColumnIndexBuilder.build(
+ Types.required(DOUBLE).named("test_double"),
+ BoundaryOrder.UNORDERED,
+ asList(false, false, false, false, false, false),
+ asList(0l, 1l, 2l, 3l, 4l, 5l),
+ toBBList(-1.0, -2.0, -3.0, -4.0, -5.0, -6.0),
+ toBBList(1.0, 2.0, 3.0, 4.0, 5.0, 6.0));
+ assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 4, 5);
+ assertCorrectNullPages(columnIndex, false, false, false, false, false, false);
+ assertCorrectValues(columnIndex.getMaxValues(), 1.0, 2.0, 3.0, 4.0, 5.0, 6.0);
+ assertCorrectValues(columnIndex.getMinValues(), -1.0, -2.0, -3.0, -4.0, -5.0, -6.0);
+ }
+
+ @Test
+ public void testBuildFloat() {
+ PrimitiveType type = Types.required(FLOAT).named("test_float");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ assertThat(builder, instanceOf(FloatColumnIndexBuilder.class));
+ assertNull(builder.build());
+ FloatColumn col = floatColumn("test_col");
+
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(type, -4.2f, -4.1f));
+ builder.add(sb.stats(type, -11.7f, 7.0f, null));
+ builder.add(sb.stats(type, 2.2f, 2.2f, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 1.9f, 2.32f));
+ builder.add(sb.stats(type, -21.0f, 8.1f));
+ assertEquals(6, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ ColumnIndex columnIndex = builder.build();
+ assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0, 0);
+ assertCorrectNullPages(columnIndex, false, false, false, true, false, false);
+ assertCorrectValues(columnIndex.getMaxValues(), -4.1f, 7.0f, 2.2f, null, 2.32f, 8.1f);
+ assertCorrectValues(columnIndex.getMinValues(), -4.2f, -11.7f, 2.2f, null, 1.9f, -21.0f);
+ assertCorrectFiltering(columnIndex, eq(col, 0.0f), 1, 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 1, 2, 3);
+ assertCorrectFiltering(columnIndex, notEq(col, 2.2f), 0, 1, 2, 3, 4, 5);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, gt(col, 2.2f), 1, 4, 5);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2.2f), 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, lt(col, 0.0f), 0, 1, 5);
+ assertCorrectFiltering(columnIndex, ltEq(col, 1.9f), 0, 1, 4, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, FloatIsInteger.class), 1, 4, 5);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, FloatIsInteger.class)), 0, 1, 2, 3, 4, 5);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, -532.3f, -345.2f, null, null));
+ builder.add(sb.stats(type, -300.6f, -234.7f, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, -234.6f, 2.99999f));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 3.0f, 42.83f));
+ builder.add(sb.stats(type, null, null));
+ assertEquals(9, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2);
+ assertCorrectNullPages(columnIndex, true, false, false, true, true, false, true, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(), null, -345.2f, -234.7f, null, null, 2.99999f, null, 42.83f, null);
+ assertCorrectValues(columnIndex.getMinValues(), null, -532.3f, -300.6f, null, null, -234.6f, null, 3.0f, null);
+ assertCorrectFiltering(columnIndex, eq(col, 0.0f), 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 4, 6, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, 2.2f), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 2, 5, 7);
+ assertCorrectFiltering(columnIndex, gt(col, 2.2f), 5, 7);
+ assertCorrectFiltering(columnIndex, gtEq(col, -234.7f), 2, 5, 7);
+ assertCorrectFiltering(columnIndex, lt(col, -234.6f), 1, 2);
+ assertCorrectFiltering(columnIndex, ltEq(col, -234.6f), 1, 2, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, FloatIsInteger.class), 1, 2, 5, 7);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, FloatIsInteger.class)), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null, null, null, null));
+ builder.add(sb.stats(type, 532.3f, 345.2f));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 234.7f, 234.6f, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 234.6f, -2.99999f));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, -3.0f, -42.83f));
+ assertEquals(9, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0);
+ assertCorrectNullPages(columnIndex, true, false, true, false, true, false, true, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(), null, 532.3f, null, 234.7f, null, 234.6f, null, null, -3.0f);
+ assertCorrectValues(columnIndex.getMinValues(), null, 345.2f, null, 234.6f, null, -2.99999f, null, null, -42.83f);
+ assertCorrectFiltering(columnIndex, eq(col, 234.65f), 3);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 4, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, 2.2f), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 3, 5, 8);
+ assertCorrectFiltering(columnIndex, gt(col, 2.2f), 1, 3, 5);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2.2f), 1, 3, 5);
+ assertCorrectFiltering(columnIndex, lt(col, 0.0f), 5, 8);
+ assertCorrectFiltering(columnIndex, ltEq(col, 0.0f), 5, 8);
+ assertCorrectFiltering(columnIndex, userDefined(col, FloatIsInteger.class), 1, 5, 8);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, FloatIsInteger.class)), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ }
+
+ @Test
+ public void testBuildFloatZeroNaN() {
+ PrimitiveType type = Types.required(FLOAT).named("test_float");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(type, -1.0f, -0.0f));
+ builder.add(sb.stats(type, 0.0f, 1.0f));
+ builder.add(sb.stats(type, 1.0f, 100.0f));
+ ColumnIndex columnIndex = builder.build();
+ assertCorrectValues(columnIndex.getMinValues(), -1.0f, -0.0f, 1.0f);
+ assertCorrectValues(columnIndex.getMaxValues(), 0.0f, 1.0f, 100.0f);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ builder.add(sb.stats(type, -1.0f, -0.0f));
+ builder.add(sb.stats(type, 0.0f, Float.NaN));
+ builder.add(sb.stats(type, 1.0f, 100.0f));
+ assertNull(builder.build());
+ }
+
+ @Test
+ public void testStaticBuildFloat() {
+ ColumnIndex columnIndex = ColumnIndexBuilder.build(
+ Types.required(FLOAT).named("test_float"),
+ BoundaryOrder.ASCENDING,
+ asList(true, true, true, false, false, false),
+ asList(9l, 8l, 7l, 6l, 0l, 0l),
+ toBBList(null, null, null, -3.0f, -2.0f, 0.1f),
+ toBBList(null, null, null, -2.0f, 0.0f, 6.0f));
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 9, 8, 7, 6, 0, 0);
+ assertCorrectNullPages(columnIndex, true, true, true, false, false, false);
+ assertCorrectValues(columnIndex.getMaxValues(), null, null, null, -2.0f, 0.0f, 6.0f);
+ assertCorrectValues(columnIndex.getMinValues(), null, null, null, -3.0f, -2.0f, 0.1f);
+ }
+
+ @Test
+ public void testBuildInt32() {
+ PrimitiveType type = Types.required(INT32).named("test_int32");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ assertThat(builder, instanceOf(IntColumnIndexBuilder.class));
+ assertNull(builder.build());
+ IntColumn col = intColumn("test_col");
+
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(type, -4, 10));
+ builder.add(sb.stats(type, -11, 7, null));
+ builder.add(sb.stats(type, 2, 2, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 1, 2));
+ builder.add(sb.stats(type, -21, 8));
+ assertEquals(6, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ ColumnIndex columnIndex = builder.build();
+ assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0, 0);
+ assertCorrectNullPages(columnIndex, false, false, false, true, false, false);
+ assertCorrectValues(columnIndex.getMaxValues(), 10, 7, 2, null, 2, 8);
+ assertCorrectValues(columnIndex.getMinValues(), -4, -11, 2, null, 1, -21);
+ assertCorrectFiltering(columnIndex, eq(col, 2), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 1, 2, 3);
+ assertCorrectFiltering(columnIndex, notEq(col, 2), 0, 1, 2, 3, 4, 5);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, gt(col, 2), 0, 1, 5);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, lt(col, 2), 0, 1, 4, 5);
+ assertCorrectFiltering(columnIndex, ltEq(col, 2), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, IntegerIsDivisableWith3.class), 0, 1, 5);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, IntegerIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, -532, -345, null, null));
+ builder.add(sb.stats(type, -500, -42, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, -42, 2));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 3, 42));
+ builder.add(sb.stats(type, null, null));
+ assertEquals(9, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2);
+ assertCorrectNullPages(columnIndex, true, false, false, true, true, false, true, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(), null, -345, -42, null, null, 2, null, 42, null);
+ assertCorrectValues(columnIndex.getMinValues(), null, -532, -500, null, null, -42, null, 3, null);
+ assertCorrectFiltering(columnIndex, eq(col, 2), 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 4, 6, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, 2), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 2, 5, 7);
+ assertCorrectFiltering(columnIndex, gt(col, 2), 7);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2), 5, 7);
+ assertCorrectFiltering(columnIndex, lt(col, 2), 1, 2, 5);
+ assertCorrectFiltering(columnIndex, ltEq(col, 2), 1, 2, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, IntegerIsDivisableWith3.class), 1, 2, 5, 7);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, IntegerIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5, 6, 7,
+ 8);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null, null, null, null));
+ builder.add(sb.stats(type, 532, 345));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 234, 42, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 42, -2));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, -3, -42));
+ assertEquals(9, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0);
+ assertCorrectNullPages(columnIndex, true, false, true, false, true, false, true, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(), null, 532, null, 234, null, 42, null, null, -3);
+ assertCorrectValues(columnIndex.getMinValues(), null, 345, null, 42, null, -2, null, null, -42);
+ assertCorrectFiltering(columnIndex, eq(col, 2), 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 4, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, 2), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 3, 5, 8);
+ assertCorrectFiltering(columnIndex, gt(col, 2), 1, 3, 5);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2), 1, 3, 5);
+ assertCorrectFiltering(columnIndex, lt(col, 2), 5, 8);
+ assertCorrectFiltering(columnIndex, ltEq(col, 2), 5, 8);
+ assertCorrectFiltering(columnIndex, userDefined(col, IntegerIsDivisableWith3.class), 1, 3, 5, 8);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, IntegerIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5, 6, 7,
+ 8);
+ }
+
+ @Test
+ public void testStaticBuildInt32() {
+ ColumnIndex columnIndex = ColumnIndexBuilder.build(
+ Types.required(INT32).named("test_int32"),
+ BoundaryOrder.DESCENDING,
+ asList(false, false, false, true, true, true),
+ asList(0l, 10l, 0l, 3l, 5l, 7l),
+ toBBList(10, 8, 6, null, null, null),
+ toBBList(9, 7, 5, null, null, null));
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 0, 10, 0, 3, 5, 7);
+ assertCorrectNullPages(columnIndex, false, false, false, true, true, true);
+ assertCorrectValues(columnIndex.getMaxValues(), 9, 7, 5, null, null, null);
+ assertCorrectValues(columnIndex.getMinValues(), 10, 8, 6, null, null, null);
+ }
+
+ @Test
+ public void testBuildUInt8() {
+ PrimitiveType type = Types.required(INT32).as(UINT_8).named("test_uint8");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ assertThat(builder, instanceOf(IntColumnIndexBuilder.class));
+ assertNull(builder.build());
+ IntColumn col = intColumn("test_col");
+
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(type, 4, 10));
+ builder.add(sb.stats(type, 11, 17, null));
+ builder.add(sb.stats(type, 2, 2, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 1, 0xFF));
+ builder.add(sb.stats(type, 0xEF, 0xFA));
+ assertEquals(6, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ ColumnIndex columnIndex = builder.build();
+ assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0, 0);
+ assertCorrectNullPages(columnIndex, false, false, false, true, false, false);
+ assertCorrectValues(columnIndex.getMaxValues(), 10, 17, 2, null, 0xFF, 0xFA);
+ assertCorrectValues(columnIndex.getMinValues(), 4, 11, 2, null, 1, 0xEF);
+ assertCorrectFiltering(columnIndex, eq(col, 2), 2, 4);
+ assertCorrectFiltering(columnIndex, eq(col, null), 1, 2, 3);
+ assertCorrectFiltering(columnIndex, notEq(col, 2), 0, 1, 2, 3, 4, 5);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, gt(col, 2), 0, 1, 4, 5);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, lt(col, 0xEF), 0, 1, 2, 4);
+ assertCorrectFiltering(columnIndex, ltEq(col, 0xEF), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, IntegerIsDivisableWith3.class), 0, 1, 4, 5);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, IntegerIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 0, 0, null, null));
+ builder.add(sb.stats(type, 0, 42, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 42, 0xEE));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 0xEF, 0xFF));
+ builder.add(sb.stats(type, null, null));
+ assertEquals(9, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2);
+ assertCorrectNullPages(columnIndex, true, false, false, true, true, false, true, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(), null, 0, 42, null, null, 0xEE, null, 0xFF, null);
+ assertCorrectValues(columnIndex.getMinValues(), null, 0, 0, null, null, 42, null, 0xEF, null);
+ assertCorrectFiltering(columnIndex, eq(col, 2), 2);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 4, 6, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, 2), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 2, 5, 7);
+ assertCorrectFiltering(columnIndex, gt(col, 0xEE), 7);
+ assertCorrectFiltering(columnIndex, gtEq(col, 0xEE), 5, 7);
+ assertCorrectFiltering(columnIndex, lt(col, 42), 1, 2);
+ assertCorrectFiltering(columnIndex, ltEq(col, 42), 1, 2, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, IntegerIsDivisableWith3.class), 1, 2, 5, 7);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, IntegerIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5, 6, 7,
+ 8);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null, null, null, null));
+ builder.add(sb.stats(type, 0xFF, 0xFF));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 0xEF, 0xEA, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 0xEE, 42));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 41, 0));
+ assertEquals(9, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0);
+ assertCorrectNullPages(columnIndex, true, false, true, false, true, false, true, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(), null, 0xFF, null, 0xEF, null, 0xEE, null, null, 41);
+ assertCorrectValues(columnIndex.getMinValues(), null, 0xFF, null, 0xEA, null, 42, null, null, 0);
+ assertCorrectFiltering(columnIndex, eq(col, 0xAB), 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 4, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, 0xFF), 0, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 3, 5, 8);
+ assertCorrectFiltering(columnIndex, gt(col, 0xFF));
+ assertCorrectFiltering(columnIndex, gtEq(col, 0xFF), 1);
+ assertCorrectFiltering(columnIndex, lt(col, 42), 8);
+ assertCorrectFiltering(columnIndex, ltEq(col, 42), 5, 8);
+ assertCorrectFiltering(columnIndex, userDefined(col, IntegerIsDivisableWith3.class), 1, 3, 5, 8);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, IntegerIsDivisableWith3.class)), 0, 2, 3, 4, 5, 6, 7,
+ 8);
+ }
+
+ @Test
+ public void testBuildInt64() {
+ PrimitiveType type = Types.required(INT64).named("test_int64");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ assertThat(builder, instanceOf(LongColumnIndexBuilder.class));
+ assertNull(builder.build());
+ LongColumn col = longColumn("test_col");
+
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(type, -4l, 10l));
+ builder.add(sb.stats(type, -11l, 7l, null));
+ builder.add(sb.stats(type, 2l, 2l, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 1l, 2l));
+ builder.add(sb.stats(type, -21l, 8l));
+ assertEquals(6, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ ColumnIndex columnIndex = builder.build();
+ assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 0l, 1l, 2l, 3l, 0l, 0l);
+ assertCorrectNullPages(columnIndex, false, false, false, true, false, false);
+ assertCorrectValues(columnIndex.getMaxValues(), 10l, 7l, 2l, null, 2l, 8l);
+ assertCorrectValues(columnIndex.getMinValues(), -4l, -11l, 2l, null, 1l, -21l);
+ assertCorrectFiltering(columnIndex, eq(col, 0l), 0, 1, 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 1, 2, 3);
+ assertCorrectFiltering(columnIndex, notEq(col, 0l), 0, 1, 2, 3, 4, 5);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, gt(col, 2l), 0, 1, 5);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2l), 0, 1, 2, 4, 5);
+ assertCorrectFiltering(columnIndex, lt(col, -21l));
+ assertCorrectFiltering(columnIndex, ltEq(col, -21l), 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, LongIsDivisableWith3.class), 0, 1, 5);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, LongIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, -532l, -345l, null, null));
+ builder.add(sb.stats(type, -234l, -42l, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, -42l, 2l));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, -3l, 42l));
+ builder.add(sb.stats(type, null, null));
+ assertEquals(9, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2);
+ assertCorrectNullPages(columnIndex, true, false, false, true, true, false, true, false, true);
+ assertCorrectValues(columnIndex.getMaxValues(), null, -345l, -42l, null, null, 2l, null, 42l, null);
+ assertCorrectValues(columnIndex.getMinValues(), null, -532l, -234l, null, null, -42l, null, -3l, null);
+ assertCorrectFiltering(columnIndex, eq(col, -42l), 2, 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 4, 6, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, -42l), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 2, 5, 7);
+ assertCorrectFiltering(columnIndex, gt(col, 2l), 7);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2l), 5, 7);
+ assertCorrectFiltering(columnIndex, lt(col, -42l), 1, 2);
+ assertCorrectFiltering(columnIndex, ltEq(col, -42l), 1, 2, 5);
+ assertCorrectFiltering(columnIndex, userDefined(col, LongIsDivisableWith3.class), 1, 2, 5, 7);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, LongIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5, 6, 7,
+ 8);
+
+ builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ sb = new StatsBuilder();
+ builder.add(sb.stats(type, null, null, null, null, null));
+ builder.add(sb.stats(type, 532l, 345l));
+ builder.add(sb.stats(type, null, null, null));
+ builder.add(sb.stats(type, 234l, 42l, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, 42l, -2l));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, null, null));
+ builder.add(sb.stats(type, -3l, -42l));
+ assertEquals(9, builder.getPageCount());
+ assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
+ columnIndex = builder.build();
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0);
+ assertCorrectNullPages(columnIndex, true, false, true, false, true, false, true, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(), null, 532l, null, 234l, null, 42l, null, null, -3l);
+ assertCorrectValues(columnIndex.getMinValues(), null, 345l, null, 42l, null, -2l, null, null, -42l);
+ assertCorrectFiltering(columnIndex, eq(col, 0l), 5);
+ assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 4, 6, 7);
+ assertCorrectFiltering(columnIndex, notEq(col, 0l), 0, 1, 2, 3, 4, 5, 6, 7, 8);
+ assertCorrectFiltering(columnIndex, notEq(col, null), 1, 3, 5, 8);
+ assertCorrectFiltering(columnIndex, gt(col, 2l), 1, 3, 5);
+ assertCorrectFiltering(columnIndex, gtEq(col, 2l), 1, 3, 5);
+ assertCorrectFiltering(columnIndex, lt(col, -42l));
+ assertCorrectFiltering(columnIndex, ltEq(col, -42l), 8);
+ assertCorrectFiltering(columnIndex, userDefined(col, LongIsDivisableWith3.class), 1, 3, 5, 8);
+ assertCorrectFiltering(columnIndex, invert(userDefined(col, LongIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5, 6, 7,
+ 8);
+ }
+
+ @Test
+ public void testStaticBuildInt64() {
+ ColumnIndex columnIndex = ColumnIndexBuilder.build(
+ Types.required(INT64).named("test_int64"),
+ BoundaryOrder.UNORDERED,
+ asList(true, false, true, false, true, false),
+ asList(1l, 2l, 3l, 4l, 5l, 6l),
+ toBBList(null, 2l, null, 4l, null, 9l),
+ toBBList(null, 3l, null, 15l, null, 10l));
+ assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
+ assertCorrectNullCounts(columnIndex, 1, 2, 3, 4, 5, 6);
+ assertCorrectNullPages(columnIndex, true, false, true, false, true, false);
+ assertCorrectValues(columnIndex.getMaxValues(), null, 3l, null, 15l, null, 10l);
+ assertCorrectValues(columnIndex.getMinValues(), null, 2l, null, 4l, null, 9l);
+ }
+
+ @Test
+ public void testNoOpBuilder() {
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getNoOpBuilder();
+ StatsBuilder sb = new StatsBuilder();
+ builder.add(sb.stats(Types.required(BINARY).as(UTF8).named("test_binary_utf8"), stringBinary("Jeltz"),
+ stringBinary("Slartibartfast"), null, null));
+ builder.add(sb.stats(Types.required(BOOLEAN).named("test_boolean"), true, true, null, null));
+ builder.add(sb.stats(Types.required(DOUBLE).named("test_double"), null, null, null));
+ builder.add(sb.stats(Types.required(INT32).named("test_int32"), null, null));
+ builder.add(sb.stats(Types.required(INT64).named("test_int64"), -234l, -42l, null));
+ assertEquals(0, builder.getPageCount());
+ assertEquals(0, builder.getMinMaxSize());
+ assertNull(builder.build());
+ }
+
+ private static List<ByteBuffer> toBBList(Binary... values) {
+ List<ByteBuffer> buffers = new ArrayList<>(values.length);
+ for (Binary value : values) {
+ if (value == null) {
+ buffers.add(ByteBuffer.allocate(0));
+ } else {
+ buffers.add(value.toByteBuffer());
+ }
+ }
+ return buffers;
+ }
+
+ private static List<ByteBuffer> toBBList(Boolean... values) {
+ List<ByteBuffer> buffers = new ArrayList<>(values.length);
+ for (Boolean value : values) {
+ if (value == null) {
+ buffers.add(ByteBuffer.allocate(0));
+ } else {
+ buffers.add(ByteBuffer.wrap(BytesUtils.booleanToBytes(value)));
+ }
+ }
+ return buffers;
+ }
+
+ private static List<ByteBuffer> toBBList(Double... values) {
+ List<ByteBuffer> buffers = new ArrayList<>(values.length);
+ for (Double value : values) {
+ if (value == null) {
+ buffers.add(ByteBuffer.allocate(0));
+ } else {
+ buffers.add(ByteBuffer.wrap(BytesUtils.longToBytes(Double.doubleToLongBits(value))));
+ }
+ }
+ return buffers;
+ }
+
+ private static List<ByteBuffer> toBBList(Float... values) {
+ List<ByteBuffer> buffers = new ArrayList<>(values.length);
+ for (Float value : values) {
+ if (value == null) {
+ buffers.add(ByteBuffer.allocate(0));
+ } else {
+ buffers.add(ByteBuffer.wrap(BytesUtils.intToBytes(Float.floatToIntBits(value))));
+ }
+ }
+ return buffers;
+ }
+
+ private static List<ByteBuffer> toBBList(Integer... values) {
+ List<ByteBuffer> buffers = new ArrayList<>(values.length);
+ for (Integer value : values) {
+ if (value == null) {
+ buffers.add(ByteBuffer.allocate(0));
+ } else {
+ buffers.add(ByteBuffer.wrap(BytesUtils.intToBytes(value)));
+ }
+ }
+ return buffers;
+ }
+
+ private static List<ByteBuffer> toBBList(Long... values) {
+ List<ByteBuffer> buffers = new ArrayList<>(values.length);
+ for (Long value : values) {
+ if (value == null) {
+ buffers.add(ByteBuffer.allocate(0));
+ } else {
+ buffers.add(ByteBuffer.wrap(BytesUtils.longToBytes(value)));
+ }
+ }
+ return buffers;
+ }
+
+ private static Binary decimalBinary(String num) {
+ return Binary.fromConstantByteArray(new BigDecimal(num).unscaledValue().toByteArray());
+ }
+
+ private static Binary stringBinary(String str) {
+ return Binary.fromString(str);
+ }
+
+ private static void assertCorrectValues(List<ByteBuffer> values, Binary... expectedValues) {
+ assertEquals(expectedValues.length, values.size());
+ for (int i = 0; i < expectedValues.length; ++i) {
+ Binary expectedValue = expectedValues[i];
+ ByteBuffer value = values.get(i);
+ if (expectedValue == null) {
+ assertFalse("The byte buffer should be empty for null pages", value.hasRemaining());
+ } else {
+ assertArrayEquals("Invalid value for page " + i, expectedValue.getBytesUnsafe(), value.array());
+ }
+ }
+ }
+
+ private static void assertCorrectValues(List<ByteBuffer> values, Boolean... expectedValues) {
+ assertEquals(expectedValues.length, values.size());
+ for (int i = 0; i < expectedValues.length; ++i) {
+ Boolean expectedValue = expectedValues[i];
+ ByteBuffer value = values.get(i);
+ if (expectedValue == null) {
+ assertFalse("The byte buffer should be empty for null pages", value.hasRemaining());
+ } else {
+ assertEquals("The byte buffer should be 1 byte long for boolean", 1, value.remaining());
+ assertEquals("Invalid value for page " + i, expectedValue.booleanValue(), value.get(0) != 0);
+ }
+ }
+ }
+
+ private static void assertCorrectValues(List<ByteBuffer> values, Double... expectedValues) {
+ assertEquals(expectedValues.length, values.size());
+ for (int i = 0; i < expectedValues.length; ++i) {
+ Double expectedValue = expectedValues[i];
+ ByteBuffer value = values.get(i);
+ if (expectedValue == null) {
+ assertFalse("The byte buffer should be empty for null pages", value.hasRemaining());
+ } else {
+ assertEquals("The byte buffer should be 8 bytes long for double", 8, value.remaining());
+ assertTrue("Invalid value for page " + i, Double.compare(expectedValue.doubleValue(), value.getDouble(0)) == 0);
+ }
+ }
+ }
+
+ private static void assertCorrectValues(List<ByteBuffer> values, Float... expectedValues) {
+ assertEquals(expectedValues.length, values.size());
+ for (int i = 0; i < expectedValues.length; ++i) {
+ Float expectedValue = expectedValues[i];
+ ByteBuffer value = values.get(i);
+ if (expectedValue == null) {
+ assertFalse("The byte buffer should be empty for null pages", value.hasRemaining());
+ } else {
+ assertEquals("The byte buffer should be 4 bytes long for double", 4, value.remaining());
+ assertTrue("Invalid value for page " + i, Float.compare(expectedValue.floatValue(), value.getFloat(0)) == 0);
+ }
+ }
+ }
+
+ private static void assertCorrectValues(List<ByteBuffer> values, Integer... expectedValues) {
+ assertEquals(expectedValues.length, values.size());
+ for (int i = 0; i < expectedValues.length; ++i) {
+ Integer expectedValue = expectedValues[i];
+ ByteBuffer value = values.get(i);
+ if (expectedValue == null) {
+ assertFalse("The byte buffer should be empty for null pages", value.hasRemaining());
+ } else {
+ assertEquals("The byte buffer should be 4 bytes long for int32", 4, value.remaining());
+ assertEquals("Invalid value for page " + i, expectedValue.intValue(), value.getInt(0));
+ }
+ }
+ }
+
+ private static void assertCorrectValues(List<ByteBuffer> values, Long... expectedValues) {
+ assertEquals(expectedValues.length, values.size());
+ for (int i = 0; i < expectedValues.length; ++i) {
+ Long expectedValue = expectedValues[i];
+ ByteBuffer value = values.get(i);
+ if (expectedValue == null) {
+ assertFalse("The byte buffer should be empty for null pages", value.hasRemaining());
+ } else {
+ assertEquals("The byte buffer should be 8 bytes long for int64", 8, value.remaining());
+ assertEquals("Invalid value for page " + i, expectedValue.intValue(), value.getLong(0));
+ }
+ }
+ }
+
+ private static void assertCorrectNullCounts(ColumnIndex columnIndex, long... expectedNullCounts) {
+ List<Long> nullCounts = columnIndex.getNullCounts();
+ assertEquals(expectedNullCounts.length, nullCounts.size());
+ for (int i = 0; i < expectedNullCounts.length; ++i) {
+ assertEquals("Invalid null count at page " + i, expectedNullCounts[i], nullCounts.get(i).longValue());
+ }
+ }
+
+ private static void assertCorrectNullPages(ColumnIndex columnIndex, boolean... expectedNullPages) {
+ List<Boolean> nullPages = columnIndex.getNullPages();
+ assertEquals(expectedNullPages.length, nullPages.size());
+ for (int i = 0; i < expectedNullPages.length; ++i) {
+ assertEquals("Invalid null pages at page " + i, expectedNullPages[i], nullPages.get(i).booleanValue());
+ }
+ }
+
+ private static class StatsBuilder {
+ private long minMaxSize;
+
+ Statistics<?> stats(PrimitiveType type, Object... values) {
+ Statistics<?> stats = Statistics.createStats(type);
+ for (Object value : values) {
+ if (value == null) {
+ stats.incrementNumNulls();
+ continue;
+ }
+ switch (type.getPrimitiveTypeName()) {
+ case BINARY:
+ case FIXED_LEN_BYTE_ARRAY:
+ case INT96:
+ stats.updateStats((Binary) value);
+ break;
+ case BOOLEAN:
+ stats.updateStats((boolean) value);
+ break;
+ case DOUBLE:
+ stats.updateStats((double) value);
+ break;
+ case FLOAT:
+ stats.updateStats((float) value);
+ break;
+ case INT32:
+ stats.updateStats((int) value);
+ break;
+ case INT64:
+ stats.updateStats((long) value);
+ break;
+ default:
+ fail("Unsupported value type for stats: " + value.getClass());
+ }
+ }
+ if (stats.hasNonNullValue()) {
+ minMaxSize += stats.getMinBytes().length;
+ minMaxSize += stats.getMaxBytes().length;
+ }
+ return stats;
+ }
+
+ long getMinMaxSize() {
+ return minMaxSize;
+ }
+ }
+
+ private static void assertCorrectFiltering(ColumnIndex ci, FilterPredicate predicate, int... expectedIndexes) {
+ TestIndexIterator.assertEquals(predicate.accept(ci), expectedIndexes);
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestIndexIterator.java b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestIndexIterator.java
new file mode 100644
index 000000000..d9047f26d
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestIndexIterator.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import java.util.Arrays;
+import java.util.PrimitiveIterator;
+
+import org.junit.Test;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+
+/**
+ * Unit test for {@link IndexIterator}.
+ */
+public class TestIndexIterator {
+ @Test
+ public void testAll() {
+ assertEquals(IndexIterator.all(10), 0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+ }
+
+ @Test
+ public void testFilter() {
+ assertEquals(IndexIterator.filter(30, value -> value % 3 == 0), 0, 3, 6, 9, 12, 15, 18, 21, 24, 27);
+ }
+
+ @Test
+ public void testFilterTranslate() {
+ assertEquals(IndexIterator.filterTranslate(20, value -> value < 5, Math::negateExact), 0, -1, -2, -3, -4);
+ }
+
+ @Test
+ public void testRangeTranslate() {
+ assertEquals(IndexIterator.rangeTranslate(11, 18, i -> i - 10), 1, 2, 3, 4, 5, 6, 7, 8);
+ }
+
+ static void assertEquals(PrimitiveIterator.OfInt actualIt, int... expectedValues) {
+ IntList actualList = new IntArrayList();
+ actualIt.forEachRemaining((int value) -> actualList.add(value));
+ int[] actualValues = actualList.toIntArray();
+ assertArrayEquals(
+ "ExpectedValues: " + Arrays.toString(expectedValues) + " ActualValues: " + Arrays.toString(actualValues),
+ expectedValues, actualValues);
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestOffsetIndexBuilder.java b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestOffsetIndexBuilder.java
new file mode 100644
index 000000000..1e1275c84
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestOffsetIndexBuilder.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndexBuilder;
+import org.junit.Test;
+
+/**
+ * Tests for {@link OffsetIndexBuilder}.
+ */
+public class TestOffsetIndexBuilder {
+ @Test
+ public void testBuilderWithSizeAndRowCount() {
+ OffsetIndexBuilder builder = OffsetIndexBuilder.getBuilder();
+ assertNull(builder.build());
+ assertNull(builder.build(1234));
+
+ builder.add(1000, 10);
+ builder.add(2000, 19);
+ builder.add(3000, 27);
+ builder.add(1200, 9);
+ assertCorrectValues(builder.build(),
+ 0, 1000, 0,
+ 1000, 2000, 10,
+ 3000, 3000, 29,
+ 6000, 1200, 56);
+ assertCorrectValues(builder.build(10000),
+ 10000, 1000, 0,
+ 11000, 2000, 10,
+ 13000, 3000, 29,
+ 16000, 1200, 56);
+ }
+
+ @Test
+ public void testNoOpBuilderWithSizeAndRowCount() {
+ OffsetIndexBuilder builder = OffsetIndexBuilder.getNoOpBuilder();
+ builder.add(1, 2);
+ builder.add(3, 4);
+ builder.add(5, 6);
+ builder.add(7, 8);
+ assertNull(builder.build());
+ assertNull(builder.build(1000));
+ }
+
+ @Test
+ public void testBuilderWithOffsetSizeIndex() {
+ OffsetIndexBuilder builder = OffsetIndexBuilder.getBuilder();
+ assertNull(builder.build());
+ assertNull(builder.build(1234));
+
+ builder.add(1000, 10000, 0);
+ builder.add(22000, 12000, 100);
+ builder.add(48000, 22000, 211);
+ builder.add(90000, 30000, 361);
+ assertCorrectValues(builder.build(),
+ 1000, 10000, 0,
+ 22000, 12000, 100,
+ 48000, 22000, 211,
+ 90000, 30000, 361);
+ assertCorrectValues(builder.build(100000),
+ 101000, 10000, 0,
+ 122000, 12000, 100,
+ 148000, 22000, 211,
+ 190000, 30000, 361);
+ }
+
+ @Test
+ public void testNoOpBuilderWithOffsetSizeIndex() {
+ OffsetIndexBuilder builder = OffsetIndexBuilder.getNoOpBuilder();
+ builder.add(1, 2, 3);
+ builder.add(4, 5, 6);
+ builder.add(7, 8, 9);
+ builder.add(10, 11, 12);
+ assertNull(builder.build());
+ assertNull(builder.build(1000));
+ }
+
+ private void assertCorrectValues(OffsetIndex offsetIndex, long... offset_size_rowIndex_triplets) {
+ assertEquals(offset_size_rowIndex_triplets.length % 3, 0);
+ int pageCount = offset_size_rowIndex_triplets.length / 3;
+ assertEquals("Invalid pageCount", pageCount, offsetIndex.getPageCount());
+ for (int i = 0; i < pageCount; ++i) {
+ assertEquals("Invalid offsetIndex at page " + i, offset_size_rowIndex_triplets[3 * i],
+ offsetIndex.getOffset(i));
+ assertEquals("Invalid compressedPageSize at page " + i, offset_size_rowIndex_triplets[3 * i + 1],
+ offsetIndex.getCompressedPageSize(i));
+ assertEquals("Invalid firstRowIndex at page " + i, offset_size_rowIndex_triplets[3 * i + 2],
+ offsetIndex.getFirstRowIndex(i));
+ long expectedLastPageIndex = (i < pageCount - 1) ? (offset_size_rowIndex_triplets[3 * i + 5] - 1) : 999;
+ assertEquals("Invalid lastRowIndex at page " + i, expectedLastPageIndex, offsetIndex.getLastRowIndex(i, 1000));
+ }
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/filter2/columnindex/TestColumnIndexFilter.java b/parquet-column/src/test/java/org/apache/parquet/internal/filter2/columnindex/TestColumnIndexFilter.java
new file mode 100644
index 000000000..ae2721458
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/internal/filter2/columnindex/TestColumnIndexFilter.java
@@ -0,0 +1,464 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.filter2.columnindex;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.parquet.filter2.predicate.FilterApi.and;
+import static org.apache.parquet.filter2.predicate.FilterApi.binaryColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.booleanColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.doubleColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.eq;
+import static org.apache.parquet.filter2.predicate.FilterApi.gt;
+import static org.apache.parquet.filter2.predicate.FilterApi.gtEq;
+import static org.apache.parquet.filter2.predicate.FilterApi.intColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.lt;
+import static org.apache.parquet.filter2.predicate.FilterApi.ltEq;
+import static org.apache.parquet.filter2.predicate.FilterApi.notEq;
+import static org.apache.parquet.filter2.predicate.FilterApi.or;
+import static org.apache.parquet.filter2.predicate.FilterApi.userDefined;
+import static org.apache.parquet.filter2.predicate.LogicalInverter.invert;
+import static org.apache.parquet.internal.column.columnindex.BoundaryOrder.ASCENDING;
+import static org.apache.parquet.internal.column.columnindex.BoundaryOrder.DESCENDING;
+import static org.apache.parquet.internal.column.columnindex.BoundaryOrder.UNORDERED;
+import static org.apache.parquet.internal.filter2.columnindex.ColumnIndexFilter.calculateRowRanges;
+import static org.apache.parquet.io.api.Binary.fromString;
+import static org.apache.parquet.schema.OriginalType.UTF8;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
+import static org.apache.parquet.schema.Types.optional;
+import static org.junit.Assert.assertArrayEquals;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.LongStream;
+
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.filter2.compat.FilterCompat;
+import org.apache.parquet.filter2.predicate.Statistics;
+import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.internal.column.columnindex.BoundaryOrder;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndexBuilder;
+import org.apache.parquet.internal.column.columnindex.TestColumnIndexBuilder.BinaryUtf8StartsWithB;
+import org.apache.parquet.internal.column.columnindex.TestColumnIndexBuilder.DoubleIsInteger;
+import org.apache.parquet.internal.column.columnindex.TestColumnIndexBuilder.IntegerIsDivisableWith3;
+import org.apache.parquet.schema.PrimitiveType;
+import org.junit.Test;
+
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongList;
+
+/**
+ * Unit tests of {@link ColumnIndexFilter}
+ */
+public class TestColumnIndexFilter {
+ private static class CIBuilder {
+ private static final ByteBuffer EMPTY = ByteBuffer.wrap(new byte[0]);
+ private final PrimitiveType type;
+ private final BoundaryOrder order;
+ private List<Boolean> nullPages = new ArrayList<>();
+ private List<Long> nullCounts = new ArrayList<>();
+ private List<ByteBuffer> minValues = new ArrayList<>();
+ private List<ByteBuffer> maxValues = new ArrayList<>();
+
+ CIBuilder(PrimitiveType type, BoundaryOrder order) {
+ this.type = type;
+ this.order = order;
+ }
+
+ CIBuilder addNullPage(long nullCount) {
+ nullPages.add(true);
+ nullCounts.add(nullCount);
+ minValues.add(EMPTY);
+ maxValues.add(EMPTY);
+ return this;
+ }
+
+ CIBuilder addPage(long nullCount, int min, int max) {
+ nullPages.add(false);
+ nullCounts.add(nullCount);
+ minValues.add(ByteBuffer.wrap(BytesUtils.intToBytes(min)));
+ maxValues.add(ByteBuffer.wrap(BytesUtils.intToBytes(max)));
+ return this;
+ }
+
+ CIBuilder addPage(long nullCount, String min, String max) {
+ nullPages.add(false);
+ nullCounts.add(nullCount);
+ minValues.add(ByteBuffer.wrap(min.getBytes(UTF_8)));
+ maxValues.add(ByteBuffer.wrap(max.getBytes(UTF_8)));
+ return this;
+ }
+
+ CIBuilder addPage(long nullCount, double min, double max) {
+ nullPages.add(false);
+ nullCounts.add(nullCount);
+ minValues.add(ByteBuffer.wrap(BytesUtils.longToBytes(Double.doubleToLongBits(min))));
+ maxValues.add(ByteBuffer.wrap(BytesUtils.longToBytes(Double.doubleToLongBits(max))));
+ return this;
+ }
+
+ ColumnIndex build() {
+ return ColumnIndexBuilder.build(type, order, nullPages, nullCounts, minValues, maxValues);
+ }
+ }
+
+ private static class OIBuilder {
+ private final OffsetIndexBuilder builder = OffsetIndexBuilder.getBuilder();
+
+ OIBuilder addPage(long rowCount) {
+ builder.add(1234, rowCount);
+ return this;
+ }
+
+ OffsetIndex build() {
+ return builder.build();
+ }
+ }
+
+ public static class AnyInt extends UserDefinedPredicate<Integer> {
+
+ @Override
+ public boolean keep(Integer value) {
+ return true;
+ }
+
+ @Override
+ public boolean canDrop(Statistics<Integer> statistics) {
+ return false;
+ }
+
+ @Override
+ public boolean inverseCanDrop(Statistics<Integer> statistics) {
+ return true;
+ }
+
+ }
+
+ /**
+ * <pre>
+ * row column1 column2 column3 column4 (no column index)
+ * ------0------ ------0------ ------0------ ------0------
+ * 0. 1 Zulu 2.03
+ * ------1------ ------1------ ------1------ ------1------
+ * 1. 2 Yankee 4.67
+ * 2. 3 Xray 3.42
+ * 3. 4 Whiskey 8.71
+ * ------2------ ------2------
+ * 4. 5 Victor 0.56
+ * 5. 6 Uniform 4.30
+ * ------2------ ------3------
+ * 6. null null null
+ * ------2------ ------4------
+ * 7. 7 Tango 3.50
+ * ------3------
+ * 8. 7 null 3.14
+ * ------3------
+ * 9. 7 null null
+ * ------3------
+ * 10. null null 9.99
+ * ------4------
+ * 11. 8 Sierra 8.78
+ * ------5------
+ * 12. 9 Romeo 9.56
+ * 13. 10 Quebec 2.71
+ * ------4------
+ * 14. 11 Papa 5.71
+ * 15. 12 Oscar 4.09
+ * ------5------ ------4------ ------6------
+ * 16. 13 November null
+ * 17. 14 Mike null
+ * 18. 15 Lima 0.36
+ * 19. 16 Kilo 2.94
+ * 20. 17 Juliett 4.23
+ * ------5------ ------6------ ------7------
+ * 21. 18 India null
+ * 22. 19 Hotel 5.32
+ * ------5------
+ * 23. 20 Golf 4.17
+ * 24. 21 Foxtrot 7.92
+ * 25. 22 Echo 7.95
+ * ------6------
+ * 26. 23 Delta null
+ * ------6------
+ * 27. 24 Charlie null
+ * ------8------
+ * 28. 25 Bravo null
+ * ------7------
+ * 29. 26 Alfa null
+ * </pre>
+ */
+ private static final long TOTAL_ROW_COUNT = 30;
+ private static final ColumnIndex COLUMN1_CI = new CIBuilder(optional(INT32).named("column1"), ASCENDING)
+ .addPage(0, 1, 1)
+ .addPage(1, 2, 6)
+ .addPage(0, 7, 7)
+ .addPage(1, 7, 10)
+ .addPage(0, 11, 17)
+ .addPage(0, 18, 23)
+ .addPage(0, 24, 26)
+ .build();
+ private static final OffsetIndex COLUMN1_OI = new OIBuilder()
+ .addPage(1)
+ .addPage(6)
+ .addPage(2)
+ .addPage(5)
+ .addPage(7)
+ .addPage(6)
+ .addPage(3)
+ .build();
+ private static final ColumnIndex COLUMN2_CI = new CIBuilder(optional(BINARY).as(UTF8).named("column2"), DESCENDING)
+ .addPage(0, "Zulu", "Zulu")
+ .addPage(0, "Whiskey", "Yankee")
+ .addPage(1, "Tango", "Victor")
+ .addNullPage(3)
+ .addPage(0, "Oscar", "Sierra")
+ .addPage(0, "Juliett", "November")
+ .addPage(0, "Bravo", "India")
+ .addPage(0, "Alfa", "Alfa")
+ .build();
+ private static final OffsetIndex COLUMN2_OI = new OIBuilder()
+ .addPage(1)
+ .addPage(3)
+ .addPage(4)
+ .addPage(3)
+ .addPage(5)
+ .addPage(5)
+ .addPage(8)
+ .addPage(1)
+ .build();
+ private static final ColumnIndex COLUMN3_CI = new CIBuilder(optional(DOUBLE).named("column3"), UNORDERED)
+ .addPage(0, 2.03, 2.03)
+ .addPage(0, 0.56, 8.71)
+ .addPage(2, 3.14, 3.50)
+ .addPage(0, 2.71, 9.99)
+ .addPage(3, 0.36, 5.32)
+ .addPage(0, 4.17, 7.95)
+ .addNullPage(4)
+ .build();
+ private static final OffsetIndex COLUMN3_OI = new OIBuilder()
+ .addPage(1)
+ .addPage(5)
+ .addPage(4)
+ .addPage(6)
+ .addPage(7)
+ .addPage(3)
+ .addPage(4)
+ .build();
+ private static final ColumnIndex COLUMN4_CI = null;
+ private static final OffsetIndex COLUMN4_OI = new OIBuilder()
+ .addPage(1)
+ .addPage(3)
+ .addPage(2)
+ .addPage(1)
+ .addPage(5)
+ .addPage(4)
+ .addPage(5)
+ .addPage(7)
+ .addPage(2)
+ .build();
+ private static final ColumnIndexStore STORE = new ColumnIndexStore() {
+ @Override
+ public ColumnIndex getColumnIndex(ColumnPath column) {
+ switch (column.toDotString()) {
+ case "column1":
+ return COLUMN1_CI;
+ case "column2":
+ return COLUMN2_CI;
+ case "column3":
+ return COLUMN3_CI;
+ case "column4":
+ return COLUMN4_CI;
+ default:
+ return null;
+ }
+ }
+
+ @Override
+ public OffsetIndex getOffsetIndex(ColumnPath column) {
+ switch (column.toDotString()) {
+ case "column1":
+ return COLUMN1_OI;
+ case "column2":
+ return COLUMN2_OI;
+ case "column3":
+ return COLUMN3_OI;
+ case "column4":
+ return COLUMN4_OI;
+ default:
+ throw new MissingOffsetIndexException(column);
+ }
+ }
+ };
+
+ private static Set<ColumnPath> paths(String... columns) {
+ Set<ColumnPath> paths = new HashSet<>();
+ for (String column : columns) {
+ paths.add(ColumnPath.fromDotString(column));
+ }
+ return paths;
+ }
+
+ private static void assertAllRows(RowRanges ranges, long rowCount) {
+ LongList actualList = new LongArrayList();
+ ranges.iterator().forEachRemaining((long value) -> actualList.add(value));
+ LongList expectedList = new LongArrayList();
+ LongStream.range(0, rowCount).forEach(expectedList::add);
+ assertArrayEquals(expectedList + " != " + actualList, expectedList.toLongArray(), actualList.toLongArray());
+ }
+
+ private static void assertRows(RowRanges ranges, long... expectedRows) {
+ LongList actualList = new LongArrayList();
+ ranges.iterator().forEachRemaining((long value) -> actualList.add(value));
+ assertArrayEquals(Arrays.toString(expectedRows) + " != " + actualList, expectedRows, actualList.toLongArray());
+ }
+
+ @Test
+ public void testFiltering() {
+ Set<ColumnPath> paths = paths("column1", "column2", "column3", "column4");
+
+ assertAllRows(
+ calculateRowRanges(FilterCompat.get(
+ userDefined(intColumn("column1"), AnyInt.class)), STORE, paths, TOTAL_ROW_COUNT),
+ TOTAL_ROW_COUNT);
+ assertRows(calculateRowRanges(FilterCompat.get(
+ and(
+ and(
+ eq(intColumn("column1"), null),
+ eq(binaryColumn("column2"), null)),
+ and(
+ eq(doubleColumn("column3"), null),
+ eq(booleanColumn("column4"), null)))),
+ STORE, paths, TOTAL_ROW_COUNT),
+ 6, 9);
+ assertRows(calculateRowRanges(FilterCompat.get(
+ and(
+ and(
+ notEq(intColumn("column1"), null),
+ notEq(binaryColumn("column2"), null)),
+ and(
+ notEq(doubleColumn("column3"), null),
+ notEq(booleanColumn("column4"), null)))),
+ STORE, paths, TOTAL_ROW_COUNT),
+ 0, 1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25);
+ assertRows(calculateRowRanges(FilterCompat.get(
+ or(
+ and(
+ lt(intColumn("column1"), 20),
+ gtEq(binaryColumn("column2"), fromString("Quebec"))),
+ and(
+ gt(doubleColumn("column3"), 5.32),
+ ltEq(binaryColumn("column4"), fromString("XYZ"))))),
+ STORE, paths, TOTAL_ROW_COUNT),
+ 0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 23, 24, 25);
+ assertRows(calculateRowRanges(FilterCompat.get(
+ and(
+ and(
+ gtEq(intColumn("column1"), 7),
+ gt(binaryColumn("column2"), fromString("India"))),
+ and(
+ eq(doubleColumn("column3"), null),
+ notEq(binaryColumn("column4"), null)))),
+ STORE, paths, TOTAL_ROW_COUNT),
+ 7, 16, 17, 18, 19, 20);
+ assertRows(calculateRowRanges(FilterCompat.get(
+ and(
+ or(
+ invert(userDefined(intColumn("column1"), AnyInt.class)),
+ eq(binaryColumn("column2"), fromString("Echo"))),
+ eq(doubleColumn("column3"), 6.0))),
+ STORE, paths, TOTAL_ROW_COUNT),
+ 23, 24, 25);
+ assertRows(calculateRowRanges(FilterCompat.get(
+ and(
+ userDefined(intColumn("column1"), IntegerIsDivisableWith3.class),
+ and(
+ userDefined(binaryColumn("column2"), BinaryUtf8StartsWithB.class),
+ userDefined(doubleColumn("column3"), DoubleIsInteger.class)))),
+ STORE, paths, TOTAL_ROW_COUNT),
+ 21, 22, 23, 24, 25);
+ assertRows(calculateRowRanges(FilterCompat.get(
+ and(
+ and(
+ gtEq(intColumn("column1"), 7),
+ lt(intColumn("column1"), 11)),
+ and(
+ gt(binaryColumn("column2"), fromString("Romeo")),
+ ltEq(binaryColumn("column2"), fromString("Tango"))))),
+ STORE, paths, TOTAL_ROW_COUNT),
+ 7, 11, 12, 13);
+ }
+
+ @Test
+ public void testFilteringOnMissingColumns() {
+ Set<ColumnPath> paths = paths("column1", "column2", "column3", "column4");
+
+ // Missing column filter is always true
+ assertAllRows(calculateRowRanges(FilterCompat.get(
+ notEq(intColumn("missing_column"), 0)),
+ STORE, paths, TOTAL_ROW_COUNT),
+ TOTAL_ROW_COUNT);
+ assertRows(calculateRowRanges(FilterCompat.get(
+ and(
+ and(
+ gtEq(intColumn("column1"), 7),
+ lt(intColumn("column1"), 11)),
+ eq(binaryColumn("missing_column"), null))),
+ STORE, paths, TOTAL_ROW_COUNT),
+ 7, 8, 9, 10, 11, 12, 13);
+
+ // Missing column filter is always false
+ assertRows(calculateRowRanges(FilterCompat.get(
+ or(
+ and(
+ gtEq(intColumn("column1"), 7),
+ lt(intColumn("column1"), 11)),
+ notEq(binaryColumn("missing_column"), null))),
+ STORE, paths, TOTAL_ROW_COUNT),
+ 7, 8, 9, 10, 11, 12, 13);
+ assertRows(calculateRowRanges(FilterCompat.get(
+ gt(intColumn("missing_column"), 0)),
+ STORE, paths, TOTAL_ROW_COUNT));
+ }
+
+ @Test
+ public void testFilteringWithMissingOffsetIndex() {
+ Set<ColumnPath> paths = paths("column1", "column2", "column3", "column4", "column_wo_oi");
+
+ assertAllRows(calculateRowRanges(FilterCompat.get(
+ and(
+ and(
+ gtEq(intColumn("column1"), 7),
+ lt(intColumn("column1"), 11)),
+ and(
+ gt(binaryColumn("column2"), fromString("Romeo")),
+ ltEq(binaryColumn("column_wo_oi"), fromString("Tango"))))),
+ STORE, paths, TOTAL_ROW_COUNT),
+ TOTAL_ROW_COUNT);
+ }
+
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/filter2/columnindex/TestRowRanges.java b/parquet-column/src/test/java/org/apache/parquet/internal/filter2/columnindex/TestRowRanges.java
new file mode 100644
index 000000000..71b884499
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/internal/filter2/columnindex/TestRowRanges.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.filter2.columnindex;
+
+import static org.apache.parquet.internal.filter2.columnindex.RowRanges.intersection;
+import static org.apache.parquet.internal.filter2.columnindex.RowRanges.union;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.PrimitiveIterator;
+
+import org.apache.parquet.internal.column.columnindex.OffsetIndexBuilder;
+import org.junit.Test;
+
+import it.unimi.dsi.fastutil.longs.LongArrayList;
+import it.unimi.dsi.fastutil.longs.LongList;
+
+/**
+ * Unit test for {@link RowRanges}
+ */
+public class TestRowRanges {
+ private static RowRanges buildRanges(long... rowIndexes) {
+ if (rowIndexes.length == 0) {
+ return RowRanges.EMPTY;
+ }
+ OffsetIndexBuilder builder = OffsetIndexBuilder.getBuilder();
+ for (int i = 0, n = rowIndexes.length; i < n; i += 2) {
+ long from = rowIndexes[i];
+ long to = rowIndexes[i + 1];
+ builder.add(0, 0, from);
+ builder.add(0, 0, to + 1);
+ }
+ PrimitiveIterator.OfInt pageIndexes = new PrimitiveIterator.OfInt() {
+ private int index = 0;
+
+ @Override
+ public boolean hasNext() {
+ return index < rowIndexes.length;
+ }
+
+ @Override
+ public int nextInt() {
+ int ret = index;
+ index += 2;
+ return ret;
+ }
+ };
+ return RowRanges.create(rowIndexes[rowIndexes.length - 1], pageIndexes, builder.build());
+ }
+
+ private static void assertAllRowsEqual(PrimitiveIterator.OfLong actualIt, long... expectedValues) {
+ LongList actualList = new LongArrayList();
+ actualIt.forEachRemaining((long value) -> actualList.add(value));
+ assertArrayEquals(Arrays.toString(expectedValues) + "!= " + actualList, expectedValues, actualList.toLongArray());
+ }
+
+ @Test
+ public void testCreate() {
+ RowRanges ranges = buildRanges(
+ 1, 2,
+ 3, 4,
+ 6, 7,
+ 7, 10,
+ 15, 17);
+ assertAllRowsEqual(ranges.iterator(), 1, 2, 3, 4, 6, 7, 8, 9, 10, 15, 16, 17);
+ assertEquals(12, ranges.rowCount());
+ assertTrue(ranges.isOverlapping(4, 5));
+ assertFalse(ranges.isOverlapping(5, 5));
+ assertTrue(ranges.isOverlapping(10, 14));
+ assertFalse(ranges.isOverlapping(11, 14));
+ assertFalse(ranges.isOverlapping(18, Long.MAX_VALUE));
+
+ ranges = RowRanges.createSingle(5);
+ assertAllRowsEqual(ranges.iterator(), 0, 1, 2, 3, 4);
+ assertEquals(5, ranges.rowCount());
+ assertTrue(ranges.isOverlapping(0, 100));
+ assertFalse(ranges.isOverlapping(5, Long.MAX_VALUE));
+
+ ranges = RowRanges.EMPTY;
+ assertAllRowsEqual(ranges.iterator());
+ assertEquals(0, ranges.rowCount());
+ assertFalse(ranges.isOverlapping(0, Long.MAX_VALUE));
+ }
+
+ @Test
+ public void testUnion() {
+ RowRanges ranges1 = buildRanges(
+ 2, 5,
+ 7, 9,
+ 14, 14,
+ 20, 24);
+ RowRanges ranges2 = buildRanges(
+ 1, 2,
+ 4, 5,
+ 11, 12,
+ 14, 15,
+ 21, 22);
+ RowRanges empty = buildRanges();
+ assertAllRowsEqual(union(ranges1, ranges2).iterator(), 1, 2, 3, 4, 5, 7, 8, 9, 11, 12, 14, 15, 20, 21, 22, 23, 24);
+ assertAllRowsEqual(union(ranges2, ranges1).iterator(), 1, 2, 3, 4, 5, 7, 8, 9, 11, 12, 14, 15, 20, 21, 22, 23, 24);
+ assertAllRowsEqual(union(ranges1, ranges1).iterator(), 2, 3, 4, 5, 7, 8, 9, 14, 20, 21, 22, 23, 24);
+ assertAllRowsEqual(union(ranges1, empty).iterator(), 2, 3, 4, 5, 7, 8, 9, 14, 20, 21, 22, 23, 24);
+ assertAllRowsEqual(union(empty, ranges1).iterator(), 2, 3, 4, 5, 7, 8, 9, 14, 20, 21, 22, 23, 24);
+ assertAllRowsEqual(union(ranges2, ranges2).iterator(), 1, 2, 4, 5, 11, 12, 14, 15, 21, 22);
+ assertAllRowsEqual(union(ranges2, empty).iterator(), 1, 2, 4, 5, 11, 12, 14, 15, 21, 22);
+ assertAllRowsEqual(union(empty, ranges2).iterator(), 1, 2, 4, 5, 11, 12, 14, 15, 21, 22);
+ assertAllRowsEqual(union(empty, empty).iterator());
+ }
+
+ @Test
+ public void testIntersection() {
+ RowRanges ranges1 = buildRanges(
+ 2, 5,
+ 7, 9,
+ 14, 14,
+ 20, 24);
+ RowRanges ranges2 = buildRanges(
+ 1, 2,
+ 6, 7,
+ 9, 9,
+ 11, 12,
+ 14, 15,
+ 21, 22);
+ RowRanges empty = buildRanges();
+ assertAllRowsEqual(intersection(ranges1, ranges2).iterator(), 2, 7, 9, 14, 21, 22);
+ assertAllRowsEqual(intersection(ranges2, ranges1).iterator(), 2, 7, 9, 14, 21, 22);
+ assertAllRowsEqual(intersection(ranges1, ranges1).iterator(), 2, 3, 4, 5, 7, 8, 9, 14, 20, 21, 22, 23, 24);
+ assertAllRowsEqual(intersection(ranges1, empty).iterator());
+ assertAllRowsEqual(intersection(empty, ranges1).iterator());
+ assertAllRowsEqual(intersection(ranges2, ranges2).iterator(), 1, 2, 6, 7, 9, 11, 12, 14, 15, 21, 22);
+ assertAllRowsEqual(intersection(ranges2, empty).iterator());
+ assertAllRowsEqual(intersection(empty, ranges2).iterator());
+ assertAllRowsEqual(intersection(empty, empty).iterator());
+ }
+
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/parser/TestParquetParser.java b/parquet-column/src/test/java/org/apache/parquet/parser/TestParquetParser.java
index d8536012b..fa200ab42 100644
--- a/parquet-column/src/test/java/org/apache/parquet/parser/TestParquetParser.java
+++ b/parquet-column/src/test/java/org/apache/parquet/parser/TestParquetParser.java
@@ -19,6 +19,7 @@
package org.apache.parquet.parser;
import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MILLIS;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.NANOS;
import static org.apache.parquet.schema.LogicalTypeAnnotation.intType;
import static org.apache.parquet.schema.LogicalTypeAnnotation.stringType;
import static org.apache.parquet.schema.LogicalTypeAnnotation.timeType;
@@ -271,7 +272,9 @@ public void testTimeAnnotations() {
" required int64 timestamp (TIMESTAMP_MILLIS);" +
" required FIXED_LEN_BYTE_ARRAY(12) interval (INTERVAL);" +
" required int32 newTime (TIME(MILLIS,true));" +
+ " required int64 nanoTime (TIME(NANOS,true));" +
" required int64 newTimestamp (TIMESTAMP(MILLIS,false));" +
+ " required int64 nanoTimestamp (TIMESTAMP(NANOS,false));" +
"}\n";
MessageType parsed = MessageTypeParser.parseMessageType(message);
@@ -281,7 +284,9 @@ public void testTimeAnnotations() {
.required(INT64).as(TIMESTAMP_MILLIS).named("timestamp")
.required(FIXED_LEN_BYTE_ARRAY).length(12).as(INTERVAL).named("interval")
.required(INT32).as(timeType(true, MILLIS)).named("newTime")
+ .required(INT64).as(timeType(true, NANOS)).named("nanoTime")
.required(INT64).as(timestampType(false, MILLIS)).named("newTimestamp")
+ .required(INT64).as(timestampType(false, NANOS)).named("nanoTimestamp")
.named("TimeMessage");
assertEquals(expected, parsed);
diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
index 53045cfb8..b5de4f850 100644
--- a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
+++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
@@ -19,15 +19,26 @@
package org.apache.parquet.schema;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Arrays.asList;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.parquet.schema.PrimitiveStringifier.DATE_STRINGIFIER;
import static org.apache.parquet.schema.PrimitiveStringifier.DEFAULT_STRINGIFIER;
import static org.apache.parquet.schema.PrimitiveStringifier.INTERVAL_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_MICROS_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_MICROS_UTC_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_MILLIS_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_MILLIS_UTC_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_NANOS_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_NANOS_UTC_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIME_NANOS_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIME_NANOS_UTC_STRINGIFIER;
import static org.apache.parquet.schema.PrimitiveStringifier.TIME_STRINGIFIER;
+import static org.apache.parquet.schema.PrimitiveStringifier.TIME_UTC_STRINGIFIER;
import static org.apache.parquet.schema.PrimitiveStringifier.UNSIGNED_STRINGIFIER;
import static org.apache.parquet.schema.PrimitiveStringifier.UTF8_STRINGIFIER;
import static org.junit.Assert.assertEquals;
@@ -35,7 +46,6 @@
import java.math.BigInteger;
import java.nio.ByteBuffer;
-import java.util.Arrays;
import java.util.Calendar;
import java.util.HashSet;
import java.util.Set;
@@ -152,74 +162,124 @@ public void testDateStringifier() {
assertEquals("2017-12-14", stringifier.stringify((int) MILLISECONDS.toDays(cal.getTimeInMillis())));
cal.clear();
- cal.set(1492, Calendar.AUGUST, 3);
- assertEquals("1492-08-03", stringifier.stringify((int) MILLISECONDS.toDays(cal.getTimeInMillis())));
+ cal.set(1583, Calendar.AUGUST, 3);
+ assertEquals("1583-08-03", stringifier.stringify((int) MILLISECONDS.toDays(cal.getTimeInMillis())));
checkThrowingUnsupportedException(stringifier, Integer.TYPE);
}
@Test
public void testTimestampMillisStringifier() {
- PrimitiveStringifier stringifier = PrimitiveStringifier.TIMESTAMP_MILLIS_STRINGIFIER;
+ for (PrimitiveStringifier stringifier : asList(TIMESTAMP_MILLIS_STRINGIFIER, TIMESTAMP_MILLIS_UTC_STRINGIFIER)) {
+ String timezoneAmendment = (stringifier == TIMESTAMP_MILLIS_STRINGIFIER ? "" : "+0000");
- assertEquals("1970-01-01T00:00:00.000", stringifier.stringify(0l));
+ assertEquals(withZoneString("1970-01-01T00:00:00.000", timezoneAmendment), stringifier.stringify(0l));
- Calendar cal = Calendar.getInstance(UTC);
- cal.clear();
- cal.set(2017, Calendar.DECEMBER, 15, 10, 9, 54);
- cal.set(Calendar.MILLISECOND, 120);
- assertEquals("2017-12-15T10:09:54.120", stringifier.stringify(cal.getTimeInMillis()));
+ Calendar cal = Calendar.getInstance(UTC);
+ cal.clear();
+ cal.set(2017, Calendar.DECEMBER, 15, 10, 9, 54);
+ cal.set(Calendar.MILLISECOND, 120);
+ assertEquals(withZoneString("2017-12-15T10:09:54.120", timezoneAmendment), stringifier.stringify(cal.getTimeInMillis()));
- cal.clear();
- cal.set(1948, Calendar.NOVEMBER, 23, 20, 19, 1);
- cal.set(Calendar.MILLISECOND, 9);
- assertEquals("1948-11-23T20:19:01.009", stringifier.stringify(cal.getTimeInMillis()));
+ cal.clear();
+ cal.set(1948, Calendar.NOVEMBER, 23, 20, 19, 1);
+ cal.set(Calendar.MILLISECOND, 9);
+ assertEquals(withZoneString("1948-11-23T20:19:01.009", timezoneAmendment), stringifier.stringify(cal.getTimeInMillis()));
- checkThrowingUnsupportedException(stringifier, Long.TYPE);
+ checkThrowingUnsupportedException(stringifier, Long.TYPE);
+ }
}
@Test
public void testTimestampMicrosStringifier() {
- PrimitiveStringifier stringifier = PrimitiveStringifier.TIMESTAMP_MICROS_STRINGIFIER;
+ for (PrimitiveStringifier stringifier : asList(TIMESTAMP_MICROS_STRINGIFIER, TIMESTAMP_MICROS_UTC_STRINGIFIER)) {
+ String timezoneAmendment = (stringifier == TIMESTAMP_MICROS_STRINGIFIER ? "" : "+0000");
- assertEquals("1970-01-01T00:00:00.000000", stringifier.stringify(0l));
+ assertEquals(withZoneString("1970-01-01T00:00:00.000000", timezoneAmendment), stringifier.stringify(0l));
- Calendar cal = Calendar.getInstance(UTC);
- cal.clear();
- cal.set(2053, Calendar.JULY, 10, 22, 13, 24);
- cal.set(Calendar.MILLISECOND, 84);
- long micros = cal.getTimeInMillis() * 1000 + 900;
- assertEquals("2053-07-10T22:13:24.084900", stringifier.stringify(micros));
+ Calendar cal = Calendar.getInstance(UTC);
+ cal.clear();
+ cal.set(2053, Calendar.JULY, 10, 22, 13, 24);
+ cal.set(Calendar.MILLISECOND, 84);
+ long micros = cal.getTimeInMillis() * 1000 + 900;
+ assertEquals(withZoneString("2053-07-10T22:13:24.084900", timezoneAmendment), stringifier.stringify(micros));
- cal.clear();
- cal.set(1848, Calendar.MARCH, 15, 9, 23, 59);
- cal.set(Calendar.MILLISECOND, 765);
- micros = cal.getTimeInMillis() * 1000 - 1;
- assertEquals("1848-03-15T09:23:59.765001", stringifier.stringify(micros));
+ cal.clear();
+ cal.set(1848, Calendar.MARCH, 15, 9, 23, 59);
+ cal.set(Calendar.MILLISECOND, 765);
+ micros = cal.getTimeInMillis() * 1000 - 1;
+ assertEquals(withZoneString("1848-03-15T09:23:59.764999", timezoneAmendment), stringifier.stringify(micros));
+
+ checkThrowingUnsupportedException(stringifier, Long.TYPE);
+ }
+ }
- checkThrowingUnsupportedException(stringifier, Long.TYPE);
+ @Test
+ public void testTimestampNanosStringifier() {
+ for (PrimitiveStringifier stringifier : asList(TIMESTAMP_NANOS_STRINGIFIER, TIMESTAMP_NANOS_UTC_STRINGIFIER)) {
+ String timezoneAmendment = (stringifier == TIMESTAMP_NANOS_STRINGIFIER ? "" : "+0000");
+
+ assertEquals(withZoneString("1970-01-01T00:00:00.000000000", timezoneAmendment), stringifier.stringify(0l));
+
+ Calendar cal = Calendar.getInstance(UTC);
+ cal.clear();
+ cal.set(2053, Calendar.JULY, 10, 22, 13, 24);
+ cal.set(Calendar.MILLISECOND, 84);
+ long nanos = cal.getTimeInMillis() * 1_000_000 + 536;
+ assertEquals(withZoneString("2053-07-10T22:13:24.084000536", timezoneAmendment), stringifier.stringify(nanos));
+
+ cal.clear();
+ cal.set(1848, Calendar.MARCH, 15, 9, 23, 59);
+ cal.set(Calendar.MILLISECOND, 765);
+ nanos = cal.getTimeInMillis() * 1_000_000 - 1;
+ assertEquals(withZoneString("1848-03-15T09:23:59.764999999", timezoneAmendment), stringifier.stringify(nanos));
+
+ checkThrowingUnsupportedException(stringifier, Long.TYPE);
+ }
}
@Test
public void testTimeStringifier() {
- PrimitiveStringifier stringifier = TIME_STRINGIFIER;
+ for (PrimitiveStringifier stringifier : asList(TIME_STRINGIFIER, TIME_UTC_STRINGIFIER)) {
+ String timezoneAmendment = (stringifier == TIME_STRINGIFIER ? "" : "+0000");
- assertEquals("00:00:00.000", stringifier.stringify(0));
- assertEquals("00:00:00.000000", stringifier.stringify(0l));
+ assertEquals(withZoneString("00:00:00.000", timezoneAmendment), stringifier.stringify(0));
+ assertEquals(withZoneString("00:00:00.000000", timezoneAmendment), stringifier.stringify(0l));
- assertEquals("12:34:56.789", stringifier.stringify((int) convert(MILLISECONDS, 12, 34, 56, 789)));
- assertEquals("12:34:56.789012", stringifier.stringify(convert(MICROSECONDS, 12, 34, 56, 789012)));
+ assertEquals(withZoneString("12:34:56.789", timezoneAmendment), stringifier.stringify((int) convert(MILLISECONDS, 12, 34, 56, 789)));
+ assertEquals(withZoneString("12:34:56.789012", timezoneAmendment), stringifier.stringify(convert(MICROSECONDS, 12, 34, 56, 789012)));
- assertEquals("-12:34:56.789", stringifier.stringify((int) convert(MILLISECONDS, -12, -34, -56, -789)));
- assertEquals("-12:34:56.789012", stringifier.stringify(convert(MICROSECONDS, -12, -34, -56, -789012)));
+ assertEquals(withZoneString("-12:34:56.789", timezoneAmendment), stringifier.stringify((int) convert(MILLISECONDS, -12, -34, -56, -789)));
+ assertEquals(withZoneString("-12:34:56.789012", timezoneAmendment), stringifier.stringify(convert(MICROSECONDS, -12, -34, -56, -789012)));
- assertEquals("123:12:34.567", stringifier.stringify((int) convert(MILLISECONDS, 123, 12, 34, 567)));
- assertEquals("12345:12:34.056789", stringifier.stringify(convert(MICROSECONDS, 12345, 12, 34, 56789)));
+ assertEquals(withZoneString("123:12:34.567", timezoneAmendment), stringifier.stringify((int) convert(MILLISECONDS, 123, 12, 34, 567)));
+ assertEquals(withZoneString("12345:12:34.056789", timezoneAmendment), stringifier.stringify(convert(MICROSECONDS, 12345, 12, 34, 56789)));
- assertEquals("-123:12:34.567", stringifier.stringify((int) convert(MILLISECONDS, -123, -12, -34, -567)));
- assertEquals("-12345:12:34.056789", stringifier.stringify(convert(MICROSECONDS, -12345, -12, -34, -56789)));
+ assertEquals(withZoneString("-123:12:34.567", timezoneAmendment), stringifier.stringify((int) convert(MILLISECONDS, -123, -12, -34, -567)));
+ assertEquals(withZoneString("-12345:12:34.056789", timezoneAmendment), stringifier.stringify(convert(MICROSECONDS, -12345, -12, -34, -56789)));
- checkThrowingUnsupportedException(stringifier, Integer.TYPE, Long.TYPE);
+ checkThrowingUnsupportedException(stringifier, Integer.TYPE, Long.TYPE);
+ }
+ }
+
+ @Test
+ public void testTimeNanoStringifier() {
+ for (PrimitiveStringifier stringifier : asList(TIME_NANOS_STRINGIFIER, TIME_NANOS_UTC_STRINGIFIER)) {
+ String timezoneAmendment = (stringifier == TIME_NANOS_STRINGIFIER ? "" : "+0000");
+
+ assertEquals(withZoneString("00:00:00.000000000", timezoneAmendment), stringifier.stringify(0l));
+
+ assertEquals(withZoneString("12:34:56.789012987", timezoneAmendment), stringifier.stringify(convert(NANOSECONDS, 12, 34, 56, 789012987)));
+ assertEquals(withZoneString("-12:34:56.000789012", timezoneAmendment), stringifier.stringify(convert(NANOSECONDS, -12, -34, -56, -789012)));
+ assertEquals(withZoneString("12345:12:34.000056789", timezoneAmendment), stringifier.stringify(convert(NANOSECONDS, 12345, 12, 34, 56789)));
+ assertEquals(withZoneString("-12345:12:34.000056789", timezoneAmendment), stringifier.stringify(convert(NANOSECONDS, -12345, -12, -34, -56789)));
+
+ checkThrowingUnsupportedException(stringifier, Integer.TYPE, Long.TYPE);
+ }
+ }
+
+ private String withZoneString(String expected, String zoneString) {
+ return expected + zoneString;
}
private long convert(TimeUnit unit, long hours, long minutes, long seconds, long rest) {
@@ -250,7 +310,7 @@ public void testDecimalStringifier() {
}
private void checkThrowingUnsupportedException(PrimitiveStringifier stringifier, Class<?>... excludes) {
- Set<Class<?>> set = new HashSet<>(Arrays.asList(excludes));
+ Set<Class<?>> set = new HashSet<>(asList(excludes));
if (!set.contains(Integer.TYPE)) {
try {
stringifier.stringify(0);
diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
new file mode 100644
index 000000000..fe13e604b
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.schema;
+
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Type.Repetition;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.Callable;
+
+import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MICROS;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MILLIS;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.NANOS;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.bsonType;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.dateType;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.decimalType;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.intType;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.jsonType;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.stringType;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.timeType;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT96;
+import static org.apache.parquet.schema.Type.Repetition.REQUIRED;
+
+public class TestTypeBuildersWithLogicalTypes {
+ @Test
+ public void testGroupTypeConstruction() {
+ PrimitiveType f1 = Types.required(BINARY).as(stringType()).named("f1");
+ PrimitiveType f2 = Types.required(INT32).named("f2");
+ PrimitiveType f3 = Types.optional(INT32).named("f3");
+ String name = "group";
+ for (Repetition repetition : Repetition.values()) {
+ GroupType expected = new GroupType(repetition, name,
+ f1,
+ new GroupType(repetition, "g1", f2, f3));
+ GroupType built = Types.buildGroup(repetition)
+ .addField(f1)
+ .group(repetition).addFields(f2, f3).named("g1")
+ .named(name);
+ Assert.assertEquals(expected, built);
+
+ switch (repetition) {
+ case REQUIRED:
+ built = Types.requiredGroup()
+ .addField(f1)
+ .requiredGroup().addFields(f2, f3).named("g1")
+ .named(name);
+ break;
+ case OPTIONAL:
+ built = Types.optionalGroup()
+ .addField(f1)
+ .optionalGroup().addFields(f2, f3).named("g1")
+ .named(name);
+ break;
+ case REPEATED:
+ built = Types.repeatedGroup()
+ .addField(f1)
+ .repeatedGroup().addFields(f2, f3).named("g1")
+ .named(name);
+ break;
+ }
+ Assert.assertEquals(expected, built);
+ }
+ }
+
+ @Test
+ public void testDecimalAnnotation() {
+ // int32 primitive type
+ MessageType expected = new MessageType("DecimalMessage",
+ new PrimitiveType(REQUIRED, INT32, 0, "aDecimal",
+ decimalType(2, 9), null));
+ MessageType builderType = Types.buildMessage()
+ .required(INT32)
+ .as(decimalType(2, 9))
+ .named("aDecimal")
+ .named("DecimalMessage");
+ Assert.assertEquals(expected, builderType);
+ // int64 primitive type
+ expected = new MessageType("DecimalMessage",
+ new PrimitiveType(REQUIRED, INT64, 0, "aDecimal",
+ decimalType(2, 18), null));
+ builderType = Types.buildMessage()
+ .required(INT64)
+ .as(decimalType(2, 18)).precision(18).scale(2)
+ .named("aDecimal")
+ .named("DecimalMessage");
+ Assert.assertEquals(expected, builderType);
+ // binary primitive type
+ expected = new MessageType("DecimalMessage",
+ new PrimitiveType(REQUIRED, BINARY, 0, "aDecimal",
+ decimalType(2, 9), null));
+ builderType = Types.buildMessage()
+ .required(BINARY).as(decimalType(2, 9))
+ .named("aDecimal")
+ .named("DecimalMessage");
+ Assert.assertEquals(expected, builderType);
+ // fixed primitive type
+ expected = new MessageType("DecimalMessage",
+ new PrimitiveType(REQUIRED, FIXED_LEN_BYTE_ARRAY, 4, "aDecimal",
+ decimalType(2, 9), null));
+ builderType = Types.buildMessage()
+ .required(FIXED_LEN_BYTE_ARRAY).length(4)
+ .as(decimalType(2, 9))
+ .named("aDecimal")
+ .named("DecimalMessage");
+ Assert.assertEquals(expected, builderType);
+ }
+
+ @Test
+ public void testDecimalAnnotationPrecisionScaleBound() {
+ assertThrows("Should reject scale greater than precision",
+ IllegalArgumentException.class, () -> Types.buildMessage()
+ .required(INT32).as(decimalType(4, 3))
+ .named("aDecimal")
+ .named("DecimalMessage"));
+ assertThrows("Should reject scale greater than precision",
+ IllegalArgumentException.class, () -> Types.buildMessage()
+ .required(INT64).as(decimalType(4, 3))
+ .named("aDecimal")
+ .named("DecimalMessage"));
+ assertThrows("Should reject scale greater than precision",
+ IllegalArgumentException.class, () -> Types.buildMessage()
+ .required(BINARY).as(decimalType(4, 3))
+ .named("aDecimal")
+ .named("DecimalMessage"));
+ assertThrows("Should reject scale greater than precision",
+ IllegalArgumentException.class, () -> Types.buildMessage()
+ .required(FIXED_LEN_BYTE_ARRAY).length(7)
+ .as(decimalType(4, 3))
+ .named("aDecimal")
+ .named("DecimalMessage")
+ );
+ }
+
+ @Test
+ public void testDecimalAnnotationLengthCheck() {
+ // maximum precision for 4 bytes is 9
+ assertThrows("should reject precision 10 with length 4",
+ IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(4)
+ .as(decimalType(2, 10))
+ .named("aDecimal"));
+ assertThrows("should reject precision 10 with length 4",
+ IllegalStateException.class, () -> Types.required(INT32)
+ .as(decimalType(2, 10))
+ .named("aDecimal"));
+ // maximum precision for 8 bytes is 19
+ assertThrows("should reject precision 19 with length 8",
+ IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(8)
+ .as(decimalType(4, 19))
+ .named("aDecimal"));
+ assertThrows("should reject precision 19 with length 8",
+ IllegalStateException.class, () -> Types.required(INT64).length(8)
+ .as(decimalType(4, 19))
+ .named("aDecimal")
+ );
+ }
+
+ @Test
+ public void testDECIMALAnnotationRejectsUnsupportedTypes() {
+ PrimitiveTypeName[] unsupported = new PrimitiveTypeName[]{
+ BOOLEAN, INT96, DOUBLE, FLOAT
+ };
+ for (final PrimitiveTypeName type : unsupported) {
+ assertThrows("Should reject non-binary type: " + type,
+ IllegalStateException.class, () -> Types.required(type)
+ .as(decimalType(2, 9))
+ .named("d"));
+ }
+ }
+
+ @Test
+ public void testBinaryAnnotations() {
+ LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
+ stringType(), jsonType(), bsonType()};
+ for (final LogicalTypeAnnotation logicalType : types) {
+ PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "col", logicalType);
+ PrimitiveType string = Types.required(BINARY).as(logicalType).named("col");
+ Assert.assertEquals(expected, string);
+ }
+ }
+
+ @Test
+ public void testBinaryAnnotationsRejectsNonBinary() {
+ LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
+ stringType(), jsonType(), bsonType()};
+ for (final LogicalTypeAnnotation logicalType : types) {
+ PrimitiveTypeName[] nonBinary = new PrimitiveTypeName[]{
+ BOOLEAN, INT32, INT64, INT96, DOUBLE, FLOAT
+ };
+ for (final PrimitiveTypeName type : nonBinary) {
+ assertThrows("Should reject non-binary type: " + type,
+ IllegalStateException.class, () -> Types.required(type).as(logicalType).named("col"));
+ }
+ assertThrows("Should reject non-binary type: FIXED_LEN_BYTE_ARRAY",
+ IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(1)
+ .as(logicalType).named("col"));
+ }
+ }
+
+ @Test
+ public void testInt32Annotations() {
+ LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
+ dateType(), timeType(true, MILLIS), timeType(false, MILLIS),
+ intType(8, false), intType(16, false), intType(32, false),
+ intType(8, true), intType(16, true), intType(32, true)};
+ for (LogicalTypeAnnotation logicalType : types) {
+ PrimitiveType expected = new PrimitiveType(REQUIRED, INT32, "col", logicalType);
+ PrimitiveType date = Types.required(INT32).as(logicalType).named("col");
+ Assert.assertEquals(expected, date);
+ }
+ }
+
+ @Test
+ public void testInt32AnnotationsRejectNonInt32() {
+ LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
+ dateType(), timeType(true, MILLIS), timeType(false, MILLIS),
+ intType(8, false), intType(16, false), intType(32, false),
+ intType(8, true), intType(16, true), intType(32, true)};
+ for (final LogicalTypeAnnotation logicalType : types) {
+ PrimitiveTypeName[] nonInt32 = new PrimitiveTypeName[]{
+ BOOLEAN, INT64, INT96, DOUBLE, FLOAT, BINARY
+ };
+ for (final PrimitiveTypeName type : nonInt32) {
+ assertThrows("Should reject non-int32 type: " + type,
+ IllegalStateException.class, () -> Types.required(type).as(logicalType).named("col"));
+ }
+ assertThrows("Should reject non-int32 type: FIXED_LEN_BYTE_ARRAY",
+ IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(1)
+ .as(logicalType).named("col"));
+ }
+ }
+
+ @Test
+ public void testInt64Annotations() {
+ LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
+ timeType(true, MICROS), timeType(false, MICROS),
+ timeType(true, NANOS), timeType(false, NANOS),
+ timestampType(true, MILLIS), timestampType(false, MILLIS),
+ timestampType(true, MICROS), timestampType(false, MICROS),
+ timestampType(true, NANOS), timestampType(false, NANOS),
+ intType(64, true), intType(64, false)};
+ for (LogicalTypeAnnotation logicalType : types) {
+ PrimitiveType expected = new PrimitiveType(REQUIRED, INT64, "col", logicalType);
+ PrimitiveType date = Types.required(INT64).as(logicalType).named("col");
+ Assert.assertEquals(expected, date);
+ }
+ }
+
+ @Test
+ public void testInt64AnnotationsRejectNonInt64() {
+ LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] {
+ timeType(true, MICROS), timeType(false, MICROS),
+ timeType(true, NANOS), timeType(false, NANOS),
+ timestampType(true, MILLIS), timestampType(false, MILLIS),
+ timestampType(true, MICROS), timestampType(false, MICROS),
+ timestampType(true, NANOS), timestampType(false, NANOS),
+ intType(64, true), intType(64, false)};
+ for (final LogicalTypeAnnotation logicalType : types) {
+ PrimitiveTypeName[] nonInt64 = new PrimitiveTypeName[]{
+ BOOLEAN, INT32, INT96, DOUBLE, FLOAT, BINARY
+ };
+ for (final PrimitiveTypeName type : nonInt64) {
+ assertThrows("Should reject non-int64 type: " + type,
+ IllegalStateException.class, (Callable<Type>) () -> Types.required(type).as(logicalType).named("col"));
+ }
+ assertThrows("Should reject non-int64 type: FIXED_LEN_BYTE_ARRAY",
+ IllegalStateException.class, (Callable<Type>) () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(1)
+ .as(logicalType).named("col"));
+ }
+ }
+
+ @Test
+ public void testIntervalAnnotationRejectsNonFixed() {
+ PrimitiveTypeName[] nonFixed = new PrimitiveTypeName[]{
+ BOOLEAN, INT32, INT64, INT96, DOUBLE, FLOAT, BINARY
+ };
+ for (final PrimitiveTypeName type : nonFixed) {
+ assertThrows("Should reject non-fixed type: " + type,
+ IllegalStateException.class, () -> Types.required(type)
+ .as(LogicalTypeAnnotation.IntervalLogicalTypeAnnotation.getInstance()).named("interval"));
+ }
+ }
+
+ @Test
+ public void testIntervalAnnotationRejectsNonFixed12() {
+ assertThrows("Should reject fixed with length != 12: " + 11,
+ IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(11)
+ .as(LogicalTypeAnnotation.IntervalLogicalTypeAnnotation.getInstance()).named("interval"));
+ }
+
+ @Test
+ public void testTypeConstructionWithUnsupportedColumnOrder() {
+ assertThrows(null, IllegalArgumentException.class,
+ () -> Types.optional(INT96).columnOrder(ColumnOrder.typeDefined()).named("int96_unsupported"));
+ assertThrows(null, IllegalArgumentException.class,
+ () -> Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(12)
+ .as(LogicalTypeAnnotation.IntervalLogicalTypeAnnotation.getInstance())
+ .columnOrder(ColumnOrder.typeDefined()).named("interval_unsupported"));
+ }
+
+ @Test
+ public void testDecimalLogicalType() {
+ PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "aDecimal",
+ LogicalTypeAnnotation.decimalType(3, 4));
+ PrimitiveType actual = Types.required(BINARY)
+ .as(LogicalTypeAnnotation.decimalType(3, 4)).named("aDecimal");
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testDecimalLogicalTypeWithDeprecatedScale() {
+ PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "aDecimal",
+ LogicalTypeAnnotation.decimalType(3, 4));
+ PrimitiveType actual = Types.required(BINARY)
+ .as(LogicalTypeAnnotation.decimalType(3, 4)).scale(3).named("aDecimal");
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testDecimalLogicalTypeWithDeprecatedPrecision() {
+ PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "aDecimal",
+ LogicalTypeAnnotation.decimalType(3, 4));
+ PrimitiveType actual = Types.required(BINARY)
+ .as(LogicalTypeAnnotation.decimalType(3, 4)).precision(4).named("aDecimal");
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testTimestampLogicalTypeWithUTCParameter() {
+ PrimitiveType utcMillisExpected = new PrimitiveType(REQUIRED, INT64, "aTimestamp",
+ timestampType(true, MILLIS));
+ PrimitiveType nonUtcMillisExpected = new PrimitiveType(REQUIRED, INT64, "aTimestamp",
+ timestampType(false, MILLIS));
+ PrimitiveType utcMicrosExpected = new PrimitiveType(REQUIRED, INT64, "aTimestamp",
+ timestampType(true, MICROS));
+ PrimitiveType nonUtcMicrosExpected = new PrimitiveType(REQUIRED, INT64, "aTimestamp",
+ timestampType(false, MICROS));
+
+ PrimitiveType utcMillisActual = Types.required(INT64)
+ .as(timestampType(true, MILLIS)).named("aTimestamp");
+ PrimitiveType nonUtcMillisActual = Types.required(INT64)
+ .as(timestampType(false, MILLIS)).named("aTimestamp");
+ PrimitiveType utcMicrosActual = Types.required(INT64)
+ .as(timestampType(true, MICROS)).named("aTimestamp");
+ PrimitiveType nonUtcMicrosActual = Types.required(INT64)
+ .as(timestampType(false, MICROS)).named("aTimestamp");
+
+ Assert.assertEquals(utcMillisExpected, utcMillisActual);
+ Assert.assertEquals(nonUtcMillisExpected, nonUtcMillisActual);
+ Assert.assertEquals(utcMicrosExpected, utcMicrosActual);
+ Assert.assertEquals(nonUtcMicrosExpected, nonUtcMicrosActual);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testDecimalLogicalTypeWithDeprecatedScaleMismatch() {
+ Types.required(BINARY)
+ .as(LogicalTypeAnnotation.decimalType(3, 4))
+ .scale(4).named("aDecimal");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testDecimalLogicalTypeWithDeprecatedPrecisionMismatch() {
+ Types.required(BINARY)
+ .as(LogicalTypeAnnotation.decimalType(3, 4))
+ .precision(5).named("aDecimal");
+ }
+
+ /**
+ * A convenience method to avoid a large number of @Test(expected=...) tests
+ * @param message A String message to describe this assertion
+ * @param expected An Exception class that the Runnable should throw
+ * @param callable A Callable that is expected to throw the exception
+ */
+ public static void assertThrows(
+ String message, Class<? extends Exception> expected, Callable callable) {
+ try {
+ callable.call();
+ Assert.fail("No exception was thrown (" + message + "), expected: " +
+ expected.getName());
+ } catch (Exception actual) {
+ Assert.assertEquals(message, expected, actual.getClass());
+ }
+ }
+}
diff --git a/parquet-common/pom.xml b/parquet-common/pom.xml
index f9a60a94b..100962854 100644
--- a/parquet-common/pom.xml
+++ b/parquet-common/pom.xml
@@ -61,6 +61,12 @@
<version>${slf4j.version}</version>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.yetus</groupId>
+ <artifactId>audience-annotations</artifactId>
+ <version>0.7.0</version>
+ </dependency>
</dependencies>
<build>
diff --git a/parquet-format-structures/src/main/java/org/apache/parquet/format/BlockCipher.java b/parquet-format-structures/src/main/java/org/apache/parquet/format/BlockCipher.java
new file mode 100755
index 000000000..6a18acedb
--- /dev/null
+++ b/parquet-format-structures/src/main/java/org/apache/parquet/format/BlockCipher.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.format;
+
+import java.io.IOException;
+
+public interface BlockCipher{
+
+ public interface Encryptor{
+ /**
+ * Encrypts the plaintext.
+ * Make sure the returned contents starts at offset 0 and fills up the byte array.
+ * Input plaintext starts at offset 0, and has a length of plaintext.length.
+ * @param plaintext
+ * @return ciphertext
+ * @throws IOException
+ */
+ public byte[] encrypt(byte[] plaintext) throws IOException;
+
+ /**
+ * Encrypts the plaintext.
+ * Make sure the returned contents starts at offset 0 and fills up the byte array.
+ * Input plaintext starts at offset, and has a length of len.
+ * @param plaintext
+ * @param offset
+ * @param len
+ * @return ciphertext
+ * @throws IOException
+ */
+ public byte[] encrypt(byte[] plaintext, int offset, int len) throws IOException;
+ }
+
+
+ public interface Decryptor{
+ /**
+ * Decrypts the ciphertext.
+ * Make sure the returned plaintext starts at offset 0 and and fills up the byte array.
+ * Input ciphertext starts at offset 0, and has a length of ciphertext.length.
+ * @param ciphertext
+ * @return plaintext
+ * @throws IOException
+ */
+ public byte[] decrypt(byte[] ciphertext) throws IOException;
+
+ /**
+ * Decrypts the ciphertext.
+ * Make sure the returned plaintext starts at offset 0 and and fills up the byte array.
+ * Input ciphertext starts at offset, and has a length of len.
+ * @param ciphertext
+ * @param offset
+ * @param len
+ * @return plaintext
+ * @throws IOException
+ */
+ public byte[] decrypt(byte[] ciphertext, int offset, int len) throws IOException;
+ }
+}
+
+
diff --git a/parquet-format-structures/src/main/java/org/apache/parquet/format/Util.java b/parquet-format-structures/src/main/java/org/apache/parquet/format/Util.java
index d09d007a2..bf838a9e1 100644
--- a/parquet-format-structures/src/main/java/org/apache/parquet/format/Util.java
+++ b/parquet-format-structures/src/main/java/org/apache/parquet/format/Util.java
@@ -30,6 +30,7 @@
import static org.apache.parquet.format.event.Consumers.listOf;
import static org.apache.parquet.format.event.Consumers.struct;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -40,7 +41,7 @@
import org.apache.thrift.protocol.TCompactProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TIOStreamTransport;
-
+import org.apache.thrift.transport.TMemoryBuffer;
import org.apache.parquet.format.event.Consumers.Consumer;
import org.apache.parquet.format.event.Consumers.DelegatingFieldConsumer;
import org.apache.parquet.format.event.EventBasedThriftReader;
@@ -55,36 +56,78 @@
public class Util {
public static void writeColumnIndex(ColumnIndex columnIndex, OutputStream to) throws IOException {
- write(columnIndex, to);
+ writeColumnIndex(columnIndex, to, (BlockCipher.Encryptor) null);
+ }
+
+ public static void writeColumnIndex(ColumnIndex columnIndex, OutputStream to, BlockCipher.Encryptor encryptor) throws IOException {
+ write(columnIndex, to, encryptor);
}
public static ColumnIndex readColumnIndex(InputStream from) throws IOException {
- return read(from, new ColumnIndex());
+ return readColumnIndex(from, (BlockCipher.Decryptor) null);
+ }
+
+ public static ColumnIndex readColumnIndex(InputStream from, BlockCipher.Decryptor decryptor) throws IOException {
+ return read(from, new ColumnIndex(), decryptor);
}
public static void writeOffsetIndex(OffsetIndex offsetIndex, OutputStream to) throws IOException {
- write(offsetIndex, to);
+ writeOffsetIndex(offsetIndex, to, (BlockCipher.Encryptor) null);
+ }
+
+ public static void writeOffsetIndex(OffsetIndex offsetIndex, OutputStream to, BlockCipher.Encryptor encryptor) throws IOException {
+ write(offsetIndex, to, encryptor);
}
public static OffsetIndex readOffsetIndex(InputStream from) throws IOException {
- return read(from, new OffsetIndex());
+ return readOffsetIndex(from, (BlockCipher.Decryptor) null);
+ }
+
+ public static OffsetIndex readOffsetIndex(InputStream from, BlockCipher.Decryptor decryptor) throws IOException {
+ return read(from, new OffsetIndex(), decryptor);
}
public static void writePageHeader(PageHeader pageHeader, OutputStream to) throws IOException {
- write(pageHeader, to);
+ writePageHeader(pageHeader, to, (BlockCipher.Encryptor) null);
+ }
+
+ public static void writePageHeader(PageHeader pageHeader, OutputStream to, BlockCipher.Encryptor encryptor) throws IOException {
+ write(pageHeader, to, encryptor);
}
public static PageHeader readPageHeader(InputStream from) throws IOException {
- return read(from, new PageHeader());
+ return readPageHeader(from, (BlockCipher.Decryptor) null);
+ }
+
+ public static PageHeader readPageHeader(InputStream from, BlockCipher.Decryptor decryptor) throws IOException {
+ return read(from, new PageHeader(), decryptor);
}
public static void writeFileMetaData(org.apache.parquet.format.FileMetaData fileMetadata, OutputStream to) throws IOException {
- write(fileMetadata, to);
+ writeFileMetaData(fileMetadata, to, (BlockCipher.Encryptor) null);
+ }
+
+ public static void writeFileMetaData(org.apache.parquet.format.FileMetaData fileMetadata, OutputStream to,
+ BlockCipher.Encryptor encryptor) throws IOException {
+ write(fileMetadata, to, encryptor);
}
public static FileMetaData readFileMetaData(InputStream from) throws IOException {
- return read(from, new FileMetaData());
+ return readFileMetaData(from, (BlockCipher.Decryptor) null);
+ }
+
+ public static FileMetaData readFileMetaData(InputStream from, BlockCipher.Decryptor decryptor) throws IOException {
+ return read(from, new FileMetaData(), decryptor);
+ }
+
+ public static void writeColumnMetaData(ColumnMetaData columnMetaData, OutputStream to, BlockCipher.Encryptor encryptor) throws IOException {
+ write(columnMetaData, to, encryptor);
+ }
+
+ public static ColumnMetaData readColumnMetaData(InputStream from, BlockCipher.Decryptor decryptor) throws IOException {
+ return read(from, new ColumnMetaData(), decryptor);
}
+
/**
* reads the meta data from the stream
* @param from the stream to read the metadata from
@@ -93,18 +136,28 @@ public static FileMetaData readFileMetaData(InputStream from) throws IOException
* @throws IOException if any I/O error occurs during the reading
*/
public static FileMetaData readFileMetaData(InputStream from, boolean skipRowGroups) throws IOException {
+ return readFileMetaData(from, skipRowGroups, (BlockCipher.Decryptor) null);
+ }
+ public static FileMetaData readFileMetaData(InputStream from, boolean skipRowGroups, BlockCipher.Decryptor decryptor) throws IOException {
FileMetaData md = new FileMetaData();
if (skipRowGroups) {
- readFileMetaData(from, new DefaultFileMetaDataConsumer(md), skipRowGroups);
+ readFileMetaData(from, new DefaultFileMetaDataConsumer(md), skipRowGroups, decryptor);
} else {
- read(from, md);
+ read(from, md, decryptor);
}
return md;
}
+
+ public static void writeFileCryptoMetaData(org.apache.parquet.format.FileCryptoMetaData cryptoMetadata, OutputStream to) throws IOException {
+ write(cryptoMetadata, to, (BlockCipher.Encryptor) null);
+ }
+
+ public static FileCryptoMetaData readFileCryptoMetaData(InputStream from) throws IOException {
+ return read(from, new FileCryptoMetaData(), (BlockCipher.Decryptor) null);
+ }
/**
* To read metadata in a streaming fashion.
- *
*/
public static abstract class FileMetaDataConsumer {
abstract public void setVersion(int version);
@@ -117,7 +170,6 @@ public static FileMetaData readFileMetaData(InputStream from, boolean skipRowGro
/**
* Simple default consumer that sets the fields
- *
*/
public static final class DefaultFileMetaDataConsumer extends FileMetaDataConsumer {
private final FileMetaData md;
@@ -158,10 +210,18 @@ public void addKeyValueMetaData(KeyValue kv) {
}
public static void readFileMetaData(InputStream from, FileMetaDataConsumer consumer) throws IOException {
- readFileMetaData(from, consumer, false);
+ readFileMetaData(from, consumer, (BlockCipher.Decryptor) null);
}
-
+ public static void readFileMetaData(InputStream from, FileMetaDataConsumer consumer, BlockCipher.Decryptor decryptor) throws IOException {
+ readFileMetaData(from, consumer, false, decryptor);
+ }
+
public static void readFileMetaData(InputStream from, final FileMetaDataConsumer consumer, boolean skipRowGroups) throws IOException {
+ readFileMetaData(from, consumer, skipRowGroups, (BlockCipher.Decryptor) null);
+ }
+
+ public static void readFileMetaData(InputStream input, final FileMetaDataConsumer consumer,
+ boolean skipRowGroups, BlockCipher.Decryptor decryptor) throws IOException {
try {
DelegatingFieldConsumer eventConsumer = fieldConsumer()
.onField(VERSION, new I32Consumer() {
@@ -198,8 +258,9 @@ public void consume(RowGroup rowGroup) {
}
})));
}
+
+ InputStream from = (null == decryptor? input : decryptStream(input, decryptor));
new EventBasedThriftReader(protocol(from)).readStruct(eventConsumer);
-
} catch (TException e) {
throw new IOException("can not read FileMetaData: " + e.getMessage(), e);
}
@@ -216,8 +277,41 @@ private static TProtocol protocol(InputStream from) {
private static InterningProtocol protocol(TIOStreamTransport t) {
return new InterningProtocol(new TCompactProtocol(t));
}
+
+ private static InputStream decryptStream(InputStream from, BlockCipher.Decryptor decryptor) throws IOException {
+ byte[] i32rd = new byte[4];
+ int got = 0;
+ // Read the length of encrypted Thrift structure
+ while (got < 4) {
+ int n = from.read(i32rd, got, 4-got);
+ if (n <= 0) {
+ throw new IOException("Tried to read int (4 bytes), but only got " + got + " bytes.");
+ }
+ got += n;
+ }
+ int clen =
+ ((i32rd[3] & 0xff) << 24) |
+ ((i32rd[2] & 0xff) << 16) |
+ ((i32rd[1] & 0xff) << 8) |
+ ((i32rd[0] & 0xff));
+ if (clen < 1) throw new IOException("Wrong length of encrypted metadata: " + clen);
+ byte[] cbuf = new byte[clen];
+ got = 0;
+ // Read the encrypted structure contents
+ while (got < clen) {
+ int n = from.read(cbuf, got, clen - got);
+ if (n <= 0) {
+ throw new IOException("Tried to read " + clen + " bytes, but only got " + got + " bytes.");
+ }
+ got += n;
+ }
+ // Decrypt the structure contents
+ byte[] pbuf = decryptor.decrypt(cbuf, 0, clen);
+ return new ByteArrayInputStream(pbuf);
+ }
- private static <T extends TBase<?,?>> T read(InputStream from, T tbase) throws IOException {
+ private static <T extends TBase<?,?>> T read(InputStream input, T tbase, BlockCipher.Decryptor decryptor) throws IOException {
+ InputStream from = (null == decryptor? input : decryptStream(input, decryptor));
try {
tbase.read(protocol(from));
return tbase;
@@ -226,11 +320,36 @@ private static InterningProtocol protocol(TIOStreamTransport t) {
}
}
- private static void write(TBase<?, ?> tbase, OutputStream to) throws IOException {
+ private static void write(TBase<?, ?> tbase, OutputStream to, BlockCipher.Encryptor encryptor) throws IOException {
+ if (null == encryptor) {
+ try {
+ tbase.write(protocol(to));
+ return;
+ } catch (TException e) {
+ throw new IOException("can not write " + tbase, e);
+ }
+ }
+ // Serialize and encrypt the structure
+ TMemoryBuffer tmb = new TMemoryBuffer(100);
try {
- tbase.write(protocol(to));
+ tbase.write(new InterningProtocol(new TCompactProtocol(tmb)));
} catch (TException e) {
throw new IOException("can not write " + tbase, e);
}
+ byte[] pbuf = tmb.getArray();
+ int plen = tmb.length();
+ byte[] cbuf = encryptor.encrypt(pbuf, 0, plen);
+ int clen = cbuf.length;
+ // Serialize the length of encrypted structure
+ byte[] i32out = new byte[4];
+ i32out[3] = (byte)(0xff & (clen >> 24));
+ i32out[2] = (byte)(0xff & (clen >> 16));
+ i32out[1] = (byte)(0xff & (clen >> 8));
+ i32out[0] = (byte)(0xff & (clen));
+ // Write and flush
+ to.write(i32out, 0, 4);
+ to.write(cbuf, 0, clen);
+ to.flush();
}
}
+
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/HadoopReadOptions.java b/parquet-hadoop/src/main/java/org/apache/parquet/HadoopReadOptions.java
index b8f481e8a..4f5c78adb 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/HadoopReadOptions.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/HadoopReadOptions.java
@@ -28,6 +28,7 @@
import java.util.Map;
+import static org.apache.parquet.hadoop.ParquetInputFormat.COLUMN_INDEX_FILTERING_ENABLED;
import static org.apache.parquet.hadoop.ParquetInputFormat.DICTIONARY_FILTERING_ENABLED;
import static org.apache.parquet.hadoop.ParquetInputFormat.RECORD_FILTERING_ENABLED;
import static org.apache.parquet.hadoop.ParquetInputFormat.STATS_FILTERING_ENABLED;
@@ -43,6 +44,7 @@ private HadoopReadOptions(boolean useSignedStringMinMax,
boolean useStatsFilter,
boolean useDictionaryFilter,
boolean useRecordFilter,
+ boolean useColumnIndexFilter,
FilterCompat.Filter recordFilter,
MetadataFilter metadataFilter,
CompressionCodecFactory codecFactory,
@@ -51,8 +53,8 @@ private HadoopReadOptions(boolean useSignedStringMinMax,
Map<String, String> properties,
Configuration conf) {
super(
- useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, recordFilter,
- metadataFilter, codecFactory, allocator, maxAllocationSize, properties
+ useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, useColumnIndexFilter,
+ recordFilter, metadataFilter, codecFactory, allocator, maxAllocationSize, properties
);
this.conf = conf;
}
@@ -83,6 +85,7 @@ public Builder(Configuration conf) {
useDictionaryFilter(conf.getBoolean(DICTIONARY_FILTERING_ENABLED, true));
useStatsFilter(conf.getBoolean(STATS_FILTERING_ENABLED, true));
useRecordFilter(conf.getBoolean(RECORD_FILTERING_ENABLED, true));
+ useColumnIndexFilter(conf.getBoolean(COLUMN_INDEX_FILTERING_ENABLED, true));
withCodecFactory(HadoopCodecs.newFactory(conf, 0));
withRecordFilter(getFilter(conf));
withMaxAllocationInBytes(conf.getInt(ALLOCATION_SIZE, 8388608));
@@ -95,7 +98,7 @@ public Builder(Configuration conf) {
@Override
public ParquetReadOptions build() {
return new HadoopReadOptions(
- useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter,
+ useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, useColumnIndexFilter,
recordFilter, metadataFilter, codecFactory, allocator, maxAllocationSize, properties,
conf);
}
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/ParquetReadOptions.java b/parquet-hadoop/src/main/java/org/apache/parquet/ParquetReadOptions.java
index 4ef24601c..846d3bd80 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/ParquetReadOptions.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/ParquetReadOptions.java
@@ -38,12 +38,14 @@
private static final boolean RECORD_FILTERING_ENABLED_DEFAULT = true;
private static final boolean STATS_FILTERING_ENABLED_DEFAULT = true;
private static final boolean DICTIONARY_FILTERING_ENABLED_DEFAULT = true;
+ private static final boolean COLUMN_INDEX_FILTERING_ENABLED_DEFAULT = true;
private static final int ALLOCATION_SIZE_DEFAULT = 8388608; // 8MB
private final boolean useSignedStringMinMax;
private final boolean useStatsFilter;
private final boolean useDictionaryFilter;
private final boolean useRecordFilter;
+ private final boolean useColumnIndexFilter;
private final FilterCompat.Filter recordFilter;
private final ParquetMetadataConverter.MetadataFilter metadataFilter;
private final CompressionCodecFactory codecFactory;
@@ -55,6 +57,7 @@
boolean useStatsFilter,
boolean useDictionaryFilter,
boolean useRecordFilter,
+ boolean useColumnIndexFilter,
FilterCompat.Filter recordFilter,
ParquetMetadataConverter.MetadataFilter metadataFilter,
CompressionCodecFactory codecFactory,
@@ -65,6 +68,7 @@
this.useStatsFilter = useStatsFilter;
this.useDictionaryFilter = useDictionaryFilter;
this.useRecordFilter = useRecordFilter;
+ this.useColumnIndexFilter = useColumnIndexFilter;
this.recordFilter = recordFilter;
this.metadataFilter = metadataFilter;
this.codecFactory = codecFactory;
@@ -89,6 +93,10 @@ public boolean useRecordFilter() {
return useRecordFilter;
}
+ public boolean useColumnIndexFilter() {
+ return useColumnIndexFilter;
+ }
+
public FilterCompat.Filter getRecordFilter() {
return recordFilter;
}
@@ -134,6 +142,7 @@ public static Builder builder() {
protected boolean useStatsFilter = STATS_FILTERING_ENABLED_DEFAULT;
protected boolean useDictionaryFilter = DICTIONARY_FILTERING_ENABLED_DEFAULT;
protected boolean useRecordFilter = RECORD_FILTERING_ENABLED_DEFAULT;
+ protected boolean useColumnIndexFilter = COLUMN_INDEX_FILTERING_ENABLED_DEFAULT;
protected FilterCompat.Filter recordFilter = null;
protected ParquetMetadataConverter.MetadataFilter metadataFilter = NO_FILTER;
// the page size parameter isn't used when only using the codec factory to get decompressors
@@ -182,6 +191,15 @@ public Builder useRecordFilter() {
return this;
}
+ public Builder useColumnIndexFilter(boolean useColumnIndexFilter) {
+ this.useColumnIndexFilter = useColumnIndexFilter;
+ return this;
+ }
+
+ public Builder useColumnIndexFilter() {
+ return useColumnIndexFilter(true);
+ }
+
public Builder withRecordFilter(FilterCompat.Filter rowGroupFilter) {
this.recordFilter = rowGroupFilter;
return this;
@@ -239,7 +257,7 @@ public Builder copy(ParquetReadOptions options) {
public ParquetReadOptions build() {
return new ParquetReadOptions(
- useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter,
+ useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, useColumnIndexFilter,
recordFilter, metadataFilter, codecFactory, allocator, maxAllocationSize, properties);
}
}
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
index 9478e9420..58ae5039c 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
@@ -18,6 +18,9 @@
*/
package org.apache.parquet.format.converter;
+import static java.util.Optional.empty;
+
+import static java.util.Optional.empty;
import static java.util.Optional.of;
import static org.apache.parquet.format.Util.readFileMetaData;
import static org.apache.parquet.format.Util.writePageHeader;
@@ -53,6 +56,7 @@
import org.apache.parquet.format.MapType;
import org.apache.parquet.format.MicroSeconds;
import org.apache.parquet.format.MilliSeconds;
+import org.apache.parquet.format.NanoSeconds;
import org.apache.parquet.format.NullType;
import org.apache.parquet.format.PageEncodingStats;
import org.apache.parquet.format.StringType;
@@ -60,7 +64,9 @@
import org.apache.parquet.format.TimeUnit;
import org.apache.parquet.format.TimestampType;
import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.format.BoundaryOrder;
import org.apache.parquet.format.ColumnChunk;
+import org.apache.parquet.format.ColumnIndex;
import org.apache.parquet.format.ColumnMetaData;
import org.apache.parquet.format.ColumnOrder;
import org.apache.parquet.format.ConvertedType;
@@ -71,7 +77,9 @@
import org.apache.parquet.format.FieldRepetitionType;
import org.apache.parquet.format.FileMetaData;
import org.apache.parquet.format.KeyValue;
+import org.apache.parquet.format.OffsetIndex;
import org.apache.parquet.format.PageHeader;
+import org.apache.parquet.format.PageLocation;
import org.apache.parquet.format.PageType;
import org.apache.parquet.format.RowGroup;
import org.apache.parquet.format.SchemaElement;
@@ -83,6 +91,9 @@
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.column.EncodingStats;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder;
+import org.apache.parquet.internal.column.columnindex.OffsetIndexBuilder;
+import org.apache.parquet.internal.hadoop.metadata.IndexReference;
import org.apache.parquet.io.ParquetDecodingException;
import org.apache.parquet.schema.ColumnOrder.ColumnOrderName;
import org.apache.parquet.schema.GroupType;
@@ -249,7 +260,7 @@ LogicalType convertToLogicalType(LogicalTypeAnnotation logicalTypeAnnotation) {
}
ConvertedType convertToConvertedType(LogicalTypeAnnotation logicalTypeAnnotation) {
- return logicalTypeAnnotation.accept(CONVERTED_TYPE_CONVERTER_VISITOR).get();
+ return logicalTypeAnnotation.accept(CONVERTED_TYPE_CONVERTER_VISITOR).orElse(null);
}
static org.apache.parquet.format.TimeUnit convertUnit(LogicalTypeAnnotation.TimeUnit unit) {
@@ -258,6 +269,8 @@ ConvertedType convertToConvertedType(LogicalTypeAnnotation logicalTypeAnnotation
return org.apache.parquet.format.TimeUnit.MICROS(new MicroSeconds());
case MILLIS:
return org.apache.parquet.format.TimeUnit.MILLIS(new MilliSeconds());
+ case NANOS:
+ return TimeUnit.NANOS(new NanoSeconds());
default:
throw new RuntimeException("Unknown time unit " + unit);
}
@@ -301,6 +314,8 @@ ConvertedType convertToConvertedType(LogicalTypeAnnotation logicalTypeAnnotation
return of(ConvertedType.TIME_MILLIS);
case MICROS:
return of(ConvertedType.TIME_MICROS);
+ case NANOS:
+ return empty();
default:
throw new RuntimeException("Unknown converted type for " + timeLogicalType.toOriginalType());
}
@@ -313,6 +328,8 @@ ConvertedType convertToConvertedType(LogicalTypeAnnotation logicalTypeAnnotation
return of(ConvertedType.TIMESTAMP_MICROS);
case MILLIS:
return of(ConvertedType.TIMESTAMP_MILLIS);
+ case NANOS:
+ return empty();
default:
throw new RuntimeException("Unknown converted type for " + timestampLogicalType.toOriginalType());
}
@@ -449,6 +466,17 @@ private void addRowGroup(ParquetMetadata parquetMetadata, List<RowGroup> rowGrou
// columnChunk.meta_data.index_page_offset = ;
// columnChunk.meta_data.key_value_metadata = ; // nothing yet
+ IndexReference columnIndexRef = columnMetaData.getColumnIndexReference();
+ if (columnIndexRef != null) {
+ columnChunk.setColumn_index_offset(columnIndexRef.getOffset());
+ columnChunk.setColumn_index_length(columnIndexRef.getLength());
+ }
+ IndexReference offsetIndexRef = columnMetaData.getOffsetIndexReference();
+ if (offsetIndexRef != null) {
+ columnChunk.setOffset_index_offset(offsetIndexRef.getOffset());
+ columnChunk.setOffset_index_length(offsetIndexRef.getLength());
+ }
+
parquetColumns.add(columnChunk);
}
RowGroup rowGroup = new RowGroup(parquetColumns, block.getTotalByteSize(), block.getRowCount());
@@ -936,6 +964,8 @@ LogicalTypeAnnotation getLogicalTypeAnnotation(LogicalType type) {
return LogicalTypeAnnotation.TimeUnit.MICROS;
case MILLIS:
return LogicalTypeAnnotation.TimeUnit.MILLIS;
+ case NANOS:
+ return LogicalTypeAnnotation.TimeUnit.NANOS;
default:
throw new RuntimeException("Unknown time unit " + unit);
}
@@ -1158,6 +1188,8 @@ public ParquetMetadata fromParquetMetadata(FileMetaData parquetMetadata) throws
metaData.num_values,
metaData.total_compressed_size,
metaData.total_uncompressed_size);
+ column.setColumnIndexReference(toColumnIndexReference(columnChunk));
+ column.setOffsetIndexReference(toOffsetIndexReference(columnChunk));
// TODO
// index_page_offset
// key_value_metadata
@@ -1179,6 +1211,20 @@ public ParquetMetadata fromParquetMetadata(FileMetaData parquetMetadata) throws
blocks);
}
+ private static IndexReference toColumnIndexReference(ColumnChunk columnChunk) {
+ if (columnChunk.isSetColumn_index_offset() && columnChunk.isSetColumn_index_length()) {
+ return new IndexReference(columnChunk.getColumn_index_offset(), columnChunk.getColumn_index_length());
+ }
+ return null;
+ }
+
+ private static IndexReference toOffsetIndexReference(ColumnChunk columnChunk) {
+ if (columnChunk.isSetOffset_index_offset() && columnChunk.isSetOffset_index_length()) {
+ return new IndexReference(columnChunk.getOffset_index_offset(), columnChunk.getOffset_index_length());
+ }
+ return null;
+ }
+
private static ColumnPath getPath(ColumnMetaData metaData) {
String[] path = metaData.path_in_schema.toArray(new String[metaData.path_in_schema.size()]);
return ColumnPath.get(path);
@@ -1373,4 +1419,78 @@ public void writeDictionaryPageHeader(
writePageHeader(pageHeader, to);
}
+ private static BoundaryOrder toParquetBoundaryOrder(
+ org.apache.parquet.internal.column.columnindex.BoundaryOrder boundaryOrder) {
+ switch (boundaryOrder) {
+ case ASCENDING:
+ return BoundaryOrder.ASCENDING;
+ case DESCENDING:
+ return BoundaryOrder.DESCENDING;
+ case UNORDERED:
+ return BoundaryOrder.UNORDERED;
+ default:
+ throw new IllegalArgumentException("Unsupported boundary order: " + boundaryOrder);
+ }
+ }
+
+ private static org.apache.parquet.internal.column.columnindex.BoundaryOrder fromParquetBoundaryOrder(
+ BoundaryOrder boundaryOrder) {
+ switch (boundaryOrder) {
+ case ASCENDING:
+ return org.apache.parquet.internal.column.columnindex.BoundaryOrder.ASCENDING;
+ case DESCENDING:
+ return org.apache.parquet.internal.column.columnindex.BoundaryOrder.DESCENDING;
+ case UNORDERED:
+ return org.apache.parquet.internal.column.columnindex.BoundaryOrder.UNORDERED;
+ default:
+ throw new IllegalArgumentException("Unsupported boundary order: " + boundaryOrder);
+ }
+ }
+
+ public static ColumnIndex toParquetColumnIndex(PrimitiveType type,
+ org.apache.parquet.internal.column.columnindex.ColumnIndex columnIndex) {
+ if (!isMinMaxStatsSupported(type) || columnIndex == null) {
+ return null;
+ }
+ ColumnIndex parquetColumnIndex = new ColumnIndex(
+ columnIndex.getNullPages(),
+ columnIndex.getMinValues(),
+ columnIndex.getMaxValues(),
+ toParquetBoundaryOrder(columnIndex.getBoundaryOrder()));
+ parquetColumnIndex.setNull_counts(columnIndex.getNullCounts());
+ return parquetColumnIndex;
+ }
+
+ public static org.apache.parquet.internal.column.columnindex.ColumnIndex fromParquetColumnIndex(PrimitiveType type,
+ ColumnIndex parquetColumnIndex) {
+ if (!isMinMaxStatsSupported(type)) {
+ return null;
+ }
+ return ColumnIndexBuilder.build(type,
+ fromParquetBoundaryOrder(parquetColumnIndex.getBoundary_order()),
+ parquetColumnIndex.getNull_pages(),
+ parquetColumnIndex.getNull_counts(),
+ parquetColumnIndex.getMin_values(),
+ parquetColumnIndex.getMax_values());
+ }
+
+ public static OffsetIndex toParquetOffsetIndex(org.apache.parquet.internal.column.columnindex.OffsetIndex offsetIndex) {
+ List<PageLocation> pageLocations = new ArrayList<>(offsetIndex.getPageCount());
+ for (int i = 0, n = offsetIndex.getPageCount(); i < n; ++i) {
+ pageLocations.add(new PageLocation(
+ offsetIndex.getOffset(i),
+ offsetIndex.getCompressedPageSize(i),
+ offsetIndex.getFirstRowIndex(i)));
+ }
+ return new OffsetIndex(pageLocations);
+ }
+
+ public static org.apache.parquet.internal.column.columnindex.OffsetIndex fromParquetOffsetIndex(
+ OffsetIndex parquetOffsetIndex) {
+ OffsetIndexBuilder builder = OffsetIndexBuilder.getBuilder();
+ for (PageLocation pageLocation : parquetOffsetIndex.getPage_locations()) {
+ builder.add(pageLocation.getOffset(), pageLocation.getCompressed_page_size(), pageLocation.getFirst_row_index());
+ }
+ return builder.build();
+ }
}
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java
index 37dfd6d39..0dc71e074 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java
@@ -18,13 +18,17 @@
*/
package org.apache.parquet.hadoop;
+import static org.apache.parquet.Ints.checkedCast;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-
+import java.util.Optional;
+import java.util.PrimitiveIterator;
import org.apache.parquet.Ints;
+import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.page.DataPage;
import org.apache.parquet.column.page.DataPageV1;
@@ -33,9 +37,9 @@
import org.apache.parquet.column.page.DictionaryPageReadStore;
import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
-import org.apache.parquet.compression.CompressionCodecFactory;
import org.apache.parquet.compression.CompressionCodecFactory.BytesInputDecompressor;
-import org.apache.parquet.hadoop.CodecFactory.BytesDecompressor;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.internal.filter2.columnindex.RowRanges;
import org.apache.parquet.io.ParquetDecodingException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,8 +66,13 @@
private final long valueCount;
private final List<DataPage> compressedPages;
private final DictionaryPage compressedDictionaryPage;
+ // null means no page synchronization is required; firstRowIndex will not be returned by the pages
+ private final OffsetIndex offsetIndex;
+ private final long rowCount;
+ private int pageIndex = 0;
- ColumnChunkPageReader(BytesInputDecompressor decompressor, List<DataPage> compressedPages, DictionaryPage compressedDictionaryPage) {
+ ColumnChunkPageReader(BytesInputDecompressor decompressor, List<DataPage> compressedPages,
+ DictionaryPage compressedDictionaryPage, OffsetIndex offsetIndex, long rowCount) {
this.decompressor = decompressor;
this.compressedPages = new LinkedList<DataPage>(compressedPages);
this.compressedDictionaryPage = compressedDictionaryPage;
@@ -72,6 +81,8 @@
count += p.getValueCount();
}
this.valueCount = count;
+ this.offsetIndex = offsetIndex;
+ this.rowCount = rowCount;
}
@Override
@@ -85,18 +96,34 @@ public DataPage readPage() {
return null;
}
DataPage compressedPage = compressedPages.remove(0);
+ final int currentPageIndex = pageIndex++;
return compressedPage.accept(new DataPage.Visitor<DataPage>() {
@Override
public DataPage visit(DataPageV1 dataPageV1) {
try {
- return new DataPageV1(
- decompressor.decompress(dataPageV1.getBytes(), dataPageV1.getUncompressedSize()),
- dataPageV1.getValueCount(),
- dataPageV1.getUncompressedSize(),
- dataPageV1.getStatistics(),
- dataPageV1.getRlEncoding(),
- dataPageV1.getDlEncoding(),
- dataPageV1.getValueEncoding());
+ BytesInput decompressed = decompressor.decompress(dataPageV1.getBytes(), dataPageV1.getUncompressedSize());
+ if (offsetIndex == null) {
+ return new DataPageV1(
+ decompressed,
+ dataPageV1.getValueCount(),
+ dataPageV1.getUncompressedSize(),
+ dataPageV1.getStatistics(),
+ dataPageV1.getRlEncoding(),
+ dataPageV1.getDlEncoding(),
+ dataPageV1.getValueEncoding());
+ } else {
+ long firstRowIndex = offsetIndex.getFirstRowIndex(currentPageIndex);
+ return new DataPageV1(
+ decompressed,
+ dataPageV1.getValueCount(),
+ dataPageV1.getUncompressedSize(),
+ firstRowIndex,
+ checkedCast(offsetIndex.getLastRowIndex(currentPageIndex, rowCount) - firstRowIndex + 1),
+ dataPageV1.getStatistics(),
+ dataPageV1.getRlEncoding(),
+ dataPageV1.getDlEncoding(),
+ dataPageV1.getValueEncoding());
+ }
} catch (IOException e) {
throw new ParquetDecodingException("could not decompress page", e);
}
@@ -105,23 +132,49 @@ public DataPage visit(DataPageV1 dataPageV1) {
@Override
public DataPage visit(DataPageV2 dataPageV2) {
if (!dataPageV2.isCompressed()) {
- return dataPageV2;
+ if (offsetIndex == null) {
+ return dataPageV2;
+ } else {
+ return DataPageV2.uncompressed(
+ dataPageV2.getRowCount(),
+ dataPageV2.getNullCount(),
+ dataPageV2.getValueCount(),
+ offsetIndex.getFirstRowIndex(currentPageIndex),
+ dataPageV2.getRepetitionLevels(),
+ dataPageV2.getDefinitionLevels(),
+ dataPageV2.getDataEncoding(),
+ dataPageV2.getData(),
+ dataPageV2.getStatistics());
+ }
}
try {
int uncompressedSize = Ints.checkedCast(
dataPageV2.getUncompressedSize()
- - dataPageV2.getDefinitionLevels().size()
- - dataPageV2.getRepetitionLevels().size());
- return DataPageV2.uncompressed(
- dataPageV2.getRowCount(),
- dataPageV2.getNullCount(),
- dataPageV2.getValueCount(),
- dataPageV2.getRepetitionLevels(),
- dataPageV2.getDefinitionLevels(),
- dataPageV2.getDataEncoding(),
- decompressor.decompress(dataPageV2.getData(), uncompressedSize),
- dataPageV2.getStatistics()
- );
+ - dataPageV2.getDefinitionLevels().size()
+ - dataPageV2.getRepetitionLevels().size());
+ BytesInput decompressed = decompressor.decompress(dataPageV2.getData(), uncompressedSize);
+ if (offsetIndex == null) {
+ return DataPageV2.uncompressed(
+ dataPageV2.getRowCount(),
+ dataPageV2.getNullCount(),
+ dataPageV2.getValueCount(),
+ dataPageV2.getRepetitionLevels(),
+ dataPageV2.getDefinitionLevels(),
+ dataPageV2.getDataEncoding(),
+ decompressed,
+ dataPageV2.getStatistics());
+ } else {
+ return DataPageV2.uncompressed(
+ dataPageV2.getRowCount(),
+ dataPageV2.getNullCount(),
+ dataPageV2.getValueCount(),
+ offsetIndex.getFirstRowIndex(currentPageIndex),
+ dataPageV2.getRepetitionLevels(),
+ dataPageV2.getDefinitionLevels(),
+ dataPageV2.getDataEncoding(),
+ decompressed,
+ dataPageV2.getStatistics());
+ }
} catch (IOException e) {
throw new ParquetDecodingException("could not decompress page", e);
}
@@ -147,9 +200,16 @@ public DictionaryPage readDictionaryPage() {
private final Map<ColumnDescriptor, ColumnChunkPageReader> readers = new HashMap<ColumnDescriptor, ColumnChunkPageReader>();
private final long rowCount;
+ private final RowRanges rowRanges;
public ColumnChunkPageReadStore(long rowCount) {
this.rowCount = rowCount;
+ rowRanges = null;
+ }
+
+ ColumnChunkPageReadStore(RowRanges rowRanges) {
+ this.rowRanges = rowRanges;
+ rowCount = rowRanges.rowCount();
}
@Override
@@ -170,6 +230,11 @@ public DictionaryPage readDictionaryPage(ColumnDescriptor descriptor) {
return readers.get(descriptor).readDictionaryPage();
}
+ @Override
+ public Optional<PrimitiveIterator.OfLong> getRowIndexes() {
+ return rowRanges == null ? Optional.empty() : Optional.of(rowRanges.iterator());
+ }
+
void addColumn(ColumnDescriptor path, ColumnChunkPageReader reader) {
if (readers.put(path, reader) != null) {
throw new RuntimeException(path+ " was added twice");
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java
index 5349dc28a..85bdbdbd9 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java
@@ -37,6 +37,8 @@
import org.apache.parquet.column.statistics.Statistics;
import org.apache.parquet.format.converter.ParquetMetadataConverter;
import org.apache.parquet.hadoop.CodecFactory.BytesCompressor;
+import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder;
+import org.apache.parquet.internal.column.columnindex.OffsetIndexBuilder;
import org.apache.parquet.io.ParquetEncodingException;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.bytes.ByteBufferAllocator;
@@ -67,21 +69,38 @@
private Set<Encoding> dlEncodings = new HashSet<Encoding>();
private List<Encoding> dataEncodings = new ArrayList<Encoding>();
+ private ColumnIndexBuilder columnIndexBuilder;
+ private OffsetIndexBuilder offsetIndexBuilder;
private Statistics totalStatistics;
private final ByteBufferAllocator allocator;
private ColumnChunkPageWriter(ColumnDescriptor path,
BytesCompressor compressor,
- ByteBufferAllocator allocator) {
+ ByteBufferAllocator allocator,
+ int columnIndexTruncateLength) {
this.path = path;
this.compressor = compressor;
this.allocator = allocator;
this.buf = new ConcatenatingByteArrayCollector();
+ this.columnIndexBuilder = ColumnIndexBuilder.getBuilder(path.getPrimitiveType(), columnIndexTruncateLength);
+ this.offsetIndexBuilder = OffsetIndexBuilder.getBuilder();
+ }
+
+ @Override
+ @Deprecated
+ public void writePage(BytesInput bytesInput, int valueCount, Statistics<?> statistics, Encoding rlEncoding,
+ Encoding dlEncoding, Encoding valuesEncoding) throws IOException {
+ // Setting the builders to the no-op ones so no column/offset indexes will be written for this column chunk
+ columnIndexBuilder = ColumnIndexBuilder.getNoOpBuilder();
+ offsetIndexBuilder = OffsetIndexBuilder.getNoOpBuilder();
+
+ writePage(bytesInput, valueCount, -1, statistics, rlEncoding, dlEncoding, valuesEncoding);
}
@Override
public void writePage(BytesInput bytes,
int valueCount,
+ int rowCount,
Statistics statistics,
Encoding rlEncoding,
Encoding dlEncoding,
@@ -121,6 +140,9 @@ public void writePage(BytesInput bytes,
totalStatistics.mergeStatistics(statistics);
}
+ columnIndexBuilder.add(statistics);
+ offsetIndexBuilder.add(toIntWithCheck(tempOutputStream.size() + compressedSize), rowCount);
+
// by concatenating before collecting instead of collecting twice,
// we only allocate one buffer to copy into instead of multiple.
buf.collect(BytesInput.concat(BytesInput.from(tempOutputStream), compressedBytes));
@@ -166,6 +188,9 @@ public void writePageV2(
totalStatistics.mergeStatistics(statistics);
}
+ columnIndexBuilder.add(statistics);
+ offsetIndexBuilder.add(toIntWithCheck((long) tempOutputStream.size() + compressedSize), rowCount);
+
// by concatenating before collecting instead of collecting twice,
// we only allocate one buffer to copy into instead of multiple.
buf.collect(
@@ -193,14 +218,20 @@ public long getMemSize() {
}
public void writeToFileWriter(ParquetFileWriter writer) throws IOException {
- writer.startColumn(path, totalValueCount, compressor.getCodecName());
- if (dictionaryPage != null) {
- writer.writeDictionaryPage(dictionaryPage);
- // tracking the dictionary encoding is handled in writeDictionaryPage
- }
- writer.writeDataPages(buf, uncompressedLength, compressedLength, totalStatistics,
- rlEncodings, dlEncodings, dataEncodings);
- writer.endColumn();
+ writer.writeColumnChunk(
+ path,
+ totalValueCount,
+ compressor.getCodecName(),
+ dictionaryPage,
+ buf,
+ uncompressedLength,
+ compressedLength,
+ totalStatistics,
+ columnIndexBuilder,
+ offsetIndexBuilder,
+ rlEncodings,
+ dlEncodings,
+ dataEncodings);
if (LOG.isDebugEnabled()) {
LOG.debug(
String.format(
@@ -243,10 +274,11 @@ public String memUsageString(String prefix) {
private final Map<ColumnDescriptor, ColumnChunkPageWriter> writers = new HashMap<ColumnDescriptor, ColumnChunkPageWriter>();
private final MessageType schema;
- public ColumnChunkPageWriteStore(BytesCompressor compressor, MessageType schema, ByteBufferAllocator allocator) {
+ public ColumnChunkPageWriteStore(BytesCompressor compressor, MessageType schema, ByteBufferAllocator allocator,
+ int columnIndexTruncateLength) {
this.schema = schema;
for (ColumnDescriptor path : schema.getColumns()) {
- writers.put(path, new ColumnChunkPageWriter(path, compressor, allocator));
+ writers.put(path, new ColumnChunkPageWriter(path, compressor, allocator, columnIndexTruncateLength));
}
}
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnIndexFilterUtils.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnIndexFilterUtils.java
new file mode 100644
index 000000000..448515e2a
--- /dev/null
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnIndexFilterUtils.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.hadoop;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Formatter;
+import java.util.List;
+
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.internal.filter2.columnindex.RowRanges;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+
+/**
+ * Internal utility class to help at column index based filtering.
+ */
+class ColumnIndexFilterUtils {
+ static class OffsetRange {
+ private final long offset;
+ private long length;
+
+ private OffsetRange(long offset, int length) {
+ this.offset = offset;
+ this.length = length;
+ }
+
+ long getOffset() {
+ return offset;
+ }
+
+ long getLength() {
+ return length;
+ }
+
+ private boolean extend(long offset, int length) {
+ if (this.offset + this.length == offset) {
+ this.length += length;
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ private static class FilteredOffsetIndex implements OffsetIndex {
+ private final OffsetIndex offsetIndex;
+ private final int[] indexMap;
+
+ private FilteredOffsetIndex(OffsetIndex offsetIndex, int[] indexMap) {
+ this.offsetIndex = offsetIndex;
+ this.indexMap = indexMap;
+ }
+
+ @Override
+ public int getPageCount() {
+ return indexMap.length;
+ }
+
+ @Override
+ public long getOffset(int pageIndex) {
+ return offsetIndex.getOffset(indexMap[pageIndex]);
+ }
+
+ @Override
+ public int getCompressedPageSize(int pageIndex) {
+ return offsetIndex.getCompressedPageSize(indexMap[pageIndex]);
+ }
+
+ @Override
+ public long getFirstRowIndex(int pageIndex) {
+ return offsetIndex.getFirstRowIndex(indexMap[pageIndex]);
+ }
+
+ @Override
+ public long getLastRowIndex(int pageIndex, long totalRowCount) {
+ int nextIndex = indexMap[pageIndex] + 1;
+ return (nextIndex >= offsetIndex.getPageCount() ? totalRowCount : offsetIndex.getFirstRowIndex(nextIndex)) - 1;
+ }
+
+ @Override
+ public String toString() {
+ try (Formatter formatter = new Formatter()) {
+ formatter.format("%-12s %20s %16s %20s\n", "", "offset", "compressed size", "first row index");
+ for (int i = 0, n = offsetIndex.getPageCount(); i < n; ++i) {
+ int index = Arrays.binarySearch(indexMap, i);
+ boolean isHidden = index < 0;
+ formatter.format("%spage-%-5d %20d %16d %20d\n",
+ isHidden ? "- " : " ",
+ isHidden ? i : index,
+ offsetIndex.getOffset(i),
+ offsetIndex.getCompressedPageSize(i),
+ offsetIndex.getFirstRowIndex(i));
+ }
+ return formatter.toString();
+ }
+ }
+ }
+
+ /*
+ * Returns the filtered offset index containing only the pages which are overlapping with rowRanges.
+ */
+ static OffsetIndex filterOffsetIndex(OffsetIndex offsetIndex, RowRanges rowRanges, long totalRowCount) {
+ IntList indexMap = new IntArrayList();
+ for (int i = 0, n = offsetIndex.getPageCount(); i < n; ++i) {
+ long from = offsetIndex.getFirstRowIndex(i);
+ if (rowRanges.isOverlapping(from, offsetIndex.getLastRowIndex(i, totalRowCount))) {
+ indexMap.add(i);
+ }
+ }
+ return new FilteredOffsetIndex(offsetIndex, indexMap.toIntArray());
+ }
+
+ static List<OffsetRange> calculateOffsetRanges(OffsetIndex offsetIndex, ColumnChunkMetaData cm,
+ long firstPageOffset) {
+ List<OffsetRange> ranges = new ArrayList<>();
+ int n = offsetIndex.getPageCount();
+ if (n > 0) {
+ OffsetRange currentRange = null;
+
+ // Add a range for the dictionary page if required
+ long rowGroupOffset = cm.getStartingPos();
+ if (rowGroupOffset < firstPageOffset) {
+ currentRange = new OffsetRange(rowGroupOffset, (int) (firstPageOffset - rowGroupOffset));
+ ranges.add(currentRange);
+ }
+
+ for (int i = 0; i < n; ++i) {
+ long offset = offsetIndex.getOffset(i);
+ int length = offsetIndex.getCompressedPageSize(i);
+ if (currentRange == null || !currentRange.extend(offset, length)) {
+ currentRange = new OffsetRange(offset, length);
+ ranges.add(currentRange);
+ }
+ }
+ }
+ return ranges;
+ }
+}
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnIndexStoreImpl.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnIndexStoreImpl.java
new file mode 100644
index 000000000..684c5f211
--- /dev/null
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnIndexStoreImpl.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.hadoop;
+
+import static java.util.Collections.emptySet;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.internal.filter2.columnindex.ColumnIndexStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Internal implementation of {@link ColumnIndexStore}.
+ */
+class ColumnIndexStoreImpl implements ColumnIndexStore {
+
+ private interface IndexStore {
+ ColumnIndex getColumnIndex();
+
+ OffsetIndex getOffsetIndex();
+ }
+
+ private class IndexStoreImpl implements IndexStore {
+ private final ColumnChunkMetaData meta;
+ private ColumnIndex columnIndex;
+ private boolean columnIndexRead;
+ private final OffsetIndex offsetIndex;
+
+ IndexStoreImpl(ColumnChunkMetaData meta) {
+ this.meta = meta;
+ OffsetIndex oi;
+ try {
+ oi = reader.readOffsetIndex(meta);
+ } catch (IOException e) {
+ // If the I/O issue still stands it will fail the reading later;
+ // otherwise we fail the filtering only with a missing offset index.
+ LOGGER.warn("Unable to read offset index for column {}", meta.getPath(), e);
+ oi = null;
+ }
+ if (oi == null) {
+ throw new MissingOffsetIndexException(meta.getPath());
+ }
+ offsetIndex = oi;
+ }
+
+ @Override
+ public ColumnIndex getColumnIndex() {
+ if (!columnIndexRead) {
+ try {
+ columnIndex = reader.readColumnIndex(meta);
+ } catch (IOException e) {
+ // If the I/O issue still stands it will fail the reading later;
+ // otherwise we fail the filtering only with a missing column index.
+ LOGGER.warn("Unable to read column index for column {}", meta.getPath(), e);
+ }
+ columnIndexRead = true;
+ }
+ return columnIndex;
+ }
+
+ @Override
+ public OffsetIndex getOffsetIndex() {
+ return offsetIndex;
+ }
+ }
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ColumnIndexStoreImpl.class);
+ // Used for columns are not in this parquet file
+ private static final IndexStore MISSING_INDEX_STORE = new IndexStore() {
+ @Override
+ public ColumnIndex getColumnIndex() {
+ return null;
+ }
+
+ @Override
+ public OffsetIndex getOffsetIndex() {
+ return null;
+ }
+ };
+ private static final ColumnIndexStoreImpl EMPTY = new ColumnIndexStoreImpl(null, new BlockMetaData(), emptySet()) {
+ @Override
+ public ColumnIndex getColumnIndex(ColumnPath column) {
+ return null;
+ }
+
+ @Override
+ public OffsetIndex getOffsetIndex(ColumnPath column) {
+ throw new MissingOffsetIndexException(column);
+ }
+ };
+
+ private final ParquetFileReader reader;
+ private final Map<ColumnPath, IndexStore> store;
+
+ /*
+ * Creates a column index store which lazily reads column/offset indexes for the columns in paths. (paths are the set
+ * of columns used for the projection)
+ */
+ static ColumnIndexStore create(ParquetFileReader reader, BlockMetaData block, Set<ColumnPath> paths) {
+ try {
+ return new ColumnIndexStoreImpl(reader, block, paths);
+ } catch (MissingOffsetIndexException e) {
+ return EMPTY;
+ }
+ }
+
+ private ColumnIndexStoreImpl(ParquetFileReader reader, BlockMetaData block, Set<ColumnPath> paths) {
+ // TODO[GS]: Offset index for every paths will be required; pre-read the consecutive ones at once?
+ // TODO[GS]: Pre-read column index based on filter?
+ this.reader = reader;
+ Map<ColumnPath, IndexStore> store = new HashMap<>();
+ for (ColumnChunkMetaData column : block.getColumns()) {
+ ColumnPath path = column.getPath();
+ if (paths.contains(path)) {
+ store.put(path, new IndexStoreImpl(column));
+ }
+ }
+ this.store = store;
+ }
+
+ @Override
+ public ColumnIndex getColumnIndex(ColumnPath column) {
+ return store.getOrDefault(column, MISSING_INDEX_STORE).getColumnIndex();
+ }
+
+ @Override
+ public OffsetIndex getOffsetIndex(ColumnPath column) {
+ return store.getOrDefault(column, MISSING_INDEX_STORE).getOffsetIndex();
+ }
+}
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordReader.java
index a04887869..e57f3cbce 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordReader.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordReader.java
@@ -124,7 +124,7 @@ private void checkRead() throws IOException {
LOG.info("at row " + current + ". reading next block");
long t0 = System.currentTimeMillis();
- PageReadStore pages = reader.readNextRowGroup();
+ PageReadStore pages = reader.readNextFilteredRowGroup();
if (pages == null) {
throw new IOException("expecting more rows but reached last block. Read " + current + " out of " + total);
}
@@ -182,7 +182,7 @@ public void initialize(ParquetFileReader reader, ParquetReadOptions options) {
this.columnCount = requestedSchema.getPaths().size();
this.recordConverter = readSupport.prepareForRead(conf, fileMetadata, fileSchema, readContext);
this.strictTypeChecking = options.isEnabled(STRICT_TYPE_CHECKING, true);
- this.total = reader.getRecordCount();
+ this.total = reader.getFilteredRecordCount();
this.unmaterializableRecordCounter = new UnmaterializableRecordCounter(options, total);
this.filterRecords = options.useRecordFilter();
reader.setRequestedSchema(requestedSchema);
@@ -204,7 +204,7 @@ public void initialize(ParquetFileReader reader, Configuration configuration)
this.recordConverter = readSupport.prepareForRead(
configuration, fileMetadata, fileSchema, readContext);
this.strictTypeChecking = configuration.getBoolean(STRICT_TYPE_CHECKING, true);
- this.total = reader.getRecordCount();
+ this.total = reader.getFilteredRecordCount();
this.unmaterializableRecordCounter = new UnmaterializableRecordCounter(configuration, total);
this.filterRecords = configuration.getBoolean(RECORD_FILTERING_ENABLED, true);
reader.setRequestedSchema(requestedSchema);
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
index 9743f9ff3..d8af379d1 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
@@ -20,7 +20,6 @@
import static java.lang.Math.max;
import static java.lang.Math.min;
-import static java.lang.String.format;
import static org.apache.parquet.Preconditions.checkNotNull;
import java.io.IOException;
@@ -102,7 +101,8 @@ public ParquetMetadata getFooter() {
}
private void initStore() {
- pageStore = new ColumnChunkPageWriteStore(compressor, schema, props.getAllocator());
+ pageStore = new ColumnChunkPageWriteStore(compressor, schema, props.getAllocator(),
+ props.getColumnIndexTruncateLength());
columnStore = props.newColumnWriteStore(schema, pageStore);
MessageColumnIO columnIO = new ColumnIOFactory(validating).getColumnIO(schema);
this.recordConsumer = columnIO.getRecordWriter(columnStore);
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java
index 527c8313b..5352309b3 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java
@@ -23,6 +23,8 @@
import static org.apache.parquet.filter2.compat.RowGroupFilter.FilterLevel.STATISTICS;
import static org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER;
import static org.apache.parquet.format.converter.ParquetMetadataConverter.SKIP_ROW_GROUPS;
+import static org.apache.parquet.hadoop.ColumnIndexFilterUtils.calculateOffsetRanges;
+import static org.apache.parquet.hadoop.ColumnIndexFilterUtils.filterOffsetIndex;
import static org.apache.parquet.hadoop.ParquetFileWriter.MAGIC;
import static org.apache.parquet.hadoop.ParquetFileWriter.PARQUET_COMMON_METADATA_FILE;
import static org.apache.parquet.hadoop.ParquetFileWriter.PARQUET_METADATA_FILE;
@@ -47,29 +49,29 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
+import org.apache.parquet.HadoopReadOptions;
import org.apache.parquet.ParquetReadOptions;
import org.apache.parquet.bytes.ByteBufferInputStream;
-import org.apache.parquet.column.Encoding;
-import org.apache.parquet.column.page.DictionaryPageReadStore;
-import org.apache.parquet.column.page.PageReader;
-import org.apache.parquet.compression.CompressionCodecFactory.BytesInputDecompressor;
-import org.apache.parquet.filter2.compat.FilterCompat;
-import org.apache.parquet.filter2.compat.RowGroupFilter;
-
import org.apache.parquet.bytes.BytesInput;
import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.page.DataPage;
import org.apache.parquet.column.page.DataPageV1;
import org.apache.parquet.column.page.DataPageV2;
import org.apache.parquet.column.page.DictionaryPage;
+import org.apache.parquet.column.page.DictionaryPageReadStore;
+import org.apache.parquet.column.page.PageReader;
import org.apache.parquet.column.page.PageReadStore;
-import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.compression.CompressionCodecFactory.BytesInputDecompressor;
+import org.apache.parquet.filter2.compat.FilterCompat;
+import org.apache.parquet.filter2.compat.RowGroupFilter;
import org.apache.parquet.format.DataPageHeader;
import org.apache.parquet.format.DataPageHeaderV2;
import org.apache.parquet.format.DictionaryPageHeader;
@@ -78,19 +80,27 @@
import org.apache.parquet.format.converter.ParquetMetadataConverter;
import org.apache.parquet.format.converter.ParquetMetadataConverter.MetadataFilter;
import org.apache.parquet.hadoop.ColumnChunkPageReadStore.ColumnChunkPageReader;
+import org.apache.parquet.hadoop.ColumnIndexFilterUtils.OffsetRange;
import org.apache.parquet.hadoop.metadata.BlockMetaData;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.hadoop.metadata.FileMetaData;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.hadoop.util.HadoopInputFile;
-import org.apache.parquet.HadoopReadOptions;
import org.apache.parquet.hadoop.util.HiddenFileFilter;
-import org.apache.parquet.io.SeekableInputStream;
import org.apache.parquet.hadoop.util.counters.BenchmarkCounter;
-import org.apache.parquet.io.ParquetDecodingException;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.internal.filter2.columnindex.ColumnIndexFilter;
+import org.apache.parquet.internal.filter2.columnindex.ColumnIndexStore;
+import org.apache.parquet.internal.filter2.columnindex.RowRanges;
+import org.apache.parquet.internal.hadoop.metadata.IndexReference;
import org.apache.parquet.io.InputFile;
+import org.apache.parquet.io.ParquetDecodingException;
+import org.apache.parquet.io.SeekableInputStream;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType;
+import org.apache.yetus.audience.InterfaceAudience.Private;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -603,6 +613,8 @@ public static ParquetFileReader open(InputFile file, ParquetReadOptions options)
private final Map<ColumnPath, ColumnDescriptor> paths = new HashMap<>();
private final FileMetaData fileMetaData; // may be null
private final List<BlockMetaData> blocks;
+ private final List<ColumnIndexStore> blockIndexStores;
+ private final List<RowRanges> blockRowRanges;
// not final. in some cases, this may be lazily loaded for backward-compat.
private ParquetMetadata footer;
@@ -644,6 +656,8 @@ public ParquetFileReader(
this.f = file.newStream();
this.options = HadoopReadOptions.builder(configuration).build();
this.blocks = filterRowGroups(blocks);
+ this.blockIndexStores = listWithNulls(this.blocks.size());
+ this.blockRowRanges = listWithNulls(this.blocks.size());
for (ColumnDescriptor col : columns) {
paths.put(ColumnPath.get(col.getPath()), col);
}
@@ -678,6 +692,8 @@ public ParquetFileReader(Configuration conf, Path file, ParquetMetadata footer)
this.footer = footer;
this.fileMetaData = footer.getFileMetaData();
this.blocks = filterRowGroups(footer.getBlocks());
+ this.blockIndexStores = listWithNulls(this.blocks.size());
+ this.blockRowRanges = listWithNulls(this.blocks.size());
for (ColumnDescriptor col : footer.getFileMetaData().getSchema().getColumns()) {
paths.put(ColumnPath.get(col.getPath()), col);
}
@@ -698,11 +714,17 @@ public ParquetFileReader(InputFile file, ParquetReadOptions options) throws IOEx
}
this.fileMetaData = footer.getFileMetaData();
this.blocks = filterRowGroups(footer.getBlocks());
+ this.blockIndexStores = listWithNulls(this.blocks.size());
+ this.blockRowRanges = listWithNulls(this.blocks.size());
for (ColumnDescriptor col : footer.getFileMetaData().getSchema().getColumns()) {
paths.put(ColumnPath.get(col.getPath()), col);
}
}
+ private static <T> List<T> listWithNulls(int size) {
+ return Stream.generate(() -> (T) null).limit(size).collect(Collectors.toCollection(ArrayList<T>::new));
+ }
+
public ParquetMetadata getFooter() {
if (footer == null) {
try {
@@ -730,6 +752,17 @@ public long getRecordCount() {
return total;
}
+ long getFilteredRecordCount() {
+ if (!options.useColumnIndexFilter()) {
+ return getRecordCount();
+ }
+ long total = 0;
+ for (int i = 0, n = blocks.size(); i < n; ++i) {
+ total += getRowRanges(i).rowCount();
+ }
+ return total;
+ }
+
/**
* @return the path for this file
* @deprecated will be removed in 2.0.0; use {@link #getFile()} instead
@@ -792,30 +825,111 @@ public PageReadStore readNextRowGroup() throws IOException {
throw new RuntimeException("Illegal row group of 0 rows");
}
this.currentRowGroup = new ColumnChunkPageReadStore(block.getRowCount());
- // prepare the list of consecutive chunks to read them in one scan
- List<ConsecutiveChunkList> allChunks = new ArrayList<ConsecutiveChunkList>();
- ConsecutiveChunkList currentChunks = null;
+ // prepare the list of consecutive parts to read them in one scan
+ List<ConsecutivePartList> allParts = new ArrayList<ConsecutivePartList>();
+ ConsecutivePartList currentParts = null;
for (ColumnChunkMetaData mc : block.getColumns()) {
ColumnPath pathKey = mc.getPath();
BenchmarkCounter.incrementTotalBytes(mc.getTotalSize());
ColumnDescriptor columnDescriptor = paths.get(pathKey);
if (columnDescriptor != null) {
long startingPos = mc.getStartingPos();
- // first chunk or not consecutive => new list
- if (currentChunks == null || currentChunks.endPos() != startingPos) {
- currentChunks = new ConsecutiveChunkList(startingPos);
- allChunks.add(currentChunks);
+ // first part or not consecutive => new list
+ if (currentParts == null || currentParts.endPos() != startingPos) {
+ currentParts = new ConsecutivePartList(startingPos);
+ allParts.add(currentParts);
}
- currentChunks.addChunk(new ChunkDescriptor(columnDescriptor, mc, startingPos, (int)mc.getTotalSize()));
+ currentParts.addChunk(new ChunkDescriptor(columnDescriptor, mc, startingPos, (int)mc.getTotalSize()));
}
}
// actually read all the chunks
- for (ConsecutiveChunkList consecutiveChunks : allChunks) {
- final List<Chunk> chunks = consecutiveChunks.readAll(f);
- for (Chunk chunk : chunks) {
- currentRowGroup.addColumn(chunk.descriptor.col, chunk.readAllPages());
+ ChunkListBuilder builder = new ChunkListBuilder();
+ for (ConsecutivePartList consecutiveChunks : allParts) {
+ consecutiveChunks.readAll(f, builder);
+ }
+ for (Chunk chunk : builder.build()) {
+ currentRowGroup.addColumn(chunk.descriptor.col, chunk.readAllPages());
+ }
+
+ // avoid re-reading bytes the dictionary reader is used after this call
+ if (nextDictionaryReader != null) {
+ nextDictionaryReader.setRowGroup(currentRowGroup);
+ }
+
+ advanceToNextBlock();
+
+ return currentRowGroup;
+ }
+
+ /**
+ * Reads all the columns requested from the row group at the current file position. It may skip specific pages based
+ * on the column indexes according to the actual filter. As the rows are not aligned among the pages of the different
+ * columns row synchronization might be required.
+ *
+ * @return the PageReadStore which can provide PageReaders for each column
+ * @throws IOException
+ * if any I/O error occurs while reading
+ * @see {@link PageReadStore#isInPageFilteringMode()}
+ */
+ public PageReadStore readNextFilteredRowGroup() throws IOException {
+ if (currentBlock == blocks.size()) {
+ return null;
+ }
+ if (!options.useColumnIndexFilter()) {
+ return readNextRowGroup();
+ }
+ BlockMetaData block = blocks.get(currentBlock);
+ if (block.getRowCount() == 0) {
+ throw new RuntimeException("Illegal row group of 0 rows");
+ }
+ ColumnIndexStore ciStore = getColumnIndexStore(currentBlock);
+ RowRanges rowRanges = getRowRanges(currentBlock);
+ long rowCount = rowRanges.rowCount();
+ if (rowCount == 0) {
+ // There are no matching rows -> skipping this row-group
+ advanceToNextBlock();
+ return readNextFilteredRowGroup();
+ }
+ if (rowCount == block.getRowCount()) {
+ // All rows are matching -> fall back to the non-filtering path
+ return readNextRowGroup();
+ }
+
+ this.currentRowGroup = new ColumnChunkPageReadStore(rowRanges);
+ // prepare the list of consecutive parts to read them in one scan
+ ChunkListBuilder builder = new ChunkListBuilder();
+ List<ConsecutivePartList> allParts = new ArrayList<ConsecutivePartList>();
+ ConsecutivePartList currentParts = null;
+ for (ColumnChunkMetaData mc : block.getColumns()) {
+ ColumnPath pathKey = mc.getPath();
+ ColumnDescriptor columnDescriptor = paths.get(pathKey);
+ if (columnDescriptor != null) {
+ OffsetIndex offsetIndex = ciStore.getOffsetIndex(mc.getPath());
+
+ OffsetIndex filteredOffsetIndex = filterOffsetIndex(offsetIndex, rowRanges,
+ block.getRowCount());
+ for (OffsetRange range : calculateOffsetRanges(filteredOffsetIndex, mc, offsetIndex.getOffset(0))) {
+ BenchmarkCounter.incrementTotalBytes(range.getLength());
+ long startingPos = range.getOffset();
+ // first part or not consecutive => new list
+ if (currentParts == null || currentParts.endPos() != startingPos) {
+ currentParts = new ConsecutivePartList(startingPos);
+ allParts.add(currentParts);
+ }
+ ChunkDescriptor chunkDescriptor = new ChunkDescriptor(columnDescriptor, mc, startingPos,
+ (int) range.getLength());
+ currentParts.addChunk(chunkDescriptor);
+ builder.setOffsetIndex(chunkDescriptor, filteredOffsetIndex);
+ }
}
}
+ // actually read all the chunks
+ for (ConsecutivePartList consecutiveChunks : allParts) {
+ consecutiveChunks.readAll(f, builder);
+ }
+ for (Chunk chunk : builder.build()) {
+ currentRowGroup.addColumn(chunk.descriptor.col, chunk.readAllPages());
+ }
// avoid re-reading bytes the dictionary reader is used after this call
if (nextDictionaryReader != null) {
@@ -827,6 +941,25 @@ public PageReadStore readNextRowGroup() throws IOException {
return currentRowGroup;
}
+ private ColumnIndexStore getColumnIndexStore(int blockIndex) {
+ ColumnIndexStore ciStore = blockIndexStores.get(blockIndex);
+ if (ciStore == null) {
+ ciStore = ColumnIndexStoreImpl.create(this, blocks.get(blockIndex), paths.keySet());
+ blockIndexStores.set(blockIndex, ciStore);
+ }
+ return ciStore;
+ }
+
+ private RowRanges getRowRanges(int blockIndex) {
+ RowRanges rowRanges = blockRowRanges.get(blockIndex);
+ if (rowRanges == null) {
+ rowRanges = ColumnIndexFilter.calculateRowRanges(options.getRecordFilter(), getColumnIndexStore(blockIndex),
+ paths.keySet(), blocks.get(blockIndex).getRowCount());
+ blockRowRanges.set(blockIndex, rowRanges);
+ }
+ return rowRanges;
+ }
+
public boolean skipNextRowGroup() {
return advanceToNextBlock();
}
@@ -912,6 +1045,40 @@ private DictionaryPage readCompressedDictionary(
converter.getEncoding(dictHeader.getEncoding()));
}
+ /**
+ * @param column
+ * the column chunk which the column index is to be returned for
+ * @return the column index for the specified column chunk or {@code null} if there is no index
+ * @throws IOException
+ * if any I/O error occurs during reading the file
+ */
+ @Private
+ public ColumnIndex readColumnIndex(ColumnChunkMetaData column) throws IOException {
+ IndexReference ref = column.getColumnIndexReference();
+ if (ref == null) {
+ return null;
+ }
+ f.seek(ref.getOffset());
+ return ParquetMetadataConverter.fromParquetColumnIndex(column.getPrimitiveType(), Util.readColumnIndex(f));
+ }
+
+ /**
+ * @param column
+ * the column chunk which the offset index is to be returned for
+ * @return the offset index for the specified column chunk or {@code null} if there is no index
+ * @throws IOException
+ * if any I/O error occurs during reading the file
+ */
+ @Private
+ public OffsetIndex readOffsetIndex(ColumnChunkMetaData column) throws IOException {
+ IndexReference ref = column.getOffsetIndexReference();
+ if (ref == null) {
+ return null;
+ }
+ f.seek(ref.getOffset());
+ return ParquetMetadataConverter.fromParquetOffsetIndex(Util.readOffsetIndex(f));
+ }
+
@Override
public void close() throws IOException {
try {
@@ -923,6 +1090,57 @@ public void close() throws IOException {
}
}
+ /*
+ * Builder to concatenate the buffers of the discontinuous parts for the same column. These parts are generated as a
+ * result of the column-index based filtering when some pages might be skipped at reading.
+ */
+ private class ChunkListBuilder {
+ private class ChunkData {
+ final List<ByteBuffer> buffers = new ArrayList<>();
+ OffsetIndex offsetIndex;
+ }
+
+ private final Map<ChunkDescriptor, ChunkData> map = new HashMap<>();
+ private ChunkDescriptor lastDescriptor;
+ private SeekableInputStream f;
+
+ void add(ChunkDescriptor descriptor, List<ByteBuffer> buffers, SeekableInputStream f) {
+ ChunkData data = map.get(descriptor);
+ if (data == null) {
+ data = new ChunkData();
+ map.put(descriptor, data);
+ }
+ data.buffers.addAll(buffers);
+
+ lastDescriptor = descriptor;
+ this.f = f;
+ }
+
+ void setOffsetIndex(ChunkDescriptor descriptor, OffsetIndex offsetIndex) {
+ ChunkData data = map.get(descriptor);
+ if (data == null) {
+ data = new ChunkData();
+ map.put(descriptor, data);
+ }
+ data.offsetIndex = offsetIndex;
+ }
+
+ List<Chunk> build() {
+ List<Chunk> chunks = new ArrayList<>();
+ for (Entry<ChunkDescriptor, ChunkData> entry : map.entrySet()) {
+ ChunkDescriptor descriptor = entry.getKey();
+ ChunkData data = entry.getValue();
+ if (descriptor.equals(lastDescriptor)) {
+ // because of a bug, the last chunk might be larger than descriptor.size
+ chunks.add(new WorkaroundChunk(lastDescriptor, data.buffers, f, data.offsetIndex));
+ } else {
+ chunks.add(new Chunk(descriptor, data.buffers, data.offsetIndex));
+ }
+ }
+ return chunks;
+ }
+ }
+
/**
* The data for a column chunk
*/
@@ -930,15 +1148,17 @@ public void close() throws IOException {
protected final ChunkDescriptor descriptor;
protected final ByteBufferInputStream stream;
+ final OffsetIndex offsetIndex;
/**
- *
* @param descriptor descriptor for the chunk
* @param buffers ByteBuffers that contain the chunk
+ * @param offsetIndex the offset index for this column; might be null
*/
- public Chunk(ChunkDescriptor descriptor, List<ByteBuffer> buffers) {
+ public Chunk(ChunkDescriptor descriptor, List<ByteBuffer> buffers, OffsetIndex offsetIndex) {
this.descriptor = descriptor;
this.stream = ByteBufferInputStream.wrap(buffers);
+ this.offsetIndex = offsetIndex;
}
protected PageHeader readPageHeader() throws IOException {
@@ -955,7 +1175,8 @@ public ColumnChunkPageReader readAllPages() throws IOException {
PrimitiveType type = getFileMetaData().getSchema()
.getType(descriptor.col.getPath()).asPrimitiveType();
long valuesCountReadSoFar = 0;
- while (valuesCountReadSoFar < descriptor.metadata.getValueCount()) {
+ int dataPageCountReadSoFar = 0;
+ while (hasMorePages(valuesCountReadSoFar, dataPageCountReadSoFar)) {
PageHeader pageHeader = readPageHeader();
int uncompressedPageSize = pageHeader.getUncompressed_page_size();
int compressedPageSize = pageHeader.getCompressed_page_size();
@@ -965,8 +1186,8 @@ public ColumnChunkPageReader readAllPages() throws IOException {
if (dictionaryPage != null) {
throw new ParquetDecodingException("more than one dictionary page in column " + descriptor.col);
}
- DictionaryPageHeader dicHeader = pageHeader.getDictionary_page_header();
- dictionaryPage =
+ DictionaryPageHeader dicHeader = pageHeader.getDictionary_page_header();
+ dictionaryPage =
new DictionaryPage(
this.readAsBytesInput(compressedPageSize),
uncompressedPageSize,
@@ -990,6 +1211,7 @@ public ColumnChunkPageReader readAllPages() throws IOException {
converter.getEncoding(dataHeaderV1.getEncoding())
));
valuesCountReadSoFar += dataHeaderV1.getNum_values();
+ ++dataPageCountReadSoFar;
break;
case DATA_PAGE_V2:
DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2();
@@ -1011,6 +1233,7 @@ public ColumnChunkPageReader readAllPages() throws IOException {
dataHeaderV2.isIs_compressed()
));
valuesCountReadSoFar += dataHeaderV2.getNum_values();
+ ++dataPageCountReadSoFar;
break;
default:
LOG.debug("skipping page of type {} of size {}", pageHeader.getType(), compressedPageSize);
@@ -1018,7 +1241,7 @@ public ColumnChunkPageReader readAllPages() throws IOException {
break;
}
}
- if (valuesCountReadSoFar != descriptor.metadata.getValueCount()) {
+ if (offsetIndex == null && valuesCountReadSoFar != descriptor.metadata.getValueCount()) {
// Would be nice to have a CorruptParquetFileException or something as a subclass?
throw new IOException(
"Expected " + descriptor.metadata.getValueCount() + " values in column chunk at " +
@@ -1027,7 +1250,13 @@ public ColumnChunkPageReader readAllPages() throws IOException {
+ " pages ending at file offset " + (descriptor.fileOffset + stream.position()));
}
BytesInputDecompressor decompressor = options.getCodecFactory().getDecompressor(descriptor.metadata.getCodec());
- return new ColumnChunkPageReader(decompressor, pagesInChunk, dictionaryPage);
+ return new ColumnChunkPageReader(decompressor, pagesInChunk, dictionaryPage, offsetIndex,
+ blocks.get(currentBlock).getRowCount());
+ }
+
+ private boolean hasMorePages(long valuesCountReadSoFar, int dataPageCountReadSoFar) {
+ return offsetIndex == null ? valuesCountReadSoFar < descriptor.metadata.getValueCount()
+ : dataPageCountReadSoFar < offsetIndex.getPageCount();
}
/**
@@ -1052,8 +1281,8 @@ public BytesInput readAsBytesInput(int size) throws IOException {
* @param descriptor the descriptor of the chunk
* @param f the file stream positioned at the end of this chunk
*/
- private WorkaroundChunk(ChunkDescriptor descriptor, List<ByteBuffer> buffers, SeekableInputStream f) {
- super(descriptor, buffers);
+ private WorkaroundChunk(ChunkDescriptor descriptor, List<ByteBuffer> buffers, SeekableInputStream f, OffsetIndex offsetIndex) {
+ super(descriptor, buffers, offsetIndex);
this.f = f;
}
@@ -1102,7 +1331,7 @@ public BytesInput readAsBytesInput(int size) throws IOException {
/**
- * information needed to read a column chunk
+ * Information needed to read a column chunk or a part of it.
*/
private static class ChunkDescriptor {
@@ -1128,12 +1357,29 @@ private ChunkDescriptor(
this.fileOffset = fileOffset;
this.size = size;
}
+
+ @Override
+ public int hashCode() {
+ return col.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ } else if (obj instanceof ChunkDescriptor) {
+ return col.equals(((ChunkDescriptor) obj).col);
+ } else {
+ return false;
+ }
+ }
}
/**
- * describes a list of consecutive column chunks to be read at once.
+ * Describes a list of consecutive parts to be read at once. A consecutive part may contain whole column chunks or
+ * only parts of them (some pages).
*/
- private class ConsecutiveChunkList {
+ private class ConsecutivePartList {
private final long offset;
private int length;
@@ -1142,7 +1388,7 @@ private ChunkDescriptor(
/**
* @param offset where the first chunk starts
*/
- ConsecutiveChunkList(long offset) {
+ ConsecutivePartList(long offset) {
this.offset = offset;
}
@@ -1158,11 +1404,10 @@ public void addChunk(ChunkDescriptor descriptor) {
/**
* @param f file to read the chunks from
- * @return the chunks
+ * @param builder used to build chunk list to read the pages for the different columns
* @throws IOException if there is an error while reading from the stream
*/
- public List<Chunk> readAll(SeekableInputStream f) throws IOException {
- List<Chunk> result = new ArrayList<>(chunks.size());
+ public void readAll(SeekableInputStream f, ChunkListBuilder builder) throws IOException {
List<ByteBuffer> buffers = readBlocks(f, offset, length);
// report in a counter the data we just scanned
@@ -1170,14 +1415,8 @@ public void addChunk(ChunkDescriptor descriptor) {
ByteBufferInputStream stream = ByteBufferInputStream.wrap(buffers);
for (int i = 0; i < chunks.size(); i++) {
ChunkDescriptor descriptor = chunks.get(i);
- if (i < chunks.size() - 1) {
- result.add(new Chunk(descriptor, stream.sliceBuffers(descriptor.size)));
- } else {
- // because of a bug, the last chunk might be larger than descriptor.size
- result.add(new WorkaroundChunk(descriptor, stream.sliceBuffers(descriptor.size), f));
- }
+ builder.add(descriptor, stream.sliceBuffers(descriptor.size), f);
}
- return result ;
}
/**
@@ -1233,7 +1472,7 @@ private ColumnChunkPageReader readChunk(SeekableInputStream f, ChunkDescriptor d
try {
List<ByteBuffer> buffers = readBlocks(f, descriptor.fileOffset, descriptor.size);
ByteBufferInputStream stream = ByteBufferInputStream.wrap(buffers);
- Chunk chunk = new WorkaroundChunk(descriptor, stream.sliceBuffers(descriptor.size), f);
+ Chunk chunk = new WorkaroundChunk(descriptor, stream.sliceBuffers(descriptor.size), f, null);
return chunk.readAllPages();
} catch (IOException e) {
throw new RuntimeException(e);
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java
index b944e9707..a8cd68602 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java
@@ -19,6 +19,7 @@
package org.apache.parquet.hadoop;
import static org.apache.parquet.format.Util.writeFileMetaData;
+import static org.apache.parquet.format.converter.ParquetMetadataConverter.MAX_STATS_SIZE;
import static org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE;
import static org.apache.parquet.hadoop.ParquetWriter.MAX_PADDING_SIZE_DEFAULT;
@@ -50,6 +51,7 @@
import org.apache.parquet.bytes.HeapByteBufferAllocator;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.ColumnReader;
+import org.apache.parquet.column.ColumnWriteStore;
import org.apache.parquet.column.ColumnWriter;
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.EncodingStats;
@@ -62,6 +64,7 @@
import org.apache.parquet.example.DummyRecordConverter;
import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel;
import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.format.Util;
import org.apache.parquet.format.converter.ParquetMetadataConverter;
import org.apache.parquet.hadoop.metadata.BlockMetaData;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -72,6 +75,11 @@
import org.apache.parquet.hadoop.util.BlocksCombiner;
import org.apache.parquet.hadoop.util.HadoopOutputFile;
import org.apache.parquet.hadoop.util.HadoopStreams;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndexBuilder;
+import org.apache.parquet.internal.hadoop.metadata.IndexReference;
import org.apache.parquet.io.InputFile;
import org.apache.parquet.io.OutputFile;
import org.apache.parquet.io.SeekableInputStream;
@@ -106,13 +114,22 @@
private final MessageType schema;
private final PositionOutputStream out;
private final AlignmentStrategy alignment;
+ private final int columnIndexTruncateLength;
// file data
private List<BlockMetaData> blocks = new ArrayList<BlockMetaData>();
+ // The column/offset indexes per blocks per column chunks
+ private final List<List<ColumnIndex>> columnIndexes = new ArrayList<>();
+ private final List<List<OffsetIndex>> offsetIndexes = new ArrayList<>();
+
// row group data
private BlockMetaData currentBlock; // appended to by endColumn
+ // The column/offset indexes for the actual block
+ private List<ColumnIndex> currentColumnIndexes;
+ private List<OffsetIndex> currentOffsetIndexes;
+
// row group data set at the start of a row group
private long currentRecordCount; // set in startBlock
@@ -122,6 +139,9 @@
private long uncompressedLength;
private long compressedLength;
private Statistics currentStatistics; // accumulated in writePage(s)
+ private ColumnIndexBuilder columnIndexBuilder;
+ private OffsetIndexBuilder offsetIndexBuilder;
+ private long firstPageOffset;
// column chunk data set at the start of a column
private CompressionCodecName currentChunkCodec; // set in startColumn
@@ -239,10 +259,27 @@ public ParquetFileWriter(Configuration configuration, MessageType schema,
* @param rowGroupSize the row group size
* @param maxPaddingSize the maximum padding
* @throws IOException if the file can not be created
+ * @deprecated will be removed in 2.0.0
*/
+ @Deprecated
public ParquetFileWriter(OutputFile file, MessageType schema, Mode mode,
long rowGroupSize, int maxPaddingSize)
throws IOException {
+ this(file, schema, mode, rowGroupSize, maxPaddingSize,
+ ParquetProperties.DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH);
+ }
+ /**
+ * @param file OutputFile to create or overwrite
+ * @param schema the schema of the data
+ * @param mode file creation mode
+ * @param rowGroupSize the row group size
+ * @param maxPaddingSize the maximum padding
+ * @param columnIndexTruncateLength the length which the min/max values in column indexes tried to be truncated to
+ * @throws IOException if the file can not be created
+ */
+ public ParquetFileWriter(OutputFile file, MessageType schema, Mode mode,
+ long rowGroupSize, int maxPaddingSize, int columnIndexTruncateLength)
+ throws IOException {
TypeUtil.checkValidWriteSchema(schema);
this.schema = schema;
@@ -262,6 +299,7 @@ public ParquetFileWriter(OutputFile file, MessageType schema, Mode mode,
}
this.encodingStatsBuilder = new EncodingStats.Builder();
+ this.columnIndexTruncateLength = columnIndexTruncateLength;
}
/**
@@ -284,6 +322,8 @@ public ParquetFileWriter(OutputFile file, MessageType schema, Mode mode,
this.out = HadoopStreams.wrap(
fs.create(file, true, 8192, fs.getDefaultReplication(file), rowAndBlockSize));
this.encodingStatsBuilder = new EncodingStats.Builder();
+ // no truncation is needed for testing
+ this.columnIndexTruncateLength = Integer.MAX_VALUE;
}
/**
* start the file
@@ -309,6 +349,9 @@ public void startBlock(long recordCount) throws IOException {
currentBlock = new BlockMetaData();
currentRecordCount = recordCount;
+
+ currentColumnIndexes = new ArrayList<>();
+ currentOffsetIndexes = new ArrayList<>();
}
/**
@@ -333,6 +376,10 @@ public void startColumn(ColumnDescriptor descriptor,
uncompressedLength = 0;
// The statistics will be copied from the first one added at writeDataPage(s) so we have the correct typed one
currentStatistics = null;
+
+ columnIndexBuilder = ColumnIndexBuilder.getBuilder(currentChunkType, columnIndexTruncateLength);
+ offsetIndexBuilder = OffsetIndexBuilder.getBuilder();
+ firstPageOffset = -1;
}
/**
@@ -380,6 +427,9 @@ public void writeDataPage(
Encoding dlEncoding,
Encoding valuesEncoding) throws IOException {
state = state.write();
+ // We are unable to build indexes without rowCount so skip them for this column
+ offsetIndexBuilder = OffsetIndexBuilder.getNoOpBuilder();
+ columnIndexBuilder = ColumnIndexBuilder.getNoOpBuilder();
long beforeHeader = out.getPos();
LOG.debug("{}: write data page: {} values", beforeHeader, valueCount);
int compressedPageSize = (int)bytes.size();
@@ -411,8 +461,50 @@ public void writeDataPage(
* @param dlEncoding encoding of the definition level
* @param valuesEncoding encoding of values
* @throws IOException if there is an error while writing
+ * @deprecated this method does not support writing column indexes; Use
+ * {@link #writeDataPage(int, int, BytesInput, Statistics, long, Encoding, Encoding, Encoding)} instead
+ */
+ @Deprecated
+ public void writeDataPage(
+ int valueCount, int uncompressedPageSize,
+ BytesInput bytes,
+ Statistics statistics,
+ Encoding rlEncoding,
+ Encoding dlEncoding,
+ Encoding valuesEncoding) throws IOException {
+ // We are unable to build indexes without rowCount so skip them for this column
+ offsetIndexBuilder = OffsetIndexBuilder.getNoOpBuilder();
+ columnIndexBuilder = ColumnIndexBuilder.getNoOpBuilder();
+ innerWriteDataPage(valueCount, uncompressedPageSize, bytes, statistics, rlEncoding, dlEncoding, valuesEncoding);
+ }
+
+ /**
+ * Writes a single page
+ * @param valueCount count of values
+ * @param uncompressedPageSize the size of the data once uncompressed
+ * @param bytes the compressed data for the page without header
+ * @param statistics the statistics of the page
+ * @param rowCount the number of rows in the page
+ * @param rlEncoding encoding of the repetition level
+ * @param dlEncoding encoding of the definition level
+ * @param valuesEncoding encoding of values
+ * @throws IOException if any I/O error occurs during writing the file
*/
public void writeDataPage(
+ int valueCount, int uncompressedPageSize,
+ BytesInput bytes,
+ Statistics statistics,
+ long rowCount,
+ Encoding rlEncoding,
+ Encoding dlEncoding,
+ Encoding valuesEncoding) throws IOException {
+ long beforeHeader = out.getPos();
+ innerWriteDataPage(valueCount, uncompressedPageSize, bytes, statistics, rlEncoding, dlEncoding, valuesEncoding);
+
+ offsetIndexBuilder.add((int) (out.getPos() - beforeHeader), rowCount);
+ }
+
+ private void innerWriteDataPage(
int valueCount, int uncompressedPageSize,
BytesInput bytes,
Statistics statistics,
@@ -421,8 +513,11 @@ public void writeDataPage(
Encoding valuesEncoding) throws IOException {
state = state.write();
long beforeHeader = out.getPos();
+ if (firstPageOffset == -1) {
+ firstPageOffset = beforeHeader;
+ }
LOG.debug("{}: write data page: {} values", beforeHeader, valueCount);
- int compressedPageSize = (int)bytes.size();
+ int compressedPageSize = (int) bytes.size();
metadataConverter.writeDataPageHeader(
uncompressedPageSize, compressedPageSize,
valueCount,
@@ -444,6 +539,8 @@ public void writeDataPage(
currentStatistics.mergeStatistics(statistics);
}
+ columnIndexBuilder.add(statistics);
+
encodingStatsBuilder.addDataEncoding(valuesEncoding);
currentEncodings.add(rlEncoding);
currentEncodings.add(dlEncoding);
@@ -451,25 +548,47 @@ public void writeDataPage(
}
/**
- * writes a number of pages at once
- * @param bytes bytes to be written including page headers
+ * Writes a column chunk at once
+ * @param descriptor the descriptor of the column
+ * @param valueCount the value count in this column
+ * @param compressionCodecName the name of the compression codec used for compressing the pages
+ * @param dictionaryPage the dictionary page for this column chunk (might be null)
+ * @param bytes the encoded pages including page headers to be written as is
* @param uncompressedTotalPageSize total uncompressed size (without page headers)
* @param compressedTotalPageSize total compressed size (without page headers)
+ * @param totalStats accumulated statistics for the column chunk
+ * @param columnIndexBuilder the builder object for the column index
+ * @param offsetIndexBuilder the builder object for the offset index
+ * @param rlEncodings the RL encodings used in this column chunk
+ * @param dlEncodings the DL encodings used in this column chunk
+ * @param dataEncodings the data encodings used in this column chunk
* @throws IOException if there is an error while writing
*/
- void writeDataPages(BytesInput bytes,
- long uncompressedTotalPageSize,
- long compressedTotalPageSize,
- Statistics totalStats,
- Set<Encoding> rlEncodings,
- Set<Encoding> dlEncodings,
- List<Encoding> dataEncodings) throws IOException {
+ void writeColumnChunk(ColumnDescriptor descriptor,
+ long valueCount,
+ CompressionCodecName compressionCodecName,
+ DictionaryPage dictionaryPage,
+ BytesInput bytes,
+ long uncompressedTotalPageSize,
+ long compressedTotalPageSize,
+ Statistics<?> totalStats,
+ ColumnIndexBuilder columnIndexBuilder,
+ OffsetIndexBuilder offsetIndexBuilder,
+ Set<Encoding> rlEncodings,
+ Set<Encoding> dlEncodings,
+ List<Encoding> dataEncodings) throws IOException {
+ startColumn(descriptor, valueCount, compressionCodecName);
+
state = state.write();
+ if (dictionaryPage != null) {
+ writeDictionaryPage(dictionaryPage);
+ }
LOG.debug("{}: write data pages", out.getPos());
long headersSize = bytes.size() - compressedTotalPageSize;
this.uncompressedLength += uncompressedTotalPageSize + headersSize;
this.compressedLength += compressedTotalPageSize + headersSize;
LOG.debug("{}: write data pages content", out.getPos());
+ firstPageOffset = out.getPos();
bytes.writeAllTo(out);
encodingStatsBuilder.addDataEncodings(dataEncodings);
if (rlEncodings.isEmpty()) {
@@ -479,6 +598,11 @@ void writeDataPages(BytesInput bytes,
currentEncodings.addAll(dlEncodings);
currentEncodings.addAll(dataEncodings);
currentStatistics = totalStats;
+
+ this.columnIndexBuilder = columnIndexBuilder;
+ this.offsetIndexBuilder = offsetIndexBuilder;
+
+ endColumn();
}
/**
@@ -488,6 +612,12 @@ void writeDataPages(BytesInput bytes,
public void endColumn() throws IOException {
state = state.endColumn();
LOG.debug("{}: end column", out.getPos());
+ if (columnIndexBuilder.getMinMaxSize() > columnIndexBuilder.getPageCount() * MAX_STATS_SIZE) {
+ currentColumnIndexes.add(null);
+ } else {
+ currentColumnIndexes.add(columnIndexBuilder.build());
+ }
+ currentOffsetIndexes.add(offsetIndexBuilder.build(firstPageOffset));
currentBlock.addColumn(ColumnChunkMetaData.get(
currentChunkPath,
currentChunkType,
@@ -503,6 +633,8 @@ public void endColumn() throws IOException {
this.currentBlock.setTotalByteSize(currentBlock.getTotalByteSize() + uncompressedLength);
this.uncompressedLength = 0;
this.compressedLength = 0;
+ columnIndexBuilder = null;
+ offsetIndexBuilder = null;
}
/**
@@ -514,6 +646,10 @@ public void endBlock() throws IOException {
LOG.debug("{}: end block", out.getPos());
currentBlock.setRowCount(currentRecordCount);
blocks.add(currentBlock);
+ columnIndexes.add(currentColumnIndexes);
+ offsetIndexes.add(currentOffsetIndexes);
+ currentColumnIndexes = null;
+ currentOffsetIndexes = null;
currentBlock = null;
}
@@ -542,8 +678,8 @@ public int merge(List<InputFile> inputFiles, CodecFactory.BytesCompressor compre
for (BlocksCombiner.SmallBlocksUnion smallBlocks : largeBlocks) {
for (int columnIndex = 0; columnIndex < schema.getColumns().size(); columnIndex++) {
ColumnDescriptor path = schema.getColumns().get(columnIndex);
- ColumnChunkPageWriteStore store = new ColumnChunkPageWriteStore(compressor, schema, allocator);
- ColumnWriteStoreV1 columnWriteStoreV1 = new ColumnWriteStoreV1(store, ParquetProperties.builder().build());
+ ColumnChunkPageWriteStore store = new ColumnChunkPageWriteStore(compressor, schema, allocator, ParquetProperties.DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH);
+ ColumnWriteStoreV1 columnWriteStoreV1 = new ColumnWriteStoreV1(schema, store, ParquetProperties.builder().build());
for (BlocksCombiner.SmallBlock smallBlock : smallBlocks.getBlocks()) {
ParquetFileReader parquetFileReader = smallBlock.getReader();
try {
@@ -552,7 +688,7 @@ public int merge(List<InputFile> inputFiles, CodecFactory.BytesCompressor compre
if (columnChunkPageReader.isPresent()) {
ColumnReader columnReader = columnReadStore.newMemColumnReader(path, columnChunkPageReader.get());
for (int i = 0; i < columnReader.getTotalValueCount(); i++) {
- consumeTriplet(columnWriter, columnReader);
+ consumeTriplet(columnWriteStoreV1, columnWriter, columnReader);
}
} else {
MessageType inputFileSchema = parquetFileReader.getFileMetaData().getSchema();
@@ -561,6 +697,10 @@ public int merge(List<InputFile> inputFiles, CodecFactory.BytesCompressor compre
int rep = parquetFileReader.getFileMetaData().getSchema().getMaxRepetitionLevel(parentPath);
for (int i = 0; i < parquetFileReader.getBlockMetaData(smallBlock.getBlockIndex()).getRowCount(); i++) {
columnWriter.writeNull(rep, def);
+ if (def == 0) {
+ // V1 pages also respect record boundaries so we have to mark them
+ columnWriteStoreV1.endRecord();
+ }
}
}
} catch (Exception e) {
@@ -598,7 +738,7 @@ public int merge(List<InputFile> inputFiles, CodecFactory.BytesCompressor compre
return readers;
}
- private void consumeTriplet(ColumnWriter columnWriter, ColumnReader columnReader) {
+ private void consumeTriplet(ColumnWriteStore columnWriteStore, ColumnWriter columnWriter, ColumnReader columnReader) {
int definitionLevel = columnReader.getCurrentDefinitionLevel();
int repetitionLevel = columnReader.getCurrentRepetitionLevel();
ColumnDescriptor column = columnReader.getDescriptor();
@@ -632,6 +772,10 @@ private void consumeTriplet(ColumnWriter columnWriter, ColumnReader columnReader
}
}
columnReader.consume();
+ if (repetitionLevel == 0) {
+ // V1 pages also respect record boundaries so we have to mark them
+ columnWriteStore.endRecord();
+ }
}
/**
@@ -728,6 +872,11 @@ public void appendRowGroup(SeekableInputStream from, BlockMetaData rowGroup,
length = 0;
}
+ // TODO: column/offset indexes are not copied
+ // (it would require seeking to the end of the file for each row groups)
+ currentColumnIndexes.add(null);
+ currentOffsetIndexes.add(null);
+
currentBlock.addColumn(ColumnChunkMetaData.get(
chunk.getPath(),
chunk.getPrimitiveType(),
@@ -794,12 +943,57 @@ private static void copy(SeekableInputStream from, PositionOutputStream to,
*/
public void end(Map<String, String> extraMetaData) throws IOException {
state = state.end();
+ serializeColumnIndexes(columnIndexes, blocks, out);
+ serializeOffsetIndexes(offsetIndexes, blocks, out);
LOG.debug("{}: end", out.getPos());
this.footer = new ParquetMetadata(new FileMetaData(schema, extraMetaData, Version.FULL_VERSION), blocks);
serializeFooter(footer, out);
out.close();
}
+ private static void serializeColumnIndexes(
+ List<List<ColumnIndex>> columnIndexes,
+ List<BlockMetaData> blocks,
+ PositionOutputStream out) throws IOException {
+ LOG.debug("{}: column indexes", out.getPos());
+ for (int bIndex = 0, bSize = blocks.size(); bIndex < bSize; ++bIndex) {
+ List<ColumnChunkMetaData> columns = blocks.get(bIndex).getColumns();
+ List<ColumnIndex> blockColumnIndexes = columnIndexes.get(bIndex);
+ for (int cIndex = 0, cSize = columns.size(); cIndex < cSize; ++cIndex) {
+ ColumnChunkMetaData column = columns.get(cIndex);
+ org.apache.parquet.format.ColumnIndex columnIndex = ParquetMetadataConverter
+ .toParquetColumnIndex(column.getPrimitiveType(), blockColumnIndexes.get(cIndex));
+ if (columnIndex == null) {
+ continue;
+ }
+ long offset = out.getPos();
+ Util.writeColumnIndex(columnIndex, out);
+ column.setColumnIndexReference(new IndexReference(offset, (int) (out.getPos() - offset)));
+ }
+ }
+ }
+
+ private static void serializeOffsetIndexes(
+ List<List<OffsetIndex>> offsetIndexes,
+ List<BlockMetaData> blocks,
+ PositionOutputStream out) throws IOException {
+ LOG.debug("{}: offset indexes", out.getPos());
+ for (int bIndex = 0, bSize = blocks.size(); bIndex < bSize; ++bIndex) {
+ List<ColumnChunkMetaData> columns = blocks.get(bIndex).getColumns();
+ List<OffsetIndex> blockOffsetIndexes = offsetIndexes.get(bIndex);
+ for (int cIndex = 0, cSize = columns.size(); cIndex < cSize; ++cIndex) {
+ OffsetIndex offsetIndex = blockOffsetIndexes.get(cIndex);
+ if (offsetIndex == null) {
+ continue;
+ }
+ ColumnChunkMetaData column = columns.get(cIndex);
+ long offset = out.getPos();
+ Util.writeOffsetIndex(ParquetMetadataConverter.toParquetOffsetIndex(offsetIndex), out);
+ column.setOffsetIndexReference(new IndexReference(offset, (int) (out.getPos() - offset)));
+ }
+ }
+ }
+
private static void serializeFooter(ParquetMetadata footer, PositionOutputStream out) throws IOException {
long footerIndex = out.getPos();
org.apache.parquet.format.FileMetaData parquetMetadata = metadataConverter.toParquetMetadata(CURRENT_VERSION, footer);
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputFormat.java
index 2c21e5203..b8fce2f65 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputFormat.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputFormat.java
@@ -129,6 +129,11 @@
*/
public static final String DICTIONARY_FILTERING_ENABLED = "parquet.filter.dictionary.enabled";
+ /**
+ * key to configure whether column index filtering of pages is enabled
+ */
+ public static final String COLUMN_INDEX_FILTERING_ENABLED = "parquet.filter.columnindex.enabled";
+
/**
* key to turn on or off task side metadata loading (default true)
* if true then metadata is read on the task side and some tasks may finish immediately.
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
index ff5bab397..0789bf50d 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
@@ -143,6 +143,7 @@
public static final String MIN_ROW_COUNT_FOR_PAGE_SIZE_CHECK = "parquet.page.size.row.check.min";
public static final String MAX_ROW_COUNT_FOR_PAGE_SIZE_CHECK = "parquet.page.size.row.check.max";
public static final String ESTIMATE_PAGE_SIZE_CHECK = "parquet.page.size.check.estimate";
+ public static final String COLUMN_INDEX_TRUNCATE_LENGTH = "parquet.columnindex.truncate.length";
public static JobSummaryLevel getJobSummaryLevel(Configuration conf) {
String level = conf.get(JOB_SUMMARY_LEVEL);
@@ -312,6 +313,18 @@ private static int getMaxPaddingSize(Configuration conf) {
return conf.getInt(MAX_PADDING_BYTES, ParquetWriter.MAX_PADDING_SIZE_DEFAULT);
}
+ public static void setColumnIndexTruncateLength(JobContext jobContext, int length) {
+ setColumnIndexTruncateLength(getConfiguration(jobContext), length);
+ }
+
+ public static void setColumnIndexTruncateLength(Configuration conf, int length) {
+ conf.setInt(COLUMN_INDEX_TRUNCATE_LENGTH, length);
+ }
+
+ private static int getColumnIndexTruncateLength(Configuration conf) {
+ return conf.getInt(COLUMN_INDEX_TRUNCATE_LENGTH, ParquetProperties.DEFAULT_COLUMN_INDEX_TRUNCATE_LENGTH);
+ }
+
private WriteSupport<T> writeSupport;
private ParquetOutputCommitter committer;
@@ -366,6 +379,7 @@ private static int getMaxPaddingSize(Configuration conf) {
.estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf))
.withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf))
.withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf))
+ .withColumnIndexTruncateLength(getColumnIndexTruncateLength(conf))
.build();
long blockSize = getLongBlockSize(conf);
@@ -383,11 +397,12 @@ private static int getMaxPaddingSize(Configuration conf) {
LOG.info("Page size checking is: {}", (props.estimateNextSizeCheck() ? "estimated" : "constant"));
LOG.info("Min row count for page size check is: {}", props.getMinRowCountForPageSizeCheck());
LOG.info("Max row count for page size check is: {}", props.getMaxRowCountForPageSizeCheck());
+ LOG.info("Truncate length for column indexes is: {}", props.getColumnIndexTruncateLength());
}
WriteContext init = writeSupport.init(conf);
ParquetFileWriter w = new ParquetFileWriter(HadoopOutputFile.fromPath(file, conf),
- init.getSchema(), Mode.CREATE, blockSize, maxPaddingSize);
+ init.getSchema(), Mode.CREATE, blockSize, maxPaddingSize, props.getColumnIndexTruncateLength());
w.start();
float maxLoad = conf.getFloat(ParquetOutputFormat.MEMORY_POOL_RATIO,
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetReader.java
index d9b273bb9..de20808ff 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetReader.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetReader.java
@@ -270,6 +270,16 @@ protected Builder(InputFile file) {
return this;
}
+ public Builder<T> useColumnIndexFilter(boolean useColumnIndexFilter) {
+ optionsBuilder.useColumnIndexFilter(useColumnIndexFilter);
+ return this;
+ }
+
+ public Builder<T> useColumnIndexFilter() {
+ optionsBuilder.useColumnIndexFilter();
+ return this;
+ }
+
public Builder<T> withFileRange(long start, long end) {
optionsBuilder.withRange(start, end);
return this;
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java
index a32df39a5..5b0e4f82d 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetWriter.java
@@ -278,7 +278,7 @@ public ParquetWriter(Path file, Configuration conf, WriteSupport<T> writeSupport
MessageType schema = writeContext.getSchema();
ParquetFileWriter fileWriter = new ParquetFileWriter(
- file, schema, mode, rowGroupSize, maxPaddingSize);
+ file, schema, mode, rowGroupSize, maxPaddingSize, encodingProps.getColumnIndexTruncateLength());
fileWriter.start();
this.codecFactory = new CodecFactory(conf, encodingProps.getPageSizeThreshold());
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java
index fb94247ed..e6aa1043b 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/metadata/ColumnChunkMetaData.java
@@ -24,9 +24,11 @@
import org.apache.parquet.column.EncodingStats;
import org.apache.parquet.column.statistics.BooleanStatistics;
import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.internal.hadoop.metadata.IndexReference;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Types;
+import org.apache.yetus.audience.InterfaceAudience.Private;
/**
* Column meta data for a block stored in the file footer and passed in the InputSplit
@@ -168,6 +170,9 @@ protected static boolean positiveLongFitsInAnInt(long value) {
// we save 3 references by storing together the column properties that have few distinct values
private final ColumnChunkProperties properties;
+ private IndexReference columnIndexReference;
+ private IndexReference offsetIndexReference;
+
protected ColumnChunkMetaData(ColumnChunkProperties columnChunkProperties) {
this(null, columnChunkProperties);
}
@@ -184,9 +189,7 @@ public CompressionCodecName getCodec() {
/**
*
* @return column identifier
- * @deprecated will be removed in 2.0.0. Use {@link #getPrimitiveType()} instead.
*/
- @Deprecated
public ColumnPath getPath() {
return properties.getPath();
}
@@ -237,6 +240,40 @@ public PrimitiveType getPrimitiveType() {
*/
abstract public Statistics getStatistics();
+ /**
+ * @return the reference to the column index
+ */
+ @Private
+ public IndexReference getColumnIndexReference() {
+ return columnIndexReference;
+ }
+
+ /**
+ * @param indexReference
+ * the reference to the column index
+ */
+ @Private
+ public void setColumnIndexReference(IndexReference indexReference) {
+ this.columnIndexReference = indexReference;
+ }
+
+ /**
+ * @return the reference to the offset index
+ */
+ @Private
+ public IndexReference getOffsetIndexReference() {
+ return offsetIndexReference;
+ }
+
+ /**
+ * @param offsetIndexReference
+ * the reference to the offset index
+ */
+ @Private
+ public void setOffsetIndexReference(IndexReference offsetIndexReference) {
+ this.offsetIndexReference = offsetIndexReference;
+ }
+
/**
* @return all the encodings used in this column
*/
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/internal/hadoop/metadata/IndexReference.java b/parquet-hadoop/src/main/java/org/apache/parquet/internal/hadoop/metadata/IndexReference.java
new file mode 100644
index 000000000..5e02f1efe
--- /dev/null
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/internal/hadoop/metadata/IndexReference.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.hadoop.metadata;
+
+/**
+ * Reference to an index (OffsetIndex and ColumnIndex) for a row-group containing the offset and length values so the
+ * reader can read the referenced data.
+ */
+public class IndexReference {
+ private final long offset;
+ private final int length;
+
+ public IndexReference(long offset, int length) {
+ this.offset = offset;
+ this.length = length;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public int getLength() {
+ return length;
+ }
+}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/filter2/recordlevel/PhoneBookWriter.java b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/recordlevel/PhoneBookWriter.java
index 7acda935c..18ddca0d9 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/filter2/recordlevel/PhoneBookWriter.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/recordlevel/PhoneBookWriter.java
@@ -31,6 +31,7 @@
import org.apache.parquet.filter2.compat.FilterCompat.Filter;
import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
import org.apache.parquet.hadoop.example.GroupReadSupport;
import org.apache.parquet.hadoop.example.GroupWriteSupport;
import org.apache.parquet.schema.MessageType;
@@ -91,6 +92,11 @@ public int hashCode() {
result = 31 * result + (lat != null ? lat.hashCode() : 0);
return result;
}
+
+ @Override
+ public String toString() {
+ return "Location [lon=" + lon + ", lat=" + lat + "]";
+ }
}
public static class PhoneNumber {
@@ -129,6 +135,11 @@ public int hashCode() {
result = 31 * result + (kind != null ? kind.hashCode() : 0);
return result;
}
+
+ @Override
+ public String toString() {
+ return "PhoneNumber [number=" + number + ", kind=" + kind + "]";
+ }
}
public static class User {
@@ -183,6 +194,11 @@ public int hashCode() {
result = 31 * result + (location != null ? location.hashCode() : 0);
return result;
}
+
+ @Override
+ public String toString() {
+ return "User [id=" + id + ", name=" + name + ", phoneNumbers=" + phoneNumbers + ", location=" + location + "]";
+ }
}
public static SimpleGroup groupFromUser(User user) {
@@ -216,6 +232,56 @@ public static SimpleGroup groupFromUser(User user) {
return root;
}
+ private static User userFromGroup(Group root) {
+ return new User(getLong(root, "id"), getString(root, "name"), getPhoneNumbers(getGroup(root, "phoneNumbers")),
+ getLocation(getGroup(root, "location")));
+ }
+
+ private static List<PhoneNumber> getPhoneNumbers(Group phoneNumbers) {
+ if (phoneNumbers == null) {
+ return null;
+ }
+ List<PhoneNumber> list = new ArrayList<>();
+ for (int i = 0, n = phoneNumbers.getFieldRepetitionCount("phone"); i < n; ++i) {
+ Group phone = phoneNumbers.getGroup("phone", i);
+ list.add(new PhoneNumber(getLong(phone, "number"), getString(phone, "kind")));
+ }
+ return list;
+ }
+
+ private static Location getLocation(Group location) {
+ if (location == null) {
+ return null;
+ }
+ return new Location(getDouble(location, "lon"), getDouble(location, "lat"));
+ }
+
+ private static boolean isNull(Group group, String field) {
+ int repetition = group.getFieldRepetitionCount(field);
+ if (repetition == 0) {
+ return true;
+ } else if (repetition == 1) {
+ return false;
+ }
+ throw new AssertionError("Invalid repetitionCount " + repetition + " for field " + field + " in group " + group);
+ }
+
+ private static Long getLong(Group group, String field) {
+ return isNull(group, field) ? null : group.getLong(field, 0);
+ }
+
+ private static String getString(Group group, String field) {
+ return isNull(group, field) ? null : group.getString(field, 0);
+ }
+
+ private static Double getDouble(Group group, String field) {
+ return isNull(group, field) ? null : group.getDouble(field, 0);
+ }
+
+ private static Group getGroup(Group group, String field) {
+ return isNull(group, field) ? null : group.getGroup(field, 0);
+ }
+
public static File writeToFile(List<User> users) throws IOException {
File f = File.createTempFile("phonebook", ".parquet");
f.deleteOnExit();
@@ -229,25 +295,30 @@ public static File writeToFile(List<User> users) throws IOException {
}
public static void writeToFile(File f, List<User> users) throws IOException {
- Configuration conf = new Configuration();
- GroupWriteSupport.setSchema(schema, conf);
+ write(ExampleParquetWriter.builder(new Path(f.getAbsolutePath())), users);
+ }
- ParquetWriter<Group> writer = new ParquetWriter<Group>(new Path(f.getAbsolutePath()), conf, new GroupWriteSupport());
- for (User u : users) {
- writer.write(groupFromUser(u));
+ public static void write(ParquetWriter.Builder<Group, ?> builder, List<User> users) throws IOException {
+ builder.config(GroupWriteSupport.PARQUET_EXAMPLE_SCHEMA, schema.toString());
+ try (ParquetWriter<Group> writer = builder.build()) {
+ for (User u : users) {
+ writer.write(groupFromUser(u));
+ }
}
- writer.close();
}
- public static List<Group> readFile(File f, Filter filter) throws IOException {
+ private static ParquetReader<Group> createReader(Path file, Filter filter) throws IOException {
Configuration conf = new Configuration();
GroupWriteSupport.setSchema(schema, conf);
- ParquetReader<Group> reader =
- ParquetReader.builder(new GroupReadSupport(), new Path(f.getAbsolutePath()))
- .withConf(conf)
- .withFilter(filter)
- .build();
+ return ParquetReader.builder(new GroupReadSupport(), file)
+ .withConf(conf)
+ .withFilter(filter)
+ .build();
+ }
+
+ public static List<Group> readFile(File f, Filter filter) throws IOException {
+ ParquetReader<Group> reader = createReader(new Path(f.getAbsolutePath()), filter);
Group current;
List<Group> users = new ArrayList<Group>();
@@ -261,6 +332,16 @@ public static void writeToFile(File f, List<User> users) throws IOException {
return users;
}
+ public static List<User> readUsers(ParquetReader.Builder<Group> builder) throws IOException {
+ ParquetReader<Group> reader = builder.set(GroupWriteSupport.PARQUET_EXAMPLE_SCHEMA, schema.toString()).build();
+
+ List<User> users = new ArrayList<>();
+ for (Group group = reader.read(); group != null; group = reader.read()) {
+ users.add(userFromGroup(group));
+ }
+ return users;
+ }
+
public static void main(String[] args) throws IOException {
File f = new File(args[0]);
writeToFile(f, TestRecordLevelFilters.makeUsers());
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
index d1a3a3c23..358a29a67 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -25,6 +25,7 @@
import static org.apache.parquet.schema.MessageTypeParser.parseMessageType;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -69,6 +70,11 @@
import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.internal.column.columnindex.BoundaryOrder;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndexBuilder;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.LogicalTypeAnnotation;
@@ -201,6 +207,12 @@ public void testTimeLogicalTypes() {
.required(PrimitiveTypeName.INT64)
.as(timestampType(true, LogicalTypeAnnotation.TimeUnit.MICROS))
.named("aTimestampUtcMicros")
+ .required(PrimitiveTypeName.INT64)
+ .as(timestampType(false, LogicalTypeAnnotation.TimeUnit.NANOS))
+ .named("aTimestampNonUtcNanos")
+ .required(PrimitiveTypeName.INT64)
+ .as(timestampType(true, LogicalTypeAnnotation.TimeUnit.NANOS))
+ .named("aTimestampUtcNanos")
.required(PrimitiveTypeName.INT32)
.as(timeType(false, LogicalTypeAnnotation.TimeUnit.MILLIS))
.named("aTimeNonUtcMillis")
@@ -213,6 +225,12 @@ public void testTimeLogicalTypes() {
.required(PrimitiveTypeName.INT64)
.as(timeType(true, LogicalTypeAnnotation.TimeUnit.MICROS))
.named("aTimeUtcMicros")
+ .required(PrimitiveTypeName.INT64)
+ .as(timeType(false, LogicalTypeAnnotation.TimeUnit.NANOS))
+ .named("aTimeNonUtcNanos")
+ .required(PrimitiveTypeName.INT64)
+ .as(timeType(true, LogicalTypeAnnotation.TimeUnit.NANOS))
+ .named("aTimeUtcNanos")
.named("Message");
List<SchemaElement> parquetSchema = parquetMetadataConverter.toParquetSchema(expected);
MessageType schema = parquetMetadataConverter.fromParquetSchema(parquetSchema, null);
@@ -973,4 +991,60 @@ public void testColumnOrders() throws IOException {
assertEquals(ColumnOrder.undefined(), columns.get(1).getPrimitiveType().columnOrder());
assertEquals(ColumnOrder.undefined(), columns.get(2).getPrimitiveType().columnOrder());
}
+
+ @Test
+ public void testOffsetIndexConversion() {
+ OffsetIndexBuilder builder = OffsetIndexBuilder.getBuilder();
+ builder.add(1000, 10000, 0);
+ builder.add(22000, 12000, 100);
+ OffsetIndex offsetIndex = ParquetMetadataConverter
+ .fromParquetOffsetIndex(ParquetMetadataConverter.toParquetOffsetIndex(builder.build(100000)));
+ assertEquals(2, offsetIndex.getPageCount());
+ assertEquals(101000, offsetIndex.getOffset(0));
+ assertEquals(10000, offsetIndex.getCompressedPageSize(0));
+ assertEquals(0, offsetIndex.getFirstRowIndex(0));
+ assertEquals(122000, offsetIndex.getOffset(1));
+ assertEquals(12000, offsetIndex.getCompressedPageSize(1));
+ assertEquals(100, offsetIndex.getFirstRowIndex(1));
+ }
+
+ @Test
+ public void testColumnIndexConversion() {
+ PrimitiveType type = Types.required(PrimitiveTypeName.INT64).named("test_int64");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+ Statistics<?> stats = Statistics.createStats(type);
+ stats.incrementNumNulls(16);
+ stats.updateStats(-100l);
+ stats.updateStats(100l);
+ builder.add(stats);
+ stats = Statistics.createStats(type);
+ stats.incrementNumNulls(111);
+ builder.add(stats);
+ stats = Statistics.createStats(type);
+ stats.updateStats(200l);
+ stats.updateStats(500l);
+ builder.add(stats);
+ org.apache.parquet.format.ColumnIndex parquetColumnIndex =
+ ParquetMetadataConverter.toParquetColumnIndex(type, builder.build());
+ ColumnIndex columnIndex = ParquetMetadataConverter.fromParquetColumnIndex(type, parquetColumnIndex);
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertTrue(Arrays.asList(false, true, false).equals(columnIndex.getNullPages()));
+ assertTrue(Arrays.asList(16l, 111l, 0l).equals(columnIndex.getNullCounts()));
+ assertTrue(Arrays.asList(
+ ByteBuffer.wrap(BytesUtils.longToBytes(-100l)),
+ ByteBuffer.allocate(0),
+ ByteBuffer.wrap(BytesUtils.longToBytes(200l))).equals(columnIndex.getMinValues()));
+ assertTrue(Arrays.asList(
+ ByteBuffer.wrap(BytesUtils.longToBytes(100l)),
+ ByteBuffer.allocate(0),
+ ByteBuffer.wrap(BytesUtils.longToBytes(500l))).equals(columnIndex.getMaxValues()));
+
+ assertNull("Should handle null column index", ParquetMetadataConverter
+ .toParquetColumnIndex(Types.required(PrimitiveTypeName.INT32).named("test_int32"), null));
+ assertNull("Should ignore unsupported types", ParquetMetadataConverter
+ .toParquetColumnIndex(Types.required(PrimitiveTypeName.INT96).named("test_int96"), columnIndex));
+ assertNull("Should ignore unsupported types",
+ ParquetMetadataConverter.fromParquetColumnIndex(Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(12).as(OriginalType.INTERVAL).named("test_interval"), parquetColumnIndex));
+ }
}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnChunkPageWriteStore.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnChunkPageWriteStore.java
index a5381f073..9a27defe1 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnChunkPageWriteStore.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnChunkPageWriteStore.java
@@ -18,8 +18,13 @@
*/
package org.apache.parquet.hadoop;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.isNull;
+import static org.mockito.Matchers.same;
import static org.mockito.Mockito.inOrder;
import static org.apache.parquet.column.Encoding.PLAIN;
import static org.apache.parquet.column.Encoding.RLE;
@@ -51,13 +56,23 @@
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.page.DataPageV2;
+import org.apache.parquet.column.page.DictionaryPage;
import org.apache.parquet.column.page.PageReadStore;
import org.apache.parquet.column.page.PageReader;
import org.apache.parquet.column.page.PageWriter;
import org.apache.parquet.column.statistics.BinaryStatistics;
import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.hadoop.ParquetFileWriter.Mode;
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.hadoop.util.HadoopOutputFile;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.ColumnIndexBuilder;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndexBuilder;
+import org.apache.parquet.io.OutputFile;
+import org.apache.parquet.io.PositionOutputStream;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
@@ -66,6 +81,40 @@
public class TestColumnChunkPageWriteStore {
+ // OutputFile implementation to expose the PositionOutputStream internally used by the writer
+ private static class OutputFileForTesting implements OutputFile {
+ private PositionOutputStream out;
+ private final HadoopOutputFile file;
+
+ OutputFileForTesting(Path path, Configuration conf) throws IOException {
+ file = HadoopOutputFile.fromPath(path, conf);
+ }
+
+ PositionOutputStream out() {
+ return out;
+ }
+
+ @Override
+ public PositionOutputStream create(long blockSizeHint) throws IOException {
+ return out = file.create(blockSizeHint);
+ }
+
+ @Override
+ public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException {
+ return out = file.createOrOverwrite(blockSizeHint);
+ }
+
+ @Override
+ public boolean supportsBlockSize() {
+ return file.supportsBlockSize();
+ }
+
+ @Override
+ public long defaultBlockSize() {
+ return file.defaultBlockSize();
+ }
+ }
+
private int pageSize = 1024;
private int initialSize = 1024;
private Configuration conf;
@@ -98,13 +147,21 @@ public void test() throws Exception {
BytesInput data = BytesInput.fromInt(v);
int rowCount = 5;
int nullCount = 1;
+ statistics.incrementNumNulls(nullCount);
+ statistics.setMinMaxFromBytes(new byte[] {0, 1, 2}, new byte[] {0, 1, 2, 3});
+ long pageOffset;
+ long pageSize;
{
- ParquetFileWriter writer = new ParquetFileWriter(conf, schema, file);
+ OutputFileForTesting outputFile = new OutputFileForTesting(file, conf);
+ ParquetFileWriter writer = new ParquetFileWriter(outputFile, schema, Mode.CREATE,
+ ParquetWriter.DEFAULT_BLOCK_SIZE, ParquetWriter.MAX_PADDING_SIZE_DEFAULT);
writer.start();
writer.startBlock(rowCount);
+ pageOffset = outputFile.out().getPos();
{
- ColumnChunkPageWriteStore store = new ColumnChunkPageWriteStore(compressor(GZIP), schema , new HeapByteBufferAllocator());
+ ColumnChunkPageWriteStore store = new ColumnChunkPageWriteStore(compressor(GZIP), schema,
+ new HeapByteBufferAllocator(), Integer.MAX_VALUE);
PageWriter pageWriter = store.getPageWriter(col);
pageWriter.writePageV2(
rowCount, nullCount, valueCount,
@@ -112,6 +169,7 @@ public void test() throws Exception {
dataEncoding, data,
statistics);
store.flushToFileWriter(writer);
+ pageSize = outputFile.out().getPos() - pageOffset;
}
writer.endBlock();
writer.end(new HashMap<String, String>());
@@ -132,6 +190,20 @@ public void test() throws Exception {
assertEquals(dataEncoding, page.getDataEncoding());
assertEquals(v, intValue(page.getData()));
assertEquals(statistics.toString(), page.getStatistics().toString());
+
+ // Checking column/offset indexes for the one page
+ ColumnChunkMetaData column = footer.getBlocks().get(0).getColumns().get(0);
+ ColumnIndex columnIndex = reader.readColumnIndex(column);
+ assertArrayEquals(statistics.getMinBytes(), columnIndex.getMinValues().get(0).array());
+ assertArrayEquals(statistics.getMaxBytes(), columnIndex.getMaxValues().get(0).array());
+ assertEquals(statistics.getNumNulls(), columnIndex.getNullCounts().get(0).longValue());
+ assertFalse(columnIndex.getNullPages().get(0));
+ OffsetIndex offsetIndex = reader.readOffsetIndex(column);
+ assertEquals(1, offsetIndex.getPageCount());
+ assertEquals(pageSize, offsetIndex.getCompressedPageSize(0));
+ assertEquals(0, offsetIndex.getFirstRowIndex(0));
+ assertEquals(pageOffset, offsetIndex.getOffset(0));
+
reader.close();
}
}
@@ -164,7 +236,7 @@ public void testColumnOrderV1() throws IOException {
// TODO - look back at this, an allocator was being passed here in the ByteBuffer changes
// see comment at this constructor
ColumnChunkPageWriteStore store = new ColumnChunkPageWriteStore(
- compressor(UNCOMPRESSED), schema, new HeapByteBufferAllocator());
+ compressor(UNCOMPRESSED), schema, new HeapByteBufferAllocator(), Integer.MAX_VALUE);
for (ColumnDescriptor col : schema.getColumns()) {
PageWriter pageWriter = store.getPageWriter(col);
@@ -175,8 +247,20 @@ public void testColumnOrderV1() throws IOException {
store.flushToFileWriter(mockFileWriter);
for (ColumnDescriptor col : schema.getColumns()) {
- inOrder.verify(mockFileWriter).startColumn(
- eq(col), eq((long) fakeCount), eq(UNCOMPRESSED));
+ inOrder.verify(mockFileWriter).writeColumnChunk(
+ eq(col),
+ eq((long) fakeCount),
+ eq(UNCOMPRESSED),
+ isNull(DictionaryPage.class),
+ any(),
+ eq(fakeData.size()),
+ eq(fakeData.size()),
+ eq(fakeStats),
+ same(ColumnIndexBuilder.getNoOpBuilder()), // Deprecated writePage -> no column index
+ same(OffsetIndexBuilder.getNoOpBuilder()), // Deprecated writePage -> no offset index
+ any(),
+ any(),
+ any());
}
}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnIndexFiltering.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnIndexFiltering.java
new file mode 100644
index 000000000..71155ced7
--- /dev/null
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestColumnIndexFiltering.java
@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.hadoop;
+
+import static java.util.Collections.emptyList;
+import static org.apache.parquet.filter2.predicate.FilterApi.and;
+import static org.apache.parquet.filter2.predicate.FilterApi.binaryColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.doubleColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.eq;
+import static org.apache.parquet.filter2.predicate.FilterApi.gtEq;
+import static org.apache.parquet.filter2.predicate.FilterApi.longColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.lt;
+import static org.apache.parquet.filter2.predicate.FilterApi.ltEq;
+import static org.apache.parquet.filter2.predicate.FilterApi.not;
+import static org.apache.parquet.filter2.predicate.FilterApi.notEq;
+import static org.apache.parquet.filter2.predicate.FilterApi.or;
+import static org.apache.parquet.filter2.predicate.FilterApi.userDefined;
+import static org.apache.parquet.filter2.predicate.LogicalInverter.invert;
+import static org.apache.parquet.hadoop.ParquetFileWriter.Mode.OVERWRITE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.column.ParquetProperties.WriterVersion;
+import org.apache.parquet.filter2.compat.FilterCompat;
+import org.apache.parquet.filter2.compat.FilterCompat.Filter;
+import org.apache.parquet.filter2.predicate.FilterPredicate;
+import org.apache.parquet.filter2.predicate.Statistics;
+import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
+import org.apache.parquet.filter2.recordlevel.PhoneBookWriter;
+import org.apache.parquet.filter2.recordlevel.PhoneBookWriter.Location;
+import org.apache.parquet.filter2.recordlevel.PhoneBookWriter.PhoneNumber;
+import org.apache.parquet.filter2.recordlevel.PhoneBookWriter.User;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.example.GroupReadSupport;
+import org.apache.parquet.io.api.Binary;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit tests for high level column index based filtering.
+ */
+@RunWith(Parameterized.class)
+public class TestColumnIndexFiltering {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TestColumnIndexFiltering.class);
+ private static final Random RANDOM = new Random(42);
+ private static final String[] PHONE_KINDS = { null, "mobile", "home", "work" };
+ private static final List<User> DATA = Collections.unmodifiableList(generateData(10000));
+ private static final Path FILE_V1 = createTempFile();
+ private static final Path FILE_V2 = createTempFile();
+
+ @Parameters
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[] { FILE_V1 }, new Object[] { FILE_V2 });
+ }
+
+ private final Path file;
+
+ public TestColumnIndexFiltering(Path file) {
+ this.file = file;
+ }
+
+ private static List<User> generateData(int rowCount) {
+ List<User> users = new ArrayList<>();
+ List<String> names = generateNames(rowCount);
+ for (int i = 0; i < rowCount; ++i) {
+ users.add(new User(i, names.get(i), generatePhoneNumbers(), generateLocation(i, rowCount)));
+ }
+ return users;
+ }
+
+ private static List<String> generateNames(int rowCount) {
+ List<String> list = new ArrayList<>();
+
+ // Adding fix values for filtering
+ list.add("anderson");
+ list.add("anderson");
+ list.add("miller");
+ list.add("miller");
+ list.add("miller");
+ list.add("thomas");
+ list.add("thomas");
+ list.add("williams");
+
+ int nullCount = rowCount / 100;
+
+ String alphabet = "aabcdeefghiijklmnoopqrstuuvwxyz";
+ int maxLength = 8;
+ for (int i = rowCount - list.size() - nullCount; i >= 0; --i) {
+ int l = RANDOM.nextInt(maxLength);
+ StringBuilder builder = new StringBuilder(l);
+ for (int j = 0; j < l; ++j) {
+ builder.append(alphabet.charAt(RANDOM.nextInt(alphabet.length())));
+ }
+ list.add(builder.toString());
+ }
+ Collections.sort(list, (str1, str2) -> -str1.compareTo(str2));
+
+ // Adding nulls to random places
+ for (int i = 0; i < nullCount; ++i) {
+ list.add(RANDOM.nextInt(list.size()), null);
+ }
+
+ return list;
+ }
+
+ private static List<PhoneNumber> generatePhoneNumbers() {
+ int length = RANDOM.nextInt(5) - 1;
+ if (length < 0) {
+ return null;
+ }
+ List<PhoneNumber> phoneNumbers = new ArrayList<>(length);
+ for (int i = 0; i < length; ++i) {
+ // 6 digits numbers
+ long number = Math.abs(RANDOM.nextLong() % 900000) + 100000;
+ phoneNumbers.add(new PhoneNumber(number, PHONE_KINDS[RANDOM.nextInt(PHONE_KINDS.length)]));
+ }
+ return phoneNumbers;
+ }
+
+ private static Location generateLocation(int id, int rowCount) {
+ if (RANDOM.nextDouble() < 0.01) {
+ return null;
+ }
+
+ double lat = RANDOM.nextDouble() * 90.0 - (id < rowCount / 2 ? 90.0 : 0.0);
+ double lon = RANDOM.nextDouble() * 90.0 - (id < rowCount / 4 || id >= 3 * rowCount / 4 ? 90.0 : 0.0);
+
+ return new Location(RANDOM.nextDouble() < 0.01 ? null : lat, RANDOM.nextDouble() < 0.01 ? null : lon);
+ }
+
+ private static Path createTempFile() {
+ try {
+ return new Path(Files.createTempFile("test-ci_", ".parquet").toAbsolutePath().toString());
+ } catch (IOException e) {
+ throw new AssertionError("Unable to create temporary file", e);
+ }
+ }
+
+ private List<User> readUsers(FilterPredicate filter, boolean useOtherFiltering) throws IOException {
+ return readUsers(FilterCompat.get(filter), useOtherFiltering, true);
+ }
+
+ private List<User> readUsers(FilterPredicate filter, boolean useOtherFiltering, boolean useColumnIndexFilter)
+ throws IOException {
+ return readUsers(FilterCompat.get(filter), useOtherFiltering, useColumnIndexFilter);
+ }
+
+ private List<User> readUsers(Filter filter, boolean useOtherFiltering) throws IOException {
+ return readUsers(filter, useOtherFiltering, true);
+ }
+
+ private List<User> readUsers(Filter filter, boolean useOtherFiltering, boolean useColumnIndexFilter)
+ throws IOException {
+ return PhoneBookWriter.readUsers(ParquetReader.builder(new GroupReadSupport(), file)
+ .withFilter(filter)
+ .useDictionaryFilter(useOtherFiltering)
+ .useStatsFilter(useOtherFiltering)
+ .useRecordFilter(useOtherFiltering)
+ .useColumnIndexFilter(useColumnIndexFilter));
+ }
+
+ // Assumes that both lists are in the same order
+ private static void assertContains(Stream<User> expected, List<User> actual) {
+ Iterator<User> expIt = expected.iterator();
+ if (!expIt.hasNext()) {
+ return;
+ }
+ User exp = expIt.next();
+ for (User act : actual) {
+ if (act.equals(exp)) {
+ if (!expIt.hasNext()) {
+ break;
+ }
+ exp = expIt.next();
+ }
+ }
+ assertFalse("Not all expected elements are in the actual list. E.g.: " + exp, expIt.hasNext());
+ }
+
+ private void assertCorrectFiltering(Predicate<User> expectedFilter, FilterPredicate actualFilter)
+ throws IOException {
+ // Check with only column index based filtering
+ List<User> result = readUsers(actualFilter, false);
+
+ assertTrue("Column-index filtering should drop some pages", result.size() < DATA.size());
+ LOGGER.info("{}/{} records read; filtering ratio: {}%", result.size(), DATA.size(),
+ 100 * result.size() / DATA.size());
+ // Asserts that all the required records are in the result
+ assertContains(DATA.stream().filter(expectedFilter), result);
+ // Asserts that all the retrieved records are in the file (validating non-matching records)
+ assertContains(result.stream(), DATA);
+
+ // Check with all the filtering filtering to ensure the result contains exactly the required values
+ result = readUsers(actualFilter, true);
+ assertEquals(DATA.stream().filter(expectedFilter).collect(Collectors.toList()), result);
+ }
+
+ @BeforeClass
+ public static void createFile() throws IOException {
+ int pageSize = DATA.size() / 10; // Ensure that several pages will be created
+ int rowGroupSize = pageSize * 6 * 5; // Ensure that there are more row-groups created
+ PhoneBookWriter.write(ExampleParquetWriter.builder(FILE_V1)
+ .withWriteMode(OVERWRITE)
+ .withRowGroupSize(rowGroupSize)
+ .withPageSize(pageSize)
+ .withWriterVersion(WriterVersion.PARQUET_1_0),
+ DATA);
+ PhoneBookWriter.write(ExampleParquetWriter.builder(FILE_V2)
+ .withWriteMode(OVERWRITE)
+ .withRowGroupSize(rowGroupSize)
+ .withPageSize(pageSize)
+ .withWriterVersion(WriterVersion.PARQUET_2_0),
+ DATA);
+ }
+
+ @AfterClass
+ public static void deleteFile() throws IOException {
+ FILE_V1.getFileSystem(new Configuration()).delete(FILE_V1, false);
+ FILE_V2.getFileSystem(new Configuration()).delete(FILE_V2, false);
+ }
+
+ @Test
+ public void testSimpleFiltering() throws IOException {
+ assertCorrectFiltering(
+ record -> record.getId() == 1234,
+ eq(longColumn("id"), 1234l));
+ assertCorrectFiltering(
+ record -> "miller".equals(record.getName()),
+ eq(binaryColumn("name"), Binary.fromString("miller")));
+ assertCorrectFiltering(
+ record -> record.getName() == null,
+ eq(binaryColumn("name"), null));
+ }
+
+ @Test
+ public void testNoFiltering() throws IOException {
+ // Column index filtering with no-op filter
+ assertEquals(DATA, readUsers(FilterCompat.NOOP, false));
+ assertEquals(DATA, readUsers(FilterCompat.NOOP, true));
+
+ // Column index filtering turned off
+ assertEquals(DATA.stream().filter(user -> user.getId() == 1234).collect(Collectors.toList()),
+ readUsers(eq(longColumn("id"), 1234l), true, false));
+ assertEquals(DATA.stream().filter(user -> "miller".equals(user.getName())).collect(Collectors.toList()),
+ readUsers(eq(binaryColumn("name"), Binary.fromString("miller")), true, false));
+ assertEquals(DATA.stream().filter(user -> user.getName() == null).collect(Collectors.toList()),
+ readUsers(eq(binaryColumn("name"), null), true, false));
+
+ // Every filtering mechanism turned off
+ assertEquals(DATA, readUsers(eq(longColumn("id"), 1234l), false, false));
+ assertEquals(DATA, readUsers(eq(binaryColumn("name"), Binary.fromString("miller")), false, false));
+ assertEquals(DATA, readUsers(eq(binaryColumn("name"), null), false, false));
+ }
+
+ @Test
+ public void testComplexFiltering() throws IOException {
+ assertCorrectFiltering(
+ record -> {
+ Location loc = record.getLocation();
+ Double lat = loc == null ? null : loc.getLat();
+ Double lon = loc == null ? null : loc.getLon();
+ return lat != null && lon != null && 37 <= lat && lat <= 70 && -21 <= lon && lon <= 35;
+ },
+ and(and(gtEq(doubleColumn("location.lat"), 37.0), ltEq(doubleColumn("location.lat"), 70.0)),
+ and(gtEq(doubleColumn("location.lon"), -21.0), ltEq(doubleColumn("location.lon"), 35.0))));
+ assertCorrectFiltering(
+ record -> {
+ Location loc = record.getLocation();
+ return loc == null || (loc.getLat() == null && loc.getLon() == null);
+ },
+ and(eq(doubleColumn("location.lat"), null), eq(doubleColumn("location.lon"), null)));
+ assertCorrectFiltering(
+ record -> {
+ String name = record.getName();
+ return name != null && name.compareTo("thomas") < 0 && record.getId() <= 3 * DATA.size() / 4;
+ },
+ and(lt(binaryColumn("name"), Binary.fromString("thomas")), ltEq(longColumn("id"), 3l * DATA.size() / 4)));
+ }
+
+ public static class NameStartsWithVowel extends UserDefinedPredicate<Binary> {
+ private static final Binary A = Binary.fromString("a");
+ private static final Binary B = Binary.fromString("b");
+ private static final Binary E = Binary.fromString("e");
+ private static final Binary F = Binary.fromString("f");
+ private static final Binary I = Binary.fromString("i");
+ private static final Binary J = Binary.fromString("j");
+ private static final Binary O = Binary.fromString("o");
+ private static final Binary P = Binary.fromString("p");
+ private static final Binary U = Binary.fromString("u");
+ private static final Binary V = Binary.fromString("v");
+
+ private static boolean isStartingWithVowel(String str) {
+ if (str == null || str.isEmpty()) {
+ return false;
+ }
+ switch (str.charAt(0)) {
+ case 'a':
+ case 'e':
+ case 'i':
+ case 'o':
+ case 'u':
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ @Override
+ public boolean keep(Binary value) {
+ return value != null && isStartingWithVowel(value.toStringUsingUTF8());
+ }
+
+ @Override
+ public boolean canDrop(Statistics<Binary> statistics) {
+ Comparator<Binary> cmp = statistics.getComparator();
+ Binary min = statistics.getMin();
+ Binary max = statistics.getMax();
+ return cmp.compare(max, A) < 0
+ || (cmp.compare(min, B) >= 0 && cmp.compare(max, E) < 0)
+ || (cmp.compare(min, F) >= 0 && cmp.compare(max, I) < 0)
+ || (cmp.compare(min, J) >= 0 && cmp.compare(max, O) < 0)
+ || (cmp.compare(min, P) >= 0 && cmp.compare(max, U) < 0)
+ || cmp.compare(min, V) >= 0;
+ }
+
+ @Override
+ public boolean inverseCanDrop(Statistics<Binary> statistics) {
+ Comparator<Binary> cmp = statistics.getComparator();
+ Binary min = statistics.getMin();
+ Binary max = statistics.getMax();
+ return (cmp.compare(min, A) >= 0 && cmp.compare(max, B) < 0)
+ || (cmp.compare(min, E) >= 0 && cmp.compare(max, F) < 0)
+ || (cmp.compare(min, I) >= 0 && cmp.compare(max, J) < 0)
+ || (cmp.compare(min, O) >= 0 && cmp.compare(max, P) < 0)
+ || (cmp.compare(min, U) >= 0 && cmp.compare(max, V) < 0);
+ }
+ }
+
+ public static class IsDivisibleBy extends UserDefinedPredicate<Long> implements Serializable {
+ private long divisor;
+
+ IsDivisibleBy(long divisor) {
+ this.divisor = divisor;
+ }
+
+ @Override
+ public boolean keep(Long value) {
+ return value != null && value % divisor == 0;
+ }
+
+ @Override
+ public boolean canDrop(Statistics<Long> statistics) {
+ long min = statistics.getMin();
+ long max = statistics.getMax();
+ return min % divisor != 0 && max % divisor != 0 && min / divisor == max / divisor;
+ }
+
+ @Override
+ public boolean inverseCanDrop(Statistics<Long> statistics) {
+ long min = statistics.getMin();
+ long max = statistics.getMax();
+ return min == max && min % divisor == 0;
+ }
+ }
+
+ @Test
+ public void testUDF() throws IOException {
+ assertCorrectFiltering(
+ record -> NameStartsWithVowel.isStartingWithVowel(record.getName()) || record.getId() % 234 == 0,
+ or(userDefined(binaryColumn("name"), NameStartsWithVowel.class),
+ userDefined(longColumn("id"), new IsDivisibleBy(234))));
+ assertCorrectFiltering(
+ record -> !(NameStartsWithVowel.isStartingWithVowel(record.getName()) || record.getId() % 234 == 0),
+ not(or(userDefined(binaryColumn("name"), NameStartsWithVowel.class),
+ userDefined(longColumn("id"), new IsDivisibleBy(234)))));
+ }
+
+ @Test
+ public void testFilteringWithMissingColumns() throws IOException {
+ // Missing column filter is always true
+ assertEquals(DATA, readUsers(notEq(binaryColumn("not-existing-binary"), Binary.EMPTY), true));
+ assertCorrectFiltering(
+ record -> record.getId() == 1234,
+ and(eq(longColumn("id"), 1234l),
+ eq(longColumn("not-existing-long"), null)));
+ assertCorrectFiltering(
+ record -> "miller".equals(record.getName()),
+ and(eq(binaryColumn("name"), Binary.fromString("miller")),
+ invert(userDefined(binaryColumn("not-existing-binary"), NameStartsWithVowel.class))));
+
+ // Missing column filter is always false
+ assertEquals(emptyList(), readUsers(lt(longColumn("not-existing-long"), 0l), true));
+ assertCorrectFiltering(
+ record -> "miller".equals(record.getName()),
+ or(eq(binaryColumn("name"), Binary.fromString("miller")),
+ gtEq(binaryColumn("not-existing-binary"), Binary.EMPTY)));
+ assertCorrectFiltering(
+ record -> record.getId() == 1234,
+ or(eq(longColumn("id"), 1234l),
+ userDefined(longColumn("not-existing-long"), new IsDivisibleBy(1))));
+ }
+}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetFileWriter.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetFileWriter.java
index 095b575c8..917ad5791 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetFileWriter.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetFileWriter.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.parquet.ParquetReadOptions;
import org.apache.parquet.Version;
import org.apache.parquet.bytes.BytesUtils;
import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel;
@@ -41,7 +42,11 @@
import org.apache.parquet.column.statistics.LongStatistics;
import org.apache.parquet.format.Statistics;
import org.apache.parquet.hadoop.metadata.*;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
import org.apache.parquet.hadoop.util.HiddenFileFilter;
+import org.apache.parquet.internal.column.columnindex.BoundaryOrder;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
import org.apache.parquet.io.api.Binary;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
@@ -51,6 +56,8 @@
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.*;
import static org.apache.parquet.CorruptStatistics.shouldIgnoreStatistics;
@@ -58,6 +65,7 @@
import static org.junit.Assert.*;
import static org.apache.parquet.column.Encoding.BIT_PACKED;
import static org.apache.parquet.column.Encoding.PLAIN;
+import static org.apache.parquet.format.converter.ParquetMetadataConverter.MAX_STATS_SIZE;
import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
import static org.apache.parquet.schema.Type.Repetition.*;
import static org.apache.parquet.hadoop.TestUtils.enforceEmptyDir;
@@ -766,4 +774,142 @@ public void testWriteMetadataFileWithRelativeOutputPath() throws IOException {
ParquetFileWriter.writeMetadataFile(conf, relativeRoot, footers, JobSummaryLevel.ALL);
}
+ @Test
+ public void testColumnIndexWriteRead() throws Exception {
+ File testFile = temp.newFile();
+ testFile.delete();
+
+ Path path = new Path(testFile.toURI());
+ Configuration configuration = new Configuration();
+
+ ParquetFileWriter w = new ParquetFileWriter(configuration, SCHEMA, path);
+ w.start();
+ w.startBlock(4);
+ w.startColumn(C1, 7, CODEC);
+ w.writeDataPage(7, 4, BytesInput.from(BYTES3), EMPTY_STATS, BIT_PACKED, BIT_PACKED, PLAIN);
+ w.endColumn();
+ w.startColumn(C2, 8, CODEC);
+ w.writeDataPage(8, 4, BytesInput.from(BYTES4), EMPTY_STATS, BIT_PACKED, BIT_PACKED, PLAIN);
+ w.endColumn();
+ w.endBlock();
+ w.startBlock(4);
+ w.startColumn(C1, 5, CODEC);
+ long c1p1Starts = w.getPos();
+ w.writeDataPage(2, 4, BytesInput.from(BYTES1), statsC1(null, Binary.fromString("aaa")), 1, BIT_PACKED, BIT_PACKED,
+ PLAIN);
+ long c1p2Starts = w.getPos();
+ w.writeDataPage(3, 4, BytesInput.from(BYTES1), statsC1(Binary.fromString("bbb"), Binary.fromString("ccc")), 3,
+ BIT_PACKED, BIT_PACKED, PLAIN);
+ w.endColumn();
+ long c1Ends = w.getPos();
+ w.startColumn(C2, 6, CODEC);
+ long c2p1Starts = w.getPos();
+ w.writeDataPage(2, 4, BytesInput.from(BYTES2), statsC2(117l, 100l), 1, BIT_PACKED, BIT_PACKED, PLAIN);
+ long c2p2Starts = w.getPos();
+ w.writeDataPage(3, 4, BytesInput.from(BYTES2), statsC2(null, null, null), 2, BIT_PACKED, BIT_PACKED, PLAIN);
+ long c2p3Starts = w.getPos();
+ w.writeDataPage(1, 4, BytesInput.from(BYTES2), statsC2(0l), 1, BIT_PACKED, BIT_PACKED, PLAIN);
+ w.endColumn();
+ long c2Ends = w.getPos();
+ w.endBlock();
+ w.startBlock(4);
+ w.startColumn(C1, 7, CODEC);
+ w.writeDataPage(7, 4, BytesInput.from(BYTES3),
+ // Creating huge stats so the column index will reach the limit and won't be written
+ statsC1(
+ Binary.fromConstantByteArray(new byte[(int) MAX_STATS_SIZE]),
+ Binary.fromConstantByteArray(new byte[1])),
+ 4, BIT_PACKED, BIT_PACKED, PLAIN);
+ w.endColumn();
+ w.startColumn(C2, 8, CODEC);
+ w.writeDataPage(8, 4, BytesInput.from(BYTES4), EMPTY_STATS, BIT_PACKED, BIT_PACKED, PLAIN);
+ w.endColumn();
+ w.endBlock();
+ w.end(new HashMap<String, String>());
+
+ try (ParquetFileReader reader = new ParquetFileReader(HadoopInputFile.fromPath(path, configuration),
+ ParquetReadOptions.builder().build())) {
+ ParquetMetadata footer = reader.getFooter();
+ assertEquals(3, footer.getBlocks().size());
+ BlockMetaData blockMeta = footer.getBlocks().get(1);
+ assertEquals(2, blockMeta.getColumns().size());
+
+ ColumnIndex columnIndex = reader.readColumnIndex(blockMeta.getColumns().get(0));
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertTrue(Arrays.asList(1l, 0l).equals(columnIndex.getNullCounts()));
+ assertTrue(Arrays.asList(false, false).equals(columnIndex.getNullPages()));
+ List<ByteBuffer> minValues = columnIndex.getMinValues();
+ assertEquals(2, minValues.size());
+ List<ByteBuffer> maxValues = columnIndex.getMaxValues();
+ assertEquals(2, maxValues.size());
+ assertEquals("aaa", new String(minValues.get(0).array(), StandardCharsets.UTF_8));
+ assertEquals("aaa", new String(maxValues.get(0).array(), StandardCharsets.UTF_8));
+ assertEquals("bbb", new String(minValues.get(1).array(), StandardCharsets.UTF_8));
+ assertEquals("ccc", new String(maxValues.get(1).array(), StandardCharsets.UTF_8));
+
+ columnIndex = reader.readColumnIndex(blockMeta.getColumns().get(1));
+ assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
+ assertTrue(Arrays.asList(0l, 3l, 0l).equals(columnIndex.getNullCounts()));
+ assertTrue(Arrays.asList(false, true, false).equals(columnIndex.getNullPages()));
+ minValues = columnIndex.getMinValues();
+ assertEquals(3, minValues.size());
+ maxValues = columnIndex.getMaxValues();
+ assertEquals(3, maxValues.size());
+ assertEquals(100, BytesUtils.bytesToLong(minValues.get(0).array()));
+ assertEquals(117, BytesUtils.bytesToLong(maxValues.get(0).array()));
+ assertEquals(0, minValues.get(1).array().length);
+ assertEquals(0, maxValues.get(1).array().length);
+ assertEquals(0, BytesUtils.bytesToLong(minValues.get(2).array()));
+ assertEquals(0, BytesUtils.bytesToLong(maxValues.get(2).array()));
+
+ OffsetIndex offsetIndex = reader.readOffsetIndex(blockMeta.getColumns().get(0));
+ assertEquals(2, offsetIndex.getPageCount());
+ assertEquals(c1p1Starts, offsetIndex.getOffset(0));
+ assertEquals(c1p2Starts, offsetIndex.getOffset(1));
+ assertEquals(c1p2Starts - c1p1Starts, offsetIndex.getCompressedPageSize(0));
+ assertEquals(c1Ends - c1p2Starts, offsetIndex.getCompressedPageSize(1));
+ assertEquals(0, offsetIndex.getFirstRowIndex(0));
+ assertEquals(1, offsetIndex.getFirstRowIndex(1));
+
+ offsetIndex = reader.readOffsetIndex(blockMeta.getColumns().get(1));
+ assertEquals(3, offsetIndex.getPageCount());
+ assertEquals(c2p1Starts, offsetIndex.getOffset(0));
+ assertEquals(c2p2Starts, offsetIndex.getOffset(1));
+ assertEquals(c2p3Starts, offsetIndex.getOffset(2));
+ assertEquals(c2p2Starts - c2p1Starts, offsetIndex.getCompressedPageSize(0));
+ assertEquals(c2p3Starts - c2p2Starts, offsetIndex.getCompressedPageSize(1));
+ assertEquals(c2Ends - c2p3Starts, offsetIndex.getCompressedPageSize(2));
+ assertEquals(0, offsetIndex.getFirstRowIndex(0));
+ assertEquals(1, offsetIndex.getFirstRowIndex(1));
+ assertEquals(3, offsetIndex.getFirstRowIndex(2));
+
+ assertNull(reader.readColumnIndex(footer.getBlocks().get(2).getColumns().get(0)));
+ }
+ }
+
+ private org.apache.parquet.column.statistics.Statistics<?> statsC1(Binary... values) {
+ org.apache.parquet.column.statistics.Statistics<?> stats = org.apache.parquet.column.statistics.Statistics
+ .createStats(C1.getPrimitiveType());
+ for (Binary value : values) {
+ if (value == null) {
+ stats.incrementNumNulls();
+ } else {
+ stats.updateStats(value);
+ }
+ }
+ return stats;
+ }
+
+ private org.apache.parquet.column.statistics.Statistics<?> statsC2(Long... values) {
+ org.apache.parquet.column.statistics.Statistics<?> stats = org.apache.parquet.column.statistics.Statistics
+ .createStats(C2.getPrimitiveType());
+ for (Long value : values) {
+ if (value == null) {
+ stats.incrementNumNulls();
+ } else {
+ stats.updateStats(value);
+ }
+ }
+ return stats;
+ }
}
diff --git a/parquet-tools/src/main/java/org/apache/parquet/tools/command/ColumnIndexCommand.java b/parquet-tools/src/main/java/org/apache/parquet/tools/command/ColumnIndexCommand.java
new file mode 100644
index 000000000..cbbd8a1fa
--- /dev/null
+++ b/parquet-tools/src/main/java/org/apache/parquet/tools/command/ColumnIndexCommand.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.tools.command;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.internal.column.columnindex.OffsetIndex;
+import org.apache.parquet.io.InputFile;
+import org.apache.parquet.tools.Main;
+
+/**
+ * parquet-tools command to print column and offset indexes.
+ */
+public class ColumnIndexCommand extends ArgsOnlyCommand {
+ public static final String[] USAGE = new String[] {
+ "<input>",
+ "where <input> is the parquet file to print the column and offset indexes for"
+ };
+
+ public static final Options OPTIONS;
+ static {
+ OPTIONS = new Options();
+ OPTIONS.addOption(Option.builder("c")
+ .longOpt("column")
+ .desc("Shows the column/offset indexes for the given column only; "
+ + "multiple columns shall be separated by commas")
+ .hasArg()
+ .build());
+ OPTIONS.addOption(Option.builder("r")
+ .longOpt("row-group")
+ .desc("Shows the column/offset indexes for the given row-groups only; "
+ + "multiple row-groups shall be speparated by commas; "
+ + "row-groups are referenced by their indexes from 0")
+ .hasArg()
+ .build());
+ OPTIONS.addOption(Option.builder("i")
+ .longOpt("column-index")
+ .desc("Shows the column indexes; "
+ + "active by default unless -o is used")
+ .hasArg(false)
+ .build());
+ OPTIONS.addOption(Option.builder("o")
+ .longOpt("offset-index")
+ .desc("Shows the offset indexes; "
+ + "active by default unless -i is used")
+ .hasArg(false)
+ .build());
+ }
+
+ public ColumnIndexCommand() {
+ super(1, 1);
+ }
+
+ @Override
+ public String[] getUsageDescription() {
+ return USAGE;
+ }
+
+ @Override
+ public String getCommandDescription() {
+ return "Prints the column and offset indexes of a Parquet file.";
+ }
+
+ @Override
+ public Options getOptions() {
+ return OPTIONS;
+ }
+
+ @Override
+ public void execute(CommandLine options) throws Exception {
+ super.execute(options);
+
+ String[] args = options.getArgs();
+ InputFile in = HadoopInputFile.fromPath(new Path(args[0]), new Configuration());
+ PrintWriter out = new PrintWriter(Main.out, true);
+ String rowGroupValue = options.getOptionValue("r");
+ Set<String> indexes = new HashSet<>();
+ if (rowGroupValue != null) {
+ indexes.addAll(Arrays.asList(rowGroupValue.split("\\s*,\\s*")));
+ }
+ boolean showColumnIndex = options.hasOption("i");
+ boolean showOffsetIndex = options.hasOption("o");
+ if (!showColumnIndex && !showOffsetIndex) {
+ showColumnIndex = true;
+ showOffsetIndex = true;
+ }
+
+ try (ParquetFileReader reader = ParquetFileReader.open(in)) {
+ boolean firstBlock = true;
+ int rowGroupIndex = 0;
+ for (BlockMetaData block : reader.getFooter().getBlocks()) {
+ if (!indexes.isEmpty() && !indexes.contains(Integer.toString(rowGroupIndex))) {
+ ++rowGroupIndex;
+ continue;
+ }
+ if (!firstBlock) {
+ out.println();
+ firstBlock = false;
+ }
+ out.format("row group %d:%n", rowGroupIndex);
+ for (ColumnChunkMetaData column : getColumns(block, options)) {
+ String path = column.getPath().toDotString();
+ if (showColumnIndex) {
+ out.format("column index for column %s:%n", path);
+ ColumnIndex columnIndex = reader.readColumnIndex(column);
+ if (columnIndex == null) {
+ out.println("NONE");
+ } else {
+ out.println(columnIndex);
+ }
+ }
+ if (showOffsetIndex) {
+ out.format("offset index for column %s:%n", path);
+ OffsetIndex offsetIndex = reader.readOffsetIndex(column);
+ if (offsetIndex == null) {
+ out.println("NONE");
+ } else {
+ out.println(offsetIndex);
+ }
+ }
+ }
+ ++rowGroupIndex;
+ }
+ }
+ }
+
+ private static List<ColumnChunkMetaData> getColumns(BlockMetaData block, CommandLine options) {
+ List<ColumnChunkMetaData> columns = block.getColumns();
+ String pathValue = options.getOptionValue("c");
+ if (pathValue == null) {
+ return columns;
+ }
+ String[] paths = pathValue.split("\\s*,\\s*");
+ Map<String, ColumnChunkMetaData> pathMap = new HashMap<>();
+ for (ColumnChunkMetaData column : columns) {
+ pathMap.put(column.getPath().toDotString(), column);
+ }
+
+ List<ColumnChunkMetaData> filtered = new ArrayList<>();
+ for (String path : paths) {
+ ColumnChunkMetaData column = pathMap.get(path);
+ if (column != null) {
+ filtered.add(column);
+ }
+ }
+ return filtered;
+ }
+
+}
diff --git a/parquet-tools/src/main/java/org/apache/parquet/tools/command/Registry.java b/parquet-tools/src/main/java/org/apache/parquet/tools/command/Registry.java
index 6df84be37..399efb731 100644
--- a/parquet-tools/src/main/java/org/apache/parquet/tools/command/Registry.java
+++ b/parquet-tools/src/main/java/org/apache/parquet/tools/command/Registry.java
@@ -34,6 +34,7 @@
registry.put("merge", MergeCommand.class);
registry.put("rowcount", RowCountCommand.class);
registry.put("size", SizeCommand.class);
+ registry.put("column-index", ColumnIndexCommand.class);
}
public static Map<String,Command> allCommands() {
diff --git a/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java b/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
index c07875ab8..7a1c81d6f 100644
--- a/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
+++ b/parquet-tools/src/main/java/org/apache/parquet/tools/read/SimpleRecordConverter.java
@@ -172,6 +172,16 @@ public DecimalConverter(String name, int scale) {
public void addBinary(Binary value) {
record.add(name, new BigDecimal(new BigInteger(value.getBytes()), scale));
}
+
+ @Override
+ public void addInt(int value) {
+ record.add(name, BigDecimal.valueOf(value).movePointLeft(scale));
+ }
+
+ @Override
+ public void addLong(long value) {
+ record.add(name, BigDecimal.valueOf(value).movePointLeft(scale));
+ }
}
}
diff --git a/pom.xml b/pom.xml
index 4c9d79c12..fb277cadc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,7 +81,7 @@
<hadoop1.version>1.2.1</hadoop1.version>
<cascading.version>2.7.1</cascading.version>
<cascading3.version>3.1.2</cascading3.version>
- <parquet.format.version>2.4.0</parquet.format.version>
+ <parquet.format.version>2.6.0</parquet.format.version>
<previous.version>1.7.0</previous.version>
<thrift.executable>thrift</thrift.executable>
<format.thrift.executable>thrift</format.thrift.executable>
@@ -221,13 +221,13 @@
</execution>
</executions>
</plugin>
-
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>2.7</version>
</plugin>
-
+
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
@@ -381,7 +381,7 @@
</execution>
</executions-->
</plugin>
-
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
@@ -396,8 +396,8 @@
<skip>true</skip>
</configuration>
</plugin>
-
-
+
+
<plugin>
<!-- Override source and target from the ASF parent -->
<groupId>org.apache.maven.plugins</groupId>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
> parquet-format code changes for encryption support
> --------------------------------------------------
>
> Key: PARQUET-1228
> URL: https://issues.apache.org/jira/browse/PARQUET-1228
> Project: Parquet
> Issue Type: Sub-task
> Components: parquet-format
> Reporter: Gidon Gershinsky
> Assignee: Gidon Gershinsky
> Priority: Major
> Labels: pull-request-available
>
> # Write/read the new Thrift structures (crypto metada)
> # Add encryption/decryption of Parquet metadata Thrift structures
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)