You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2023/01/16 09:15:04 UTC
[flink-table-store] branch master updated: [FLINK-30676] Introduce Data Structures for table store
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
The following commit(s) were added to refs/heads/master by this push:
new 6a326fe9 [FLINK-30676] Introduce Data Structures for table store
6a326fe9 is described below
commit 6a326fe9d001140fe99960e2c1e43c17bc7665e2
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Mon Jan 16 17:14:57 2023 +0800
[FLINK-30676] Introduce Data Structures for table store
This closes #483
---
.../file/mergetree/MergeTreeBenchmark.java | 24 +-
.../file/mergetree/MergeTreeReaderBenchmark.java | 8 +-
.../table/store/codegen/CodeGeneratorImpl.java | 12 +-
.../CodeGeneratorContext.scala | 8 +-
.../ComparatorCodeGenerator.scala | 2 +-
.../ExprCodeGenerator.scala | 20 +-
.../GenerateUtils.scala | 248 ++---
.../GeneratedExpression.scala | 4 +-
.../ProjectionCodeGenerator.scala | 12 +-
.../SortCodeGenerator.scala | 31 +-
.../flink/table/store/codegen/CodeGenerator.java | 8 +-
.../table/store/codegen/NormalizedKeyComputer.java | 6 +-
.../flink/table/store/codegen/Projection.java | 6 +-
.../table/store/codegen/RecordComparator.java | 6 +-
.../table/store/data/AbstractBinaryWriter.java | 60 +-
.../table/store/data/AbstractPagedInputView.java | 2 +-
.../table/store/data/AbstractPagedOutputView.java | 4 +-
.../store/data/AbstractRowDataSerializer.java | 12 +-
.../table/store/data/ArrayDataSerializer.java | 111 +-
.../apache/flink/table/store/data/BinaryArray.java | 564 ++++++++++
.../flink/table/store/data/BinaryArrayWriter.java | 41 +-
.../apache/flink/table/store/data/BinaryMap.java | 117 ++
.../apache/flink/table/store/data/BinaryRow.java | 442 ++++++++
...ataSerializer.java => BinaryRowSerializer.java} | 105 +-
.../flink/table/store/data/BinaryRowWriter.java | 21 +-
.../flink/table/store/data/BinarySection.java | 113 ++
.../flink/table/store/data/BinaryString.java | 1007 +++++++++++++++++
...Serializer.java => BinaryStringSerializer.java} | 55 +-
.../flink/table/store/data/BinaryWriter.java | 83 +-
.../apache/flink/table/store/data/DataGetters.java | 82 ++
.../apache/flink/table/store/data/DataSetters.java | 60 +
.../org/apache/flink/table/store/data/Decimal.java | 234 ++++
...lDataSerializer.java => DecimalSerializer.java} | 60 +-
.../flink/table/store/data/GenericArray.java | 316 ++++++
.../apache/flink/table/store/data/GenericMap.java | 123 +++
.../apache/flink/table/store/data/GenericRow.java | 270 +++++
.../flink/table/store/data/InternalArray.java | 153 +++
.../apache/flink/table/store/data/InternalMap.java | 52 +
.../apache/flink/table/store/data/InternalRow.java | 233 ++++
.../table/store/data/InternalSerializers.java | 39 +-
.../apache/flink/table/store/data/JoinedRow.java | 257 +++++
.../flink/table/store/data/MapDataSerializer.java | 110 +-
.../apache/flink/table/store/data/NestedRow.java | 331 ++++++
.../table/store/data/PagedTypeSerializer.java | 5 +-
.../table/store/data/RandomAccessInputView.java | 2 +-
.../table/store/data/RandomAccessOutputView.java | 51 +
.../flink/table/store/data/RowDataSerializer.java | 150 +--
.../store/data/SimpleCollectingOutputView.java | 4 +-
.../apache/flink/table/store/data/Timestamp.java | 202 ++++
...ataSerializer.java => TimestampSerializer.java} | 64 +-
.../store/data/columnar/ArrayColumnVector.java | 6 +-
.../store/data/columnar/BooleanColumnVector.java | 3 -
.../{ColumnarArrayData.java => ColumnarArray.java} | 44 +-
.../{ColumnarMapData.java => ColumnarMap.java} | 18 +-
.../{ColumnarRowData.java => ColumnarRow.java} | 52 +-
.../store/data/columnar/ColumnarRowIterator.java | 16 +-
.../store/data/columnar/DecimalColumnVector.java | 6 +-
.../table/store/data/columnar/Dictionary.java | 6 +-
.../table/store/data/columnar/MapColumnVector.java | 6 +-
.../table/store/data/columnar/RowColumnVector.java | 2 +-
.../store/data/columnar/TimestampColumnVector.java | 6 +-
.../store/data/columnar/VectorizedColumnBatch.java | 22 +-
.../store/data/columnar/heap/HeapArrayVector.java | 10 +-
.../store/data/columnar/heap/HeapMapVector.java | 10 +-
.../store/data/columnar/heap/HeapRowVector.java | 10 +-
.../data/columnar/heap/HeapTimestampVector.java | 12 +-
.../columnar/writable/WritableTimestampVector.java | 10 +-
.../data/converter/ArrayBooleanArrayConverter.java | 41 -
.../data/converter/ArrayByteArrayConverter.java | 41 -
.../data/converter/ArrayDoubleArrayConverter.java | 41 -
.../data/converter/ArrayFloatArrayConverter.java | 41 -
.../data/converter/ArrayIntArrayConverter.java | 41 -
.../store/data/converter/ArrayListConverter.java | 83 --
.../data/converter/ArrayLongArrayConverter.java | 41 -
.../data/converter/ArrayObjectArrayConverter.java | 208 ----
.../data/converter/ArrayShortArrayConverter.java | 41 -
.../data/converter/DataStructureConverter.java | 92 --
.../data/converter/DataStructureConverters.java | 300 -----
.../store/data/converter/DateDateConverter.java | 40 -
.../data/converter/DateLocalDateConverter.java | 42 -
.../DayTimeIntervalDurationConverter.java | 44 -
.../data/converter/DecimalBigDecimalConverter.java | 61 --
.../store/data/converter/IdentityConverter.java | 43 -
.../LocalZonedTimestampInstantConverter.java | 41 -
.../converter/LocalZonedTimestampIntConverter.java | 41 -
.../LocalZonedTimestampLongConverter.java | 41 -
.../LocalZonedTimestampTimestampConverter.java | 47 -
.../store/data/converter/MapMapConverter.java | 124 ---
.../data/converter/RawByteArrayConverter.java | 69 --
.../store/data/converter/RawObjectConverter.java | 69 --
.../store/data/converter/RowRowConverter.java | 134 ---
.../data/converter/StringByteArrayConverter.java | 41 -
.../data/converter/StringStringConverter.java | 41 -
.../data/converter/StructuredObjectConverter.java | 360 ------
.../data/converter/TimeLocalTimeConverter.java | 41 -
.../store/data/converter/TimeLongConverter.java | 39 -
.../store/data/converter/TimeTimeConverter.java | 40 -
.../converter/TimestampLocalDateTimeConverter.java | 41 -
.../converter/TimestampTimestampConverter.java | 41 -
.../YearMonthIntervalPeriodConverter.java | 80 --
.../table/store/file/predicate/CompareUtils.java | 4 +-
.../flink/table/store/file/predicate/Equal.java | 6 +-
.../flink/table/store/file/predicate/FieldRef.java | 8 +-
.../table/store/file/predicate/GreaterOrEqual.java | 6 +-
.../table/store/file/predicate/GreaterThan.java | 6 +-
.../flink/table/store/file/predicate/In.java | 6 +-
.../table/store/file/predicate/IsNotNull.java | 6 +-
.../flink/table/store/file/predicate/IsNull.java | 6 +-
.../table/store/file/predicate/LeafFunction.java | 6 +-
.../table/store/file/predicate/LeafPredicate.java | 8 +-
.../store/file/predicate/LeafUnaryFunction.java | 10 +-
.../table/store/file/predicate/LessOrEqual.java | 6 +-
.../flink/table/store/file/predicate/LessThan.java | 6 +-
.../flink/table/store/file/predicate/NotEqual.java | 6 +-
.../flink/table/store/file/predicate/NotIn.java | 6 +-
.../predicate/NullFalseLeafBinaryFunction.java | 10 +-
.../store/file/predicate/PredicateBuilder.java | 69 +-
.../store/file/predicate/PredicateFilter.java | 10 +-
.../table/store/file/predicate/StartsWith.java | 18 +-
.../table/store/format/FieldStatsCollector.java | 8 +-
.../flink/table/store/format/FileFormat.java | 6 +-
.../table/store/format/FormatReaderFactory.java | 4 +-
.../flink/table/store/memory/MemorySegment.java | 610 +++++++++++
.../MemorySegmentSource.java} | 19 +-
.../table/store/memory/MemorySegmentUtils.java | 1153 ++++++++++++++++++++
.../table/store/memory/MemorySegmentWritable.java | 20 +-
.../flink/table/store/memory/MemoryUtils.java | 176 +++
.../flink/table/store/types/BooleanType.java | 2 +-
.../apache/flink/table/store/types/DataField.java | 4 +
.../apache/flink/table/store/types/DataType.java | 8 +
.../flink/table/store/types/DataTypeCasts.java | 283 +++++
.../flink/table/store/types/DataTypeChecks.java | 272 +++++
.../table/store/types/DataTypeDefaultVisitor.java | 128 +++
.../apache/flink/table/store/types/DataTypes.java | 125 +++
.../flink/table/store/types/ReassignFieldId.java | 66 ++
.../apache/flink/table/store/types/RowKind.java | 114 ++
.../apache/flink/table/store/types/RowType.java | 86 ++
.../flink/table/store/utils/BinaryRowDataUtil.java | 16 +-
.../flink/table/store/utils/DateTimeUtils.java | 12 +-
.../flink/table/store/utils/DecimalUtils.java | 33 +-
...yProjectedRowData.java => KeyProjectedRow.java} | 44 +-
.../flink/table/store/utils/MurmurHashUtils.java | 178 +++
.../{ProjectedRowData.java => ProjectedRow.java} | 76 +-
.../apache/flink/table/store/utils/Projection.java | 74 +-
.../store/utils/RowDataToObjectArrayConverter.java | 12 +-
.../flink/table/store/utils/RowDataUtils.java | 178 ++-
.../flink/table/store/utils/SegmentsUtil.java | 82 --
.../apache/flink/table/store/utils/SortUtil.java | 115 +-
.../flink/table/store/utils/StringUtils.java | 18 +-
.../apache/flink/table/store/utils/TypeUtils.java | 67 +-
.../flink/table/store/data/BinaryRowTest.java | 901 +++++++++++++++
.../flink/table/store/data/BinaryStringTest.java | 505 +++++++++
.../flink/table/store/data/BinaryWriterTest.java | 159 ---
.../flink/table/store/data/DataFormatTestUtil.java | 152 +++
.../apache/flink/table/store/data/DecimalTest.java | 115 ++
.../apache/flink/table/store/data/MapDataUtil.java | 47 +
.../flink/table/store/data/NestedRowTest.java | 123 +++
.../apache/flink/table/store/data/RowDataTest.java | 240 ++++
.../flink/table/store/data/TimestampTest.java | 133 +++
.../converter/DataStructureConvertersTest.java | 808 --------------
.../table/store/datagen/DataGenVisitorBase.java | 18 +-
.../store/datagen/RandomGeneratorVisitor.java | 113 +-
.../table/store/datagen/RowDataGenerator.java | 12 +-
.../store/format/FileStatsExtractorTestBase.java | 94 +-
.../table/store/memory/MemorySegmentTestBase.java | 1070 ++++++++++++++++++
.../memory/OffHeapDirectMemorySegmentTest.java | 70 ++
.../store/memory/OnHeapMemorySegmentTest.java | 96 ++
.../table/store/types/DataTypeChecksTest.java | 65 ++
.../table/store/types/DataTypeFactoryMock.java | 112 --
.../flink/table/store/utils/RowDataUtilsTest.java | 87 +-
.../store/connector/TableStoreManagedFactory.java | 2 +-
.../store/connector/AbstractTableStoreFactory.java | 7 +-
.../store/connector}/DataTypeToLogicalType.java | 27 +-
.../flink/table/store/connector/FlinkCatalog.java | 34 +-
.../flink/table/store/connector/FlinkRowData.java | 328 ++++++
.../table/store/connector/FlinkRowWrapper.java | 326 ++++++
.../store/connector}/LogicalTypeConversion.java | 18 +-
.../store/connector}/LogicalTypeToDataType.java | 4 +-
.../table/store/connector}/PredicateConverter.java | 44 +-
.../table/store/connector}/ProjectedRowData.java | 14 +-
.../flink/table/store/connector/Projection.java | 235 ++++
.../table/store/connector/SystemCatalogTable.java | 5 +-
.../connector/TableStoreConnectorFactory.java | 3 +-
.../connector/lookup/FileStoreLookupFunction.java | 16 +-
.../table/store/connector/lookup/LookupTable.java | 10 +-
.../connector/lookup/PrimaryKeyLookupTable.java | 26 +-
.../store/connector/lookup/RocksDBSetState.java | 16 +-
.../table/store/connector/lookup/RocksDBState.java | 12 +-
.../connector/lookup/RocksDBStateFactory.java | 10 +-
.../store/connector/lookup/RocksDBValueState.java | 16 +-
.../lookup/SecondaryIndexLookupTable.java | 32 +-
.../connector/sink/BucketStreamPartitioner.java | 4 +-
.../sink/FullChangelogStoreSinkWrite.java | 42 +-
.../sink/OffsetRowDataHashStreamPartitioner.java | 16 +-
.../store/connector/sink/StoreCompactOperator.java | 15 +-
.../table/store/connector/sink/StoreSinkWrite.java | 11 +-
.../store/connector/sink/StoreSinkWriteImpl.java | 11 +-
.../store/connector/sink/StoreWriteOperator.java | 3 +-
.../connector/source/CompactorSourceBuilder.java | 10 +-
.../source/FileStoreSourceSplitReader.java | 39 +-
.../store/connector/source/FlinkSourceBuilder.java | 6 +-
.../store/connector/source/FlinkTableSource.java | 5 +-
.../store/kafka/KafkaLogDeserializationSchema.java | 35 +-
.../store/kafka/KafkaLogSerializationSchema.java | 10 +-
.../table/store/connector/ChangelogModeTest.java | 4 +-
.../table/store/connector/FileStoreITCase.java | 3 +-
.../store/connector/ForceCompactionITCase.java | 8 +-
.../store/connector}/PredicateConverterTest.java | 102 +-
.../store/connector/ReadWriteTableITCase.java | 3 +-
.../connector/TableStoreManagedFactoryTest.java | 400 -------
.../connector/action/CompactActionITCase.java | 57 +-
.../store/connector/lookup/LookupTableTest.java | 24 +-
.../connector/sink/CommitterOperatorTest.java | 22 +-
.../connector/sink/CommitterOperatorTestBase.java | 19 +-
.../store/connector/sink/CompactorSinkITCase.java | 33 +-
.../connector/source/CompactorSourceITCase.java | 75 +-
.../source/FileStoreSourceReaderTest.java | 13 +-
.../source/FileStoreSourceSplitReaderTest.java | 25 +-
.../source/FileStoreSourceSplitSerializerTest.java | 12 +-
.../source/TestChangelogDataReadWrite.java | 34 +-
.../flink/table/store/kafka/KafkaLogITCase.java | 27 +-
.../store/kafka/KafkaLogSerializationTest.java | 13 +-
.../flink/table/store/kafka/KafkaLogTestUtils.java | 8 +-
flink-table-store-core/pom.xml | 8 +
.../flink/table/store/codegen/CodeGenUtils.java | 14 +-
.../flink/table/store/file/AbstractFileStore.java | 6 +-
.../table/store/file/AppendOnlyFileStore.java | 10 +-
.../apache/flink/table/store/file/FileStore.java | 2 +-
.../apache/flink/table/store/file/KeyValue.java | 33 +-
.../flink/table/store/file/KeyValueFileStore.java | 8 +-
.../flink/table/store/file/KeyValueSerializer.java | 41 +-
.../table/store/file/append/AppendOnlyWriter.java | 10 +-
.../table/store/file/casting/CastExecutors.java | 92 +-
.../table/store/file/disk/BufferFileWriter.java | 2 +-
.../store/file/disk/ChannelReaderInputView.java | 7 +-
.../file/disk/ChannelReaderInputViewIterator.java | 2 +-
.../store/file/disk/ChannelWriterOutputView.java | 8 +-
.../store/file/io/AbstractFileRecordIterator.java | 14 +-
.../flink/table/store/file/io/DataFileMeta.java | 60 +-
.../store/file/io/DataFileMetaSerializer.java | 14 +-
.../file/io/KeyValueDataFileRecordReader.java | 14 +-
.../store/file/io/KeyValueDataFileWriter.java | 14 +-
.../store/file/io/KeyValueFileReaderFactory.java | 8 +-
.../store/file/io/KeyValueFileWriterFactory.java | 12 +-
.../store/file/io/RowDataFileRecordReader.java | 22 +-
.../table/store/file/io/RowDataFileWriter.java | 14 +-
.../store/file/io/RowDataRollingFileWriter.java | 8 +-
.../table/store/file/io/SingleFileWriter.java | 14 +-
.../file/io/StatsCollectingSingleFileWriter.java | 10 +-
.../table/store/file/manifest/ManifestEntry.java | 35 +-
.../file/manifest/ManifestEntrySerializer.java | 10 +-
.../table/store/file/manifest/ManifestFile.java | 10 +-
.../store/file/manifest/ManifestFileMeta.java | 21 +-
.../file/manifest/ManifestFileMetaSerializer.java | 14 +-
.../table/store/file/manifest/ManifestList.java | 10 +-
.../flink/table/store/file/memory/Buffer.java | 2 +-
.../store/file/memory/HeapMemorySegmentPool.java | 5 +-
.../table/store/file/memory/MemoryPoolFactory.java | 2 +-
.../table/store/file/memory/MemorySegmentPool.java | 4 +-
.../store/file/mergetree/DropDeleteReader.java | 2 +-
.../flink/table/store/file/mergetree/Levels.java | 7 +-
.../store/file/mergetree/MergeTreeReaders.java | 6 +-
.../store/file/mergetree/MergeTreeWriter.java | 8 +-
.../file/mergetree/SortBufferWriteBuffer.java | 48 +-
.../table/store/file/mergetree/SortedRun.java | 6 +-
.../table/store/file/mergetree/WriteBuffer.java | 8 +-
.../compact/FullChangelogMergeFunctionWrapper.java | 2 +-
.../FullChangelogMergeTreeCompactRewriter.java | 4 +-
.../file/mergetree/compact/IntervalPartition.java | 10 +-
.../mergetree/compact/MergeTreeCompactManager.java | 6 +-
.../compact/MergeTreeCompactRewriter.java | 6 +-
.../mergetree/compact/MergeTreeCompactTask.java | 4 +-
.../compact/PartialUpdateMergeFunction.java | 24 +-
.../file/mergetree/compact/SortMergeReader.java | 8 +-
.../mergetree/compact/ValueCountMergeFunction.java | 13 +-
.../compact/aggregate/AggregateMergeFunction.java | 29 +-
.../compact/aggregate/FieldAggregator.java | 10 +-
.../compact/aggregate/FieldBoolAndAgg.java | 6 +-
.../compact/aggregate/FieldBoolOrAgg.java | 6 +-
.../aggregate/FieldLastNonNullValueAgg.java | 6 +-
.../compact/aggregate/FieldLastValueAgg.java | 6 +-
.../compact/aggregate/FieldListaggAgg.java | 17 +-
.../mergetree/compact/aggregate/FieldMaxAgg.java | 10 +-
.../mergetree/compact/aggregate/FieldMinAgg.java | 10 +-
.../mergetree/compact/aggregate/FieldSumAgg.java | 16 +-
.../file/operation/AbstractFileStoreScan.java | 10 +-
.../file/operation/AbstractFileStoreWrite.java | 27 +-
.../file/operation/AppendOnlyFileStoreRead.java | 14 +-
.../file/operation/AppendOnlyFileStoreScan.java | 2 +-
.../file/operation/AppendOnlyFileStoreWrite.java | 26 +-
.../store/file/operation/FileStoreCommitImpl.java | 20 +-
.../table/store/file/operation/FileStoreScan.java | 8 +-
.../table/store/file/operation/FileStoreWrite.java | 9 +-
.../file/operation/KeyValueFileStoreRead.java | 12 +-
.../file/operation/KeyValueFileStoreScan.java | 5 +-
.../file/operation/KeyValueFileStoreWrite.java | 24 +-
.../table/store/file/predicate/BucketSelector.java | 8 +-
.../table/store/file/schema/SchemaChange.java | 42 +-
.../store/file/schema/SchemaEvolutionUtil.java | 13 +-
.../table/store/file/schema/SchemaManager.java | 74 +-
.../flink/table/store/file/schema/TableSchema.java | 13 +-
.../table/store/file/schema/UpdateSchema.java | 49 +-
.../store/file/sort/BinaryExternalMerger.java | 24 +-
.../store/file/sort/BinaryExternalSortBuffer.java | 32 +-
.../store/file/sort/BinaryInMemorySortBuffer.java | 31 +-
.../store/file/sort/BinaryIndexedSortable.java | 26 +-
.../table/store/file/sort/BinaryMergeIterator.java | 2 +-
.../flink/table/store/file/sort/SortBuffer.java | 8 +-
.../table/store/file/stats/BinaryTableStats.java | 35 +-
.../file/stats/FieldStatsArraySerializer.java | 43 +-
.../table/store/file/utils/BulkFormatMapping.java | 7 +-
.../store/file/utils/FileStorePathFactory.java | 12 +-
.../flink/table/store/file/utils/FileUtils.java | 6 +-
.../store/file/utils/KeyComparatorSupplier.java | 8 +-
.../table/store/file/utils/ObjectSerializer.java | 12 +-
.../utils/{OffsetRowData.java => OffsetRow.java} | 46 +-
.../store/file/utils/RowDataPartitionComputer.java | 12 +-
.../table/store/file/utils/SerializationUtils.java | 42 +-
.../file/utils/VersionedObjectSerializer.java | 27 +-
.../store/table/AppendOnlyFileStoreTable.java | 6 +-
.../table/ChangelogValueCountFileStoreTable.java | 19 +-
.../table/ChangelogWithKeyFileStoreTable.java | 22 +-
.../flink/table/store/table/FileStoreTable.java | 5 +-
.../flink/table/store/table/SupportsWrite.java | 10 +-
.../org/apache/flink/table/store/table/Table.java | 2 +-
.../table/store/table/sink/BucketComputer.java | 36 +-
.../table/store/table/sink/FileCommittable.java | 8 +-
.../table/store/table/sink/SequenceGenerator.java | 52 +-
.../flink/table/store/table/sink/SinkRecord.java | 20 +-
.../store/table/sink/SinkRecordConverter.java | 22 +-
.../flink/table/store/table/sink/TableWrite.java | 13 +-
.../table/store/table/sink/TableWriteImpl.java | 11 +-
.../store/table/source/AbstractDataTableScan.java | 8 +-
.../flink/table/store/table/source/DataSplit.java | 12 +-
.../store/table/source/KeyValueTableRead.java | 13 +-
.../table/source/MergeTreeSplitGenerator.java | 6 +-
.../table/source/ResetRowKindRecordIterator.java | 7 +-
.../flink/table/store/table/source/TableRead.java | 10 +-
.../store/table/source/TableStreamingReader.java | 11 +-
.../source/ValueContentRowDataRecordIterator.java | 6 +-
.../source/ValueCountRowDataRecordIterator.java | 12 +-
.../table/store/table/system/AuditLogTable.java | 34 +-
.../table/store/table/system/BucketsTable.java | 41 +-
.../table/store/table/system/OptionsTable.java | 29 +-
.../table/store/table/system/SchemasTable.java | 47 +-
.../table/store/table/system/SnapshotsTable.java | 49 +-
.../flink/table/store/file/FileFormatTest.java | 29 +-
.../flink/table/store/file/TestFileStore.java | 30 +-
.../table/store/file/TestKeyValueGenerator.java | 52 +-
.../store/file/append/AppendOnlyWriterTest.java | 40 +-
.../table/store/file/casting/CastExecutorTest.java | 106 +-
.../file/disk/BufferFileWriterReaderTest.java | 5 +-
.../store/file/format/FileFormatSuffixTest.java | 16 +-
.../file/format/FileStatsExtractingAvroFormat.java | 6 +-
.../store/file/format/FlushingFileFormat.java | 12 +-
.../store/file/io/DataFileTestDataGenerator.java | 26 +-
.../table/store/file/io/DataFileTestUtils.java | 12 +-
.../store/file/io/KeyValueFileReadWriteTest.java | 30 +-
.../table/store/file/io/RollingFileWriterTest.java | 16 +-
.../ManifestCommittableSerializerTest.java | 4 +-
.../store/file/manifest/ManifestFileMetaTest.java | 18 +-
.../file/manifest/ManifestTestDataGenerator.java | 6 +-
.../table/store/file/mergetree/LevelsTest.java | 4 +-
.../table/store/file/mergetree/MergeTreeTest.java | 30 +-
.../mergetree/SortBufferWriteBufferTestBase.java | 13 +-
.../compact/CombiningRecordReaderTestBase.java | 4 +-
.../FullChangelogMergeFunctionWrapperTestBase.java | 2 +-
.../mergetree/compact/IntervalPartitionTest.java | 8 +-
.../mergetree/compact/MergeFunctionTestUtils.java | 2 +-
.../compact/MergeTreeCompactManagerTest.java | 8 +-
.../ReducerMergeFunctionWrapperTestBase.java | 6 +-
.../compact/aggregate/FieldAggregatorTest.java | 13 +-
.../file/operation/CleanedFileStoreExpireTest.java | 6 +-
.../store/file/operation/FileStoreCommitTest.java | 48 +-
.../file/operation/FileStoreExpireTestBase.java | 6 +-
.../file/operation/KeyValueFileStoreReadTest.java | 50 +-
.../file/operation/KeyValueFileStoreScanTest.java | 34 +-
.../store/file/operation/TestCommitThread.java | 34 +-
.../store/file/predicate/BucketSelectorTest.java | 4 +-
.../store/file/predicate/PredicateBuilderTest.java | 4 +-
.../table/store/file/predicate/PredicateTest.java | 4 +-
.../store/file/schema/SchemaEvolutionUtilTest.java | 20 +-
.../table/store/file/schema/SchemaManagerTest.java | 12 +-
.../table/store/file/schema/TableSchemaTest.java | 69 +-
.../file/sort/BinaryExternalSortBufferTest.java | 52 +-
.../store/file/sort/IntNormalizedKeyComputer.java | 6 +-
.../table/store/file/sort/IntRecordComparator.java | 4 +-
.../store/file/stats/BinaryTableStatsTest.java | 20 +-
.../file/stats/FieldStatsArraySerializerTest.java | 6 +-
.../store/file/stats/FieldStatsCollectorTest.java | 46 +-
.../table/store/file/stats/StatsTestUtils.java | 4 +-
.../store/file/stats/TestFileStatsExtractor.java | 14 +-
.../store/file/utils/FileStorePathFactoryTest.java | 22 +-
.../table/store/file/utils/OffsetRowDataTest.java | 156 ---
.../table/store/file/utils/OffsetRowTest.java | 147 +++
.../table/store/file/utils/ReusingKeyValue.java | 12 +-
.../table/store/file/utils/ReusingTestData.java | 2 +-
.../store/table/AppendOnlyFileStoreTableTest.java | 10 +-
.../ChangelogValueCountFileStoreTableTest.java | 2 +-
.../table/ChangelogWithKeyFileDataTableTest.java | 23 +-
.../table/ChangelogWithKeyFileStoreTableTest.java | 66 +-
...hangelogWithKeyTableColumnTypeFileMetaTest.java | 16 +-
.../store/table/ColumnTypeFileMetaTestBase.java | 37 +-
.../table/store/table/FileDataFilterTestBase.java | 59 +-
.../table/store/table/FileStoreTableTestBase.java | 88 +-
.../store/table/SchemaEvolutionTableTestBase.java | 24 +-
.../table/store/table/SchemaEvolutionTest.java | 63 +-
.../table/store/table/WritePreemptMemoryTest.java | 4 +-
.../store/table/sink/SinkRecordConverterTest.java | 18 +-
.../source/RowDataRecordIteratorTestBase.java | 16 +-
.../store/table/source/SplitGeneratorTest.java | 4 +-
.../ValueContentRowDataRecordIteratorTest.java | 4 +-
.../ValueCountRowDataRecordIteratorTest.java | 10 +-
.../snapshot/CompactedStartingScannerTest.java | 2 +-
.../CompactionChangelogFollowUpScannerTest.java | 2 +-
.../ContinuousCompactorFollowUpScannerTest.java | 6 +-
.../ContinuousCompactorStartingScannerTest.java | 2 +-
.../ContinuousDataFileSnapshotEnumeratorTest.java | 2 +-
...ContinuousFromTimestampStartingScannerTest.java | 2 +-
.../ContinuousLatestStartingScannerTest.java | 2 +-
.../source/snapshot/DeltaFollowUpScannerTest.java | 2 +-
.../source/snapshot/FullStartingScannerTest.java | 2 +-
.../InputChangelogFollowUpScannerTest.java | 2 +-
.../snapshot/SnapshotEnumeratorTestBase.java | 44 +-
.../StaticDataFileSnapshotEnumeratorTest.java | 2 +-
.../store/format/avro/AbstractAvroBulkFormat.java | 24 +-
.../table/store/format/avro/AvroFileFormat.java | 26 +-
.../store/format/avro/AvroSchemaConverter.java | 66 +-
.../store/format/avro/AvroToRowDataConverters.java | 61 +-
.../store/format/avro/RowDataToAvroConverters.java | 76 +-
.../table/store/format/orc/OrcFileFormat.java | 56 +-
.../table/store/format/orc/OrcReaderFactory.java | 27 +-
.../format/orc/filter/OrcFileStatsExtractor.java | 37 +-
.../table/store/format/orc/filter/OrcFilters.java | 338 ------
.../orc/filter/OrcPredicateFunctionVisitor.java | 12 +-
.../format/orc/reader/AbstractOrcColumnVector.java | 20 +-
.../format/orc/reader/OrcArrayColumnVector.java | 10 +-
.../format/orc/reader/OrcDecimalColumnVector.java | 6 +-
.../orc/reader/OrcLegacyTimestampColumnVector.java | 21 +-
.../format/orc/reader/OrcMapColumnVector.java | 10 +-
.../format/orc/reader/OrcRowColumnVector.java | 14 +-
.../format/orc/reader/OrcSplitReaderUtil.java | 28 +-
.../orc/reader/OrcTimestampColumnVector.java | 10 +-
.../store/format/orc/writer/RowDataVectorizer.java | 81 +-
.../store/format/parquet/ParquetFileFormat.java | 6 +-
.../format/parquet/ParquetFileStatsExtractor.java | 45 +-
.../store/format/parquet/ParquetReaderFactory.java | 25 +-
.../format/parquet/ParquetSchemaConverter.java | 16 +-
.../table/store/format/parquet/ParquetUtil.java | 6 +-
.../format/parquet/reader/ArrayColumnReader.java | 24 +-
.../parquet/reader/BaseVectorizedColumnReader.java | 8 +-
.../parquet/reader/FixedLenBytesColumnReader.java | 2 +-
.../parquet/reader/ParquetDataColumnReader.java | 6 +-
.../reader/ParquetDataColumnReaderFactory.java | 12 +-
.../parquet/reader/ParquetDecimalVector.java | 11 +-
.../format/parquet/reader/ParquetDictionary.java | 4 +-
.../parquet/reader/ParquetSplitReaderUtil.java | 14 +-
.../parquet/reader/TimestampColumnReader.java | 18 +-
.../parquet/writer/ParquetRowDataBuilder.java | 15 +-
.../parquet/writer/ParquetRowDataWriter.java | 133 ++-
.../parquet/writer/RowDataParquetBuilder.java | 10 +-
.../table/store/format/BulkFileFormatTest.java | 32 +-
.../store/format/avro/AvroBulkFormatTest.java | 39 +-
.../store/format/avro/AvroBulkFormatTestUtils.java | 25 +-
.../format/orc/OrcFileStatsExtractorTest.java | 80 +-
.../store/format/orc/OrcFilterConverterTest.java | 7 +-
.../store/format/orc/OrcReaderFactoryTest.java | 58 +-
.../store/format/orc/OrcSplitReaderUtilTest.java | 21 +-
.../parquet/ParquetFileStatsExtractorTest.java | 52 +-
.../store/format/parquet/ParquetReadWriteTest.java | 268 +++--
.../apache/flink/table/store/hive/HiveCatalog.java | 5 +-
.../flink-table-store-hive-common/pom.xml | 7 +
.../flink/table/store/hive/HiveTypeUtils.java | 16 +-
.../TableStoreTimestampObjectInspector.java | 20 +-
.../TableStoreTimestampObjectInspector.java | 8 +-
.../apache/flink/table/store/RowDataContainer.java | 10 +-
.../store/SearchArgumentToPredicateConverter.java | 14 +-
.../apache/flink/table/store/hive/HiveSchema.java | 8 +-
.../TableStoreCharObjectInspector.java | 6 +-
.../TableStoreDecimalObjectInspector.java | 8 +-
.../TableStoreListObjectInspector.java | 16 +-
.../TableStoreMapObjectInspector.java | 32 +-
.../TableStoreObjectInspectorFactory.java | 18 +-
.../TableStoreRowDataObjectInspector.java | 18 +-
.../TableStoreStringObjectInspector.java | 6 +-
.../TableStoreVarcharObjectInspector.java | 6 +-
.../table/store/mapred/TableStoreInputFormat.java | 4 +-
.../table/store/mapred/TableStoreOutputFormat.java | 6 +-
.../table/store/mapred/TableStoreRecordReader.java | 14 +-
.../flink/table/store/FileStoreTestUtils.java | 2 +-
.../SearchArgumentToPredicateConverterTest.java | 65 +-
.../table/store/hive/HiveTableSchemaTest.java | 24 +-
.../store/hive/RandomGenericRowDataGenerator.java | 88 +-
.../hive/TableStoreHiveStorageHandlerITCase.java | 235 ++--
.../table/store/hive/TableStoreSerDeTest.java | 6 +-
.../TableStoreCharObjectInspectorTest.java | 12 +-
.../TableStoreDecimalObjectInspectorTest.java | 8 +-
.../TableStoreListObjectInspectorTest.java | 24 +-
.../TableStoreMapObjectInspectorTest.java | 30 +-
.../TableStoreRowDataObjectInspectorTest.java | 4 +-
.../TableStoreStringObjectInspectorTest.java | 8 +-
.../TableStoreTimestampObjectInspectorTest.java | 13 +-
.../TableStoreVarcharObjectInspectorTest.java | 12 +-
.../store/mapred/TableStoreInputSplitTest.java | 4 +-
.../store/mapred/TableStoreRecordReaderTest.java | 77 +-
.../flink/table/store/spark/SparkArrayData.java | 20 +-
.../flink/table/store/spark/SparkCatalog.java | 2 +-
.../table/store/spark/SparkFilterConverter.java | 6 +-
.../flink/table/store/spark/SparkInternalRow.java | 85 +-
.../table/store/spark/SparkReaderFactory.java | 16 +-
.../spark/{SparkRowData.java => SparkRow.java} | 112 +-
.../flink/table/store/spark/SparkTypeUtils.java | 97 +-
.../apache/flink/table/store/spark/SparkWrite.java | 6 +-
.../table/store/spark/SimpleTableTestHelper.java | 6 +-
.../store/spark/SparkFilterConverterTest.java | 23 +-
.../table/store/spark/SparkInternalRowTest.java | 60 +-
.../flink/table/store/spark/SparkReadITCase.java | 20 +-
.../flink/table/store/spark/SparkReadTestBase.java | 106 +-
.../store/spark/SparkSchemaEvolutionITCase.java | 44 +-
.../flink/table/store/spark/SparkTypeTest.java | 71 +-
.../flink/table/store/spark/SparkArrayData.java | 16 +-
.../table/store/spark/SparkDataSourceReader.java | 2 +-
.../table/store/spark/SparkFilterConverter.java | 6 +-
.../table/store/spark/SparkInputPartition.java | 18 +-
.../flink/table/store/spark/SparkInternalRow.java | 81 +-
.../flink/table/store/spark/SparkTypeUtils.java | 62 +-
.../table/store/spark/SimpleTableTestHelper.java | 6 +-
.../store/spark/SparkFilterConverterTest.java | 23 +-
.../table/store/spark/SparkInternalRowTest.java | 58 +-
.../flink/table/store/spark/SparkReadITCase.java | 37 +-
.../flink/table/store/spark/SparkTypeTest.java | 71 +-
tools/maven/suppressions.xml | 2 +
532 files changed, 17568 insertions(+), 10099 deletions(-)
diff --git a/flink-table-store-benchmark/flink-table-store-micro-benchmarks/src/main/java/org/apache/flink/table/store/benchmark/file/mergetree/MergeTreeBenchmark.java b/flink-table-store-benchmark/flink-table-store-micro-benchmarks/src/main/java/org/apache/flink/table/store/benchmark/file/mergetree/MergeTreeBenchmark.java
index 2e66fe82..5fb88612 100644
--- a/flink-table-store-benchmark/flink-table-store-micro-benchmarks/src/main/java/org/apache/flink/table/store/benchmark/file/mergetree/MergeTreeBenchmark.java
+++ b/flink-table-store-benchmark/flink-table-store-micro-benchmarks/src/main/java/org/apache/flink/table/store/benchmark/file/mergetree/MergeTreeBenchmark.java
@@ -20,11 +20,11 @@ package org.apache.flink.table.store.benchmark.file.mergetree;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
-import org.apache.flink.table.data.GenericRowData;
-import org.apache.flink.table.data.RowData;
import org.apache.flink.table.store.CoreOptions;
import org.apache.flink.table.store.benchmark.config.ConfigUtil;
import org.apache.flink.table.store.benchmark.config.FileBenchmarkOptions;
+import org.apache.flink.table.store.data.GenericRow;
+import org.apache.flink.table.store.data.InternalRow;
import org.apache.flink.table.store.file.KeyValue;
import org.apache.flink.table.store.file.compact.CompactResult;
import org.apache.flink.table.store.file.io.DataFileMeta;
@@ -50,10 +50,10 @@ import org.apache.flink.table.store.file.utils.RecordReaderIterator;
import org.apache.flink.table.store.file.utils.RecordWriter;
import org.apache.flink.table.store.format.FileFormat;
import org.apache.flink.table.store.types.DataField;
+import org.apache.flink.table.store.types.IntType;
+import org.apache.flink.table.store.types.RowKind;
+import org.apache.flink.table.store.types.RowType;
import org.apache.flink.table.store.utils.BinaryRowDataUtil;
-import org.apache.flink.table.types.logical.IntType;
-import org.apache.flink.table.types.logical.RowType;
-import org.apache.flink.types.RowKind;
import org.apache.commons.io.FileUtils;
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -101,7 +101,7 @@ public class MergeTreeBenchmark {
protected ExecutorService service;
protected File file;
- protected Comparator<RowData> comparator;
+ protected Comparator<InternalRow> comparator;
protected CoreOptions options;
protected KeyValueFileReaderFactory readerFactory;
private KeyValueFileReaderFactory compactReaderFactory;
@@ -134,8 +134,8 @@ public class MergeTreeBenchmark {
private RecordWriter<KeyValue> recreateMergeTree(
Configuration configuration, Path path, FileStorePathFactory pathFactory) {
options = new CoreOptions(configuration);
- RowType keyType = new RowType(singletonList(new RowType.RowField("k", new IntType())));
- RowType valueType = new RowType(singletonList(new RowType.RowField("v", new IntType())));
+ RowType keyType = new RowType(singletonList(new DataField(0, "k", new IntType())));
+ RowType valueType = new RowType(singletonList(new DataField(1, "v", new IntType())));
FileFormat flushingFormat = FileFormat.fromIdentifier(format, new Configuration());
KeyValueFileReaderFactory.Builder readerBuilder =
KeyValueFileReaderFactory.builder(
@@ -255,18 +255,18 @@ public class MergeTreeBenchmark {
/** Key value data to be written in {@link MergeTreeWriterBenchmark}. */
@State(Scope.Thread)
public static class KeyValueData {
- GenericRowData key;
+ GenericRow key;
RowKind kind;
- GenericRowData value;
+ GenericRow value;
@Setup(Level.Invocation)
public void kvSetup() {
- key = new GenericRowData(1);
+ key = new GenericRow(1);
key.setField(0, ThreadLocalRandom.current().nextInt());
kind = RowKind.INSERT;
- value = new GenericRowData(1);
+ value = new GenericRow(1);
value.setField(0, ThreadLocalRandom.current().nextInt());
}
diff --git a/flink-table-store-benchmark/flink-table-store-micro-benchmarks/src/main/java/org/apache/flink/table/store/benchmark/file/mergetree/MergeTreeReaderBenchmark.java b/flink-table-store-benchmark/flink-table-store-micro-benchmarks/src/main/java/org/apache/flink/table/store/benchmark/file/mergetree/MergeTreeReaderBenchmark.java
index 353745e4..1df91d28 100644
--- a/flink-table-store-benchmark/flink-table-store-micro-benchmarks/src/main/java/org/apache/flink/table/store/benchmark/file/mergetree/MergeTreeReaderBenchmark.java
+++ b/flink-table-store-benchmark/flink-table-store-micro-benchmarks/src/main/java/org/apache/flink/table/store/benchmark/file/mergetree/MergeTreeReaderBenchmark.java
@@ -18,7 +18,7 @@
package org.apache.flink.table.store.benchmark.file.mergetree;
-import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.store.data.GenericRow;
import org.apache.flink.table.store.file.KeyValue;
import org.apache.flink.table.store.file.mergetree.MergeTreeReaders;
import org.apache.flink.table.store.file.mergetree.compact.DeduplicateMergeFunction;
@@ -26,7 +26,7 @@ import org.apache.flink.table.store.file.mergetree.compact.IntervalPartition;
import org.apache.flink.table.store.file.utils.RecordReader;
import org.apache.flink.table.store.file.utils.RecordReaderIterator;
import org.apache.flink.table.store.file.utils.RecordWriter;
-import org.apache.flink.types.RowKind;
+import org.apache.flink.table.store.types.RowKind;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -64,11 +64,11 @@ public class MergeTreeReaderBenchmark extends MergeTreeBenchmark {
Random random = new Random();
for (int k = 0; k < batchCount; k++) {
for (int i = 0; i < countPerBatch; i++) {
- GenericRowData key = new GenericRowData(1);
+ GenericRow key = new GenericRow(1);
key.setField(0, random.nextInt());
key.setRowKind(RowKind.INSERT);
- GenericRowData value = new GenericRowData(1);
+ GenericRow value = new GenericRow(1);
value.setField(0, random.nextInt());
kv.replace(key, sequenceNumber++, RowKind.INSERT, value);
diff --git a/flink-table-store-codegen/src/main/java/org/apache/flink/table/store/codegen/CodeGeneratorImpl.java b/flink-table-store-codegen/src/main/java/org/apache/flink/table/store/codegen/CodeGeneratorImpl.java
index 6cacd170..7c2309c2 100644
--- a/flink-table-store-codegen/src/main/java/org/apache/flink/table/store/codegen/CodeGeneratorImpl.java
+++ b/flink-table-store-codegen/src/main/java/org/apache/flink/table/store/codegen/CodeGeneratorImpl.java
@@ -18,9 +18,9 @@
package org.apache.flink.table.store.codegen;
+import org.apache.flink.table.store.types.DataType;
+import org.apache.flink.table.store.types.RowType;
import org.apache.flink.table.store.utils.TypeUtils;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.RowType;
import java.util.List;
@@ -37,19 +37,19 @@ public class CodeGeneratorImpl implements CodeGenerator {
@Override
public GeneratedClass<NormalizedKeyComputer> generateNormalizedKeyComputer(
- List<LogicalType> fieldTypes, String name) {
+ List<DataType> fieldTypes, String name) {
return new SortCodeGenerator(
- RowType.of(fieldTypes.toArray(new LogicalType[0])),
+ RowType.builder().fields(fieldTypes).build(),
getAscendingSortSpec(fieldTypes.size()))
.generateNormalizedKeyComputer(name);
}
@Override
public GeneratedClass<RecordComparator> generateRecordComparator(
- List<LogicalType> fieldTypes, String name) {
+ List<DataType> fieldTypes, String name) {
return ComparatorCodeGenerator.gen(
name,
- RowType.of(fieldTypes.toArray(new LogicalType[0])),
+ RowType.builder().fields(fieldTypes).build(),
getAscendingSortSpec(fieldTypes.size()));
}
diff --git a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/CodeGeneratorContext.scala b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/CodeGeneratorContext.scala
index b0395111..e3d3bcc1 100644
--- a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/CodeGeneratorContext.scala
+++ b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/CodeGeneratorContext.scala
@@ -19,7 +19,7 @@ package org.apache.flink.table.store.codegen
import org.apache.flink.table.store.codegen.GenerateUtils.{newName, newNames}
import org.apache.flink.table.store.data.InternalSerializers
-import org.apache.flink.table.types.logical.LogicalType
+import org.apache.flink.table.store.types.DataType
import org.apache.flink.util.InstantiationUtil
import scala.collection.mutable
@@ -47,8 +47,8 @@ class CodeGeneratorContext {
// map of type serializer that will be added only once
// LogicalType -> reused_term
- private val reusableTypeSerializers: mutable.Map[LogicalType, String] =
- mutable.Map[LogicalType, String]()
+ private val reusableTypeSerializers: mutable.Map[DataType, String] =
+ mutable.Map[DataType, String]()
// local variable statements.
private val reusableLocalVariableStatements = mutable.LinkedHashSet[String]()
@@ -154,7 +154,7 @@ class CodeGeneratorContext {
* @return
* member variable term
*/
- def addReusableTypeSerializer(t: LogicalType): String = {
+ def addReusableTypeSerializer(t: DataType): String = {
// if type serializer has been used before, we can reuse the code that
// has already been generated
reusableTypeSerializers.get(t) match {
diff --git a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ComparatorCodeGenerator.scala b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ComparatorCodeGenerator.scala
index 093e5e17..cdc051e9 100644
--- a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ComparatorCodeGenerator.scala
+++ b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ComparatorCodeGenerator.scala
@@ -18,7 +18,7 @@
package org.apache.flink.table.store.codegen
import org.apache.flink.table.store.codegen.GenerateUtils.{newName, ROW_DATA}
-import org.apache.flink.table.types.logical.RowType
+import org.apache.flink.table.store.types.RowType
/** A code generator for generating [[RecordComparator]]. */
object ComparatorCodeGenerator {
diff --git a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ExprCodeGenerator.scala b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ExprCodeGenerator.scala
index db126401..048f1f93 100644
--- a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ExprCodeGenerator.scala
+++ b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ExprCodeGenerator.scala
@@ -17,12 +17,11 @@
*/
package org.apache.flink.table.store.codegen
-import org.apache.flink.table.data.RowData
-import org.apache.flink.table.data.binary.BinaryRowData
import org.apache.flink.table.store.codegen.GeneratedExpression.{NEVER_NULL, NO_CODE}
import org.apache.flink.table.store.codegen.GenerateUtils.{generateRecordStatement, rowSetField, DEFAULT_OUT_RECORD_TERM, DEFAULT_OUT_RECORD_WRITER_TERM}
+import org.apache.flink.table.store.data.{BinaryRow, InternalRow}
+import org.apache.flink.table.store.types.{RowType, TimestampType}
import org.apache.flink.table.store.utils.TypeUtils.isInteroperable
-import org.apache.flink.table.types.logical._
class ExprCodeGenerator(ctx: CodeGeneratorContext) {
@@ -49,7 +48,7 @@ class ExprCodeGenerator(ctx: CodeGeneratorContext) {
def generateResultExpression(
fieldExprs: Seq[GeneratedExpression],
returnType: RowType,
- returnTypeClazz: Class[_ <: RowData],
+ returnTypeClazz: Class[_ <: InternalRow],
outRow: String = DEFAULT_OUT_RECORD_TERM,
outRowWriter: Option[String] = Some(DEFAULT_OUT_RECORD_WRITER_TERM),
reusedOutRow: Boolean = true,
@@ -92,7 +91,7 @@ class ExprCodeGenerator(ctx: CodeGeneratorContext) {
fieldExprs: Seq[GeneratedExpression],
fieldExprIdxToOutputRowPosMap: Map[Int, Int],
returnType: RowType,
- returnTypeClazz: Class[_ <: RowData],
+ returnTypeClazz: Class[_ <: InternalRow],
outRow: String,
outRowWriter: Option[String],
reusedOutRow: Boolean,
@@ -111,13 +110,8 @@ class ExprCodeGenerator(ctx: CodeGeneratorContext) {
// type check
fieldExprs.zipWithIndex.foreach {
// timestamp type(Include TimeIndicator) and generic type can compatible with each other.
- case (fieldExpr, i)
- if fieldExpr.resultType.isInstanceOf[TypeInformationRawType[_]] ||
- fieldExpr.resultType.isInstanceOf[TimestampType] =>
- if (
- returnType.getTypeAt(i).getClass != fieldExpr.resultType.getClass
- && !returnType.getTypeAt(i).isInstanceOf[TypeInformationRawType[_]]
- ) {
+ case (fieldExpr, i) if fieldExpr.resultType.isInstanceOf[TimestampType] =>
+ if (returnType.getTypeAt(i).getClass != fieldExpr.resultType.getClass) {
throw new CodeGenException(
s"Incompatible types of expression and result type, Expression[$fieldExpr] type is " +
s"[${fieldExpr.resultType}], result type is [${returnType.getTypeAt(i)}]")
@@ -150,7 +144,7 @@ class ExprCodeGenerator(ctx: CodeGeneratorContext) {
NO_CODE
}
- val code = if (returnTypeClazz == classOf[BinaryRowData] && outRowWriter.isDefined) {
+ val code = if (returnTypeClazz == classOf[BinaryRow] && outRowWriter.isDefined) {
val writer = outRowWriter.get
val resetWriter = s"$writer.reset();"
val completeWriter: String = s"$writer.complete();"
diff --git a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/GenerateUtils.scala b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/GenerateUtils.scala
index 2280ee06..e08206ce 100644
--- a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/GenerateUtils.scala
+++ b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/GenerateUtils.scala
@@ -17,20 +17,15 @@
*/
package org.apache.flink.table.store.codegen
-import org.apache.flink.api.common.ExecutionConfig
-import org.apache.flink.api.common.typeinfo.{AtomicType => AtomicTypeInfo}
-import org.apache.flink.core.memory.MemorySegment
-import org.apache.flink.table.data._
-import org.apache.flink.table.data.binary.{BinaryRawValueData, BinaryRowData, BinaryStringData}
-import org.apache.flink.table.store.data.BinaryRowWriter
-import org.apache.flink.table.types.logical._
-import org.apache.flink.table.types.logical.LogicalTypeRoot._
-import org.apache.flink.table.types.logical.utils.LogicalTypeChecks.{getFieldCount, getFieldTypes, getPrecision, getScale}
-
-import java.lang.{Boolean => JBoolean, Byte => JByte, Double => JDouble, Float => JFloat, Integer => JInt, Long => JLong, Object => JObject, Short => JShort}
+import org.apache.flink.table.store.data._
+import org.apache.flink.table.store.memory.MemorySegment
+import org.apache.flink.table.store.types._
+import org.apache.flink.table.store.types.DataTypeChecks.{getFieldCount, getFieldTypes, getPrecision, getScale}
+import org.apache.flink.table.store.types.DataTypeRoot._
+
+import java.lang.{Boolean => JBoolean, Byte => JByte, Double => JDouble, Float => JFloat, Integer => JInt, Long => JLong, Short => JShort}
import java.util.concurrent.atomic.AtomicLong
-import scala.annotation.tailrec
import scala.collection.mutable
/** Utilities to generate code for general purpose. */
@@ -42,15 +37,13 @@ object GenerateUtils {
val DEFAULT_OUT_RECORD_WRITER_TERM = "outWriter"
- val BINARY_RAW_VALUE: String = className[BinaryRawValueData[_]]
-
- val ARRAY_DATA: String = className[ArrayData]
+ val ARRAY_DATA: String = className[InternalArray]
- val MAP_DATA: String = className[MapData]
+ val MAP_DATA: String = className[InternalMap]
- val ROW_DATA: String = className[RowData]
+ val ROW_DATA: String = className[InternalRow]
- val BINARY_STRING: String = className[BinaryStringData]
+ val BINARY_STRING: String = className[BinaryString]
val SEGMENT: String = className[MemorySegment]
@@ -66,29 +59,25 @@ object GenerateUtils {
}
/** Gets the default value for a primitive type, and null for generic types */
- @tailrec
- def primitiveDefaultValue(t: LogicalType): String = t.getTypeRoot match {
+ def primitiveDefaultValue(t: DataType): String = t.getTypeRoot match {
// ordered by type root definition
case CHAR | VARCHAR => s"$BINARY_STRING.EMPTY_UTF8"
case BOOLEAN => "false"
- case TINYINT | SMALLINT | INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => "-1"
- case BIGINT | INTERVAL_DAY_TIME => "-1L"
+ case TINYINT | SMALLINT | INTEGER | DATE | TIME_WITHOUT_TIME_ZONE => "-1"
+ case BIGINT => "-1L"
case FLOAT => "-1.0f"
case DOUBLE => "-1.0d"
- case DISTINCT_TYPE => primitiveDefaultValue(t.asInstanceOf[DistinctType].getSourceType)
-
case _ => "null"
}
- @tailrec
def generateFieldAccess(
ctx: CodeGeneratorContext,
- inputType: LogicalType,
+ inputType: DataType,
inputTerm: String,
index: Int): GeneratedExpression = inputType.getTypeRoot match {
// ordered by type root definition
- case ROW | STRUCTURED_TYPE =>
+ case ROW =>
val fieldType = getFieldTypes(inputType).get(index)
val resultTypeTerm = primitiveTypeTermForType(fieldType)
val defaultValue = primitiveDefaultValue(fieldType)
@@ -107,9 +96,6 @@ object GenerateUtils {
GeneratedExpression(fieldTerm, nullTerm, inputCode, fieldType)
- case DISTINCT_TYPE =>
- generateFieldAccess(ctx, inputType.asInstanceOf[DistinctType].getSourceType, inputTerm, index)
-
case _ =>
val fieldTypeTerm = boxedTypeTermForType(inputType)
val inputCode = s"($fieldTypeTerm) $inputTerm"
@@ -117,10 +103,9 @@ object GenerateUtils {
}
/** Generates code for comparing two fields. */
- @tailrec
def generateCompare(
ctx: CodeGeneratorContext,
- t: LogicalType,
+ t: DataType,
nullsIsLast: Boolean,
leftTerm: String,
rightTerm: String): String = t.getTypeRoot match {
@@ -133,11 +118,8 @@ object GenerateUtils {
val sortUtil =
classOf[org.apache.flink.table.store.utils.SortUtil].getCanonicalName
s"$sortUtil.compareBinary($leftTerm, $rightTerm)"
- case TINYINT | SMALLINT | INTEGER | BIGINT | FLOAT | DOUBLE | DATE | TIME_WITHOUT_TIME_ZONE |
- INTERVAL_YEAR_MONTH | INTERVAL_DAY_TIME =>
+ case TINYINT | SMALLINT | INTEGER | BIGINT | FLOAT | DOUBLE | DATE | TIME_WITHOUT_TIME_ZONE =>
s"($leftTerm > $rightTerm ? 1 : $leftTerm < $rightTerm ? -1 : 0)"
- case TIMESTAMP_WITH_TIME_ZONE =>
- throw new UnsupportedOperationException()
case ARRAY =>
val at = t.asInstanceOf[ArrayType]
val compareFunc = newName("compareArray")
@@ -177,7 +159,7 @@ object GenerateUtils {
"""
ctx.addReusableMember(funcCode)
s"$compareFunc($leftTerm, $rightTerm)"
- case ROW | STRUCTURED_TYPE =>
+ case ROW =>
val fieldCount = getFieldCount(t)
val comparisons =
generateRowCompare(ctx, t, getAscendingSortSpec((0 until fieldCount).toArray), "a", "b")
@@ -191,38 +173,7 @@ object GenerateUtils {
"""
ctx.addReusableMember(funcCode)
s"$compareFunc($leftTerm, $rightTerm)"
- case DISTINCT_TYPE =>
- generateCompare(
- ctx,
- t.asInstanceOf[DistinctType].getSourceType,
- nullsIsLast,
- leftTerm,
- rightTerm)
- case RAW =>
- t match {
- case rawType: RawType[_] =>
- val clazz = rawType.getOriginatingClass
- if (!classOf[Comparable[_]].isAssignableFrom(clazz)) {
- throw new CodeGenException(
- s"Raw type class '$clazz' must implement ${className[Comparable[_]]} to be used " +
- s"in a comparison of two '${rawType.asSummaryString()}' types.")
- }
- val serializer = rawType.getTypeSerializer
- val serializerTerm = ctx.addReusableObject(serializer, "serializer")
- s"((${className[Comparable[_]]}) $leftTerm.toObject($serializerTerm))" +
- s".compareTo($rightTerm.toObject($serializerTerm))"
-
- case rawType: TypeInformationRawType[_] =>
- val serializer = rawType.getTypeInformation.createSerializer(new ExecutionConfig)
- val ser = ctx.addReusableObject(serializer, "serializer")
- val comp = ctx.addReusableObject(
- rawType.getTypeInformation
- .asInstanceOf[AtomicTypeInfo[_]]
- .createComparator(true, new ExecutionConfig),
- "comparator")
- s"$comp.compare($leftTerm.toObject($ser), $rightTerm.toObject($ser))"
- }
- case NULL | SYMBOL | UNRESOLVED =>
+ case _ =>
throw new IllegalArgumentException("Illegal type: " + t)
}
@@ -344,7 +295,7 @@ object GenerateUtils {
/** Generates code for comparing row keys. */
def generateRowCompare(
ctx: CodeGeneratorContext,
- inputType: LogicalType,
+ inputType: DataType,
sortSpec: SortSpec,
leftTerm: String,
rightTerm: String): String = {
@@ -395,51 +346,42 @@ object GenerateUtils {
// works, but for boxed types we need this:
// Float a = 1.0f;
// Byte b = (byte)(float) a;
- @tailrec
- def primitiveTypeTermForType(t: LogicalType): String = t.getTypeRoot match {
+ def primitiveTypeTermForType(t: DataType): String = t.getTypeRoot match {
// ordered by type root definition
case BOOLEAN => "boolean"
case TINYINT => "byte"
case SMALLINT => "short"
- case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => "int"
- case BIGINT | INTERVAL_DAY_TIME => "long"
+ case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE => "int"
+ case BIGINT => "long"
case FLOAT => "float"
case DOUBLE => "double"
- case DISTINCT_TYPE => primitiveTypeTermForType(t.asInstanceOf[DistinctType].getSourceType)
case _ => boxedTypeTermForType(t)
}
- @tailrec
- def boxedTypeTermForType(t: LogicalType): String = t.getTypeRoot match {
+ def boxedTypeTermForType(t: DataType): String = t.getTypeRoot match {
// ordered by type root definition
case CHAR | VARCHAR => BINARY_STRING
case BOOLEAN => className[JBoolean]
case BINARY | VARBINARY => "byte[]"
- case DECIMAL => className[DecimalData]
+ case DECIMAL => className[Decimal]
case TINYINT => className[JByte]
case SMALLINT => className[JShort]
- case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH => className[JInt]
- case BIGINT | INTERVAL_DAY_TIME => className[JLong]
+ case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE => className[JInt]
+ case BIGINT => className[JLong]
case FLOAT => className[JFloat]
case DOUBLE => className[JDouble]
- case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE => className[TimestampData]
- case TIMESTAMP_WITH_TIME_ZONE =>
- throw new UnsupportedOperationException("Unsupported type: " + t)
- case ARRAY => className[ArrayData]
- case MULTISET | MAP => className[MapData]
- case ROW | STRUCTURED_TYPE => className[RowData]
- case DISTINCT_TYPE => boxedTypeTermForType(t.asInstanceOf[DistinctType].getSourceType)
- case NULL => className[JObject] // special case for untyped null literals
- case RAW => className[BinaryRawValueData[_]]
- case SYMBOL | UNRESOLVED =>
+ case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE => className[Timestamp]
+ case ARRAY => className[InternalArray]
+ case MULTISET | MAP => className[InternalMap]
+ case ROW => className[InternalRow]
+ case _ =>
throw new IllegalArgumentException("Illegal type: " + t)
}
- def rowFieldReadAccess(index: Int, rowTerm: String, fieldType: LogicalType): String =
+ def rowFieldReadAccess(index: Int, rowTerm: String, fieldType: DataType): String =
rowFieldReadAccess(index.toString, rowTerm, fieldType)
- @tailrec
- def rowFieldReadAccess(indexTerm: String, rowTerm: String, t: LogicalType): String =
+ def rowFieldReadAccess(indexTerm: String, rowTerm: String, t: DataType): String =
t.getTypeRoot match {
// ordered by type root definition
case CHAR | VARCHAR =>
@@ -454,9 +396,9 @@ object GenerateUtils {
s"$rowTerm.getByte($indexTerm)"
case SMALLINT =>
s"$rowTerm.getShort($indexTerm)"
- case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH =>
+ case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE =>
s"$rowTerm.getInt($indexTerm)"
- case BIGINT | INTERVAL_DAY_TIME =>
+ case BIGINT =>
s"$rowTerm.getLong($indexTerm)"
case FLOAT =>
s"$rowTerm.getFloat($indexTerm)"
@@ -464,19 +406,13 @@ object GenerateUtils {
s"$rowTerm.getDouble($indexTerm)"
case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE =>
s"$rowTerm.getTimestamp($indexTerm, ${getPrecision(t)})"
- case TIMESTAMP_WITH_TIME_ZONE =>
- throw new UnsupportedOperationException("Unsupported type: " + t)
case ARRAY =>
s"$rowTerm.getArray($indexTerm)"
case MULTISET | MAP =>
s"$rowTerm.getMap($indexTerm)"
- case ROW | STRUCTURED_TYPE =>
+ case ROW =>
s"$rowTerm.getRow($indexTerm, ${getFieldCount(t)})"
- case DISTINCT_TYPE =>
- rowFieldReadAccess(indexTerm, rowTerm, t.asInstanceOf[DistinctType].getSourceType)
- case RAW =>
- s"(($BINARY_RAW_VALUE) $rowTerm.getRawValue($indexTerm))"
- case NULL | SYMBOL | UNRESOLVED =>
+ case _ =>
throw new IllegalArgumentException("Illegal type: " + t)
}
@@ -497,7 +433,7 @@ object GenerateUtils {
*/
def generateInputFieldUnboxing(
ctx: CodeGeneratorContext,
- inputType: LogicalType,
+ inputType: DataType,
inputTerm: String,
inputUnboxingTerm: String): GeneratedExpression = {
@@ -558,7 +494,7 @@ object GenerateUtils {
def rowSetField(
ctx: CodeGeneratorContext,
- rowClass: Class[_ <: RowData],
+ rowClass: Class[_ <: InternalRow],
rowTerm: String,
indexTerm: String,
fieldExpr: GeneratedExpression,
@@ -567,7 +503,7 @@ object GenerateUtils {
val fieldType = fieldExpr.resultType
val fieldTerm = fieldExpr.resultTerm
- if (rowClass == classOf[BinaryRowData]) {
+ if (rowClass == classOf[BinaryRow]) {
binaryRowWriterTerm match {
case Some(writer) =>
// use writer to set field
@@ -604,7 +540,7 @@ object GenerateUtils {
index: Int,
fieldValTerm: String,
writerTerm: String,
- fieldType: LogicalType): String =
+ fieldType: DataType): String =
binaryWriterWriteField(
t => ctx.addReusableTypeSerializer(t),
index.toString,
@@ -617,7 +553,7 @@ object GenerateUtils {
indexTerm: String,
fieldValTerm: String,
writerTerm: String,
- t: LogicalType): String =
+ t: DataType): String =
binaryWriterWriteField(
t => ctx.addReusableTypeSerializer(t),
indexTerm,
@@ -625,13 +561,12 @@ object GenerateUtils {
writerTerm,
t)
- @tailrec
def binaryWriterWriteField(
- addSerializer: LogicalType => String,
+ addSerializer: DataType => String,
indexTerm: String,
fieldValTerm: String,
writerTerm: String,
- t: LogicalType): String = t.getTypeRoot match {
+ t: DataType): String = t.getTypeRoot match {
// ordered by type root definition
case CHAR | VARCHAR =>
s"$writerTerm.writeString($indexTerm, $fieldValTerm)"
@@ -645,9 +580,9 @@ object GenerateUtils {
s"$writerTerm.writeByte($indexTerm, $fieldValTerm)"
case SMALLINT =>
s"$writerTerm.writeShort($indexTerm, $fieldValTerm)"
- case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH =>
+ case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE =>
s"$writerTerm.writeInt($indexTerm, $fieldValTerm)"
- case BIGINT | INTERVAL_DAY_TIME =>
+ case BIGINT =>
s"$writerTerm.writeLong($indexTerm, $fieldValTerm)"
case FLOAT =>
s"$writerTerm.writeFloat($indexTerm, $fieldValTerm)"
@@ -655,55 +590,39 @@ object GenerateUtils {
s"$writerTerm.writeDouble($indexTerm, $fieldValTerm)"
case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE =>
s"$writerTerm.writeTimestamp($indexTerm, $fieldValTerm, ${getPrecision(t)})"
- case TIMESTAMP_WITH_TIME_ZONE =>
- throw new UnsupportedOperationException("Unsupported type: " + t)
case ARRAY =>
val ser = addSerializer(t)
s"$writerTerm.writeArray($indexTerm, $fieldValTerm, $ser)"
case MULTISET | MAP =>
val ser = addSerializer(t)
s"$writerTerm.writeMap($indexTerm, $fieldValTerm, $ser)"
- case ROW | STRUCTURED_TYPE =>
+ case ROW =>
val ser = addSerializer(t)
s"$writerTerm.writeRow($indexTerm, $fieldValTerm, $ser)"
- case DISTINCT_TYPE =>
- binaryWriterWriteField(
- addSerializer,
- indexTerm,
- fieldValTerm,
- writerTerm,
- t.asInstanceOf[DistinctType].getSourceType)
- case RAW =>
- val ser = addSerializer(t)
- s"$writerTerm.writeRawValue($indexTerm, $fieldValTerm, $ser)"
- case NULL | SYMBOL | UNRESOLVED =>
+ case _ =>
throw new IllegalArgumentException("Illegal type: " + t);
}
- def binaryWriterWriteNull(index: Int, writerTerm: String, t: LogicalType): String =
+ def binaryWriterWriteNull(index: Int, writerTerm: String, t: DataType): String =
binaryWriterWriteNull(index.toString, writerTerm, t)
- @tailrec
- def binaryWriterWriteNull(indexTerm: String, writerTerm: String, t: LogicalType): String =
+ def binaryWriterWriteNull(indexTerm: String, writerTerm: String, t: DataType): String =
t.getTypeRoot match {
// ordered by type root definition
- case DECIMAL if !DecimalData.isCompact(getPrecision(t)) =>
+ case DECIMAL if !Decimal.isCompact(getPrecision(t)) =>
s"$writerTerm.writeDecimal($indexTerm, null, ${getPrecision(t)})"
case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE
- if !TimestampData.isCompact(getPrecision(t)) =>
+ if !Timestamp.isCompact(getPrecision(t)) =>
s"$writerTerm.writeTimestamp($indexTerm, null, ${getPrecision(t)})"
- case DISTINCT_TYPE =>
- binaryWriterWriteNull(indexTerm, writerTerm, t.asInstanceOf[DistinctType].getSourceType)
case _ =>
s"$writerTerm.setNullAt($indexTerm)"
}
- @tailrec
def boxedWrapperRowFieldSetAccess(
rowTerm: String,
indexTerm: String,
fieldTerm: String,
- t: LogicalType): String = t.getTypeRoot match {
+ t: DataType): String = t.getTypeRoot match {
// ordered by type root definition
case BOOLEAN =>
s"$rowTerm.setBoolean($indexTerm, $fieldTerm)"
@@ -711,28 +630,21 @@ object GenerateUtils {
s"$rowTerm.setByte($indexTerm, $fieldTerm)"
case SMALLINT =>
s"$rowTerm.setShort($indexTerm, $fieldTerm)"
- case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH =>
+ case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE =>
s"$rowTerm.setInt($indexTerm, $fieldTerm)"
- case BIGINT | INTERVAL_DAY_TIME =>
+ case BIGINT =>
s"$rowTerm.setLong($indexTerm, $fieldTerm)"
case FLOAT =>
s"$rowTerm.setFloat($indexTerm, $fieldTerm)"
case DOUBLE =>
s"$rowTerm.setDouble($indexTerm, $fieldTerm)"
- case DISTINCT_TYPE =>
- boxedWrapperRowFieldSetAccess(
- rowTerm,
- indexTerm,
- fieldTerm,
- t.asInstanceOf[DistinctType].getSourceType)
case _ =>
s"$rowTerm.setNonPrimitiveValue($indexTerm, $fieldTerm)"
}
// -------------------------- BinaryArray Set Access -------------------------------
- @tailrec
- def binaryArraySetNull(index: Int, arrayTerm: String, t: LogicalType): String =
+ def binaryArraySetNull(index: Int, arrayTerm: String, t: DataType): String =
t.getTypeRoot match {
// ordered by type root definition
case BOOLEAN =>
@@ -741,14 +653,12 @@ object GenerateUtils {
s"$arrayTerm.setNullByte($index)"
case SMALLINT =>
s"$arrayTerm.setNullShort($index)"
- case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH =>
+ case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE =>
s"$arrayTerm.setNullInt($index)"
case FLOAT =>
s"$arrayTerm.setNullFloat($index)"
case DOUBLE =>
s"$arrayTerm.setNullDouble($index)"
- case DISTINCT_TYPE =>
- binaryArraySetNull(index, arrayTerm, t)
case _ =>
s"$arrayTerm.setNullLong($index)"
}
@@ -756,15 +666,14 @@ object GenerateUtils {
def binaryRowFieldSetAccess(
index: Int,
binaryRowTerm: String,
- fieldType: LogicalType,
+ fieldType: DataType,
fieldValTerm: String): String =
binaryRowFieldSetAccess(index.toString, binaryRowTerm, fieldType, fieldValTerm)
- @tailrec
def binaryRowFieldSetAccess(
index: String,
binaryRowTerm: String,
- t: LogicalType,
+ t: DataType,
fieldValTerm: String): String = t.getTypeRoot match {
// ordered by type root definition
case BOOLEAN =>
@@ -775,9 +684,9 @@ object GenerateUtils {
s"$binaryRowTerm.setByte($index, $fieldValTerm)"
case SMALLINT =>
s"$binaryRowTerm.setShort($index, $fieldValTerm)"
- case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE | INTERVAL_YEAR_MONTH =>
+ case INTEGER | DATE | TIME_WITHOUT_TIME_ZONE =>
s"$binaryRowTerm.setInt($index, $fieldValTerm)"
- case BIGINT | INTERVAL_DAY_TIME =>
+ case BIGINT =>
s"$binaryRowTerm.setLong($index, $fieldValTerm)"
case FLOAT =>
s"$binaryRowTerm.setFloat($index, $fieldValTerm)"
@@ -785,31 +694,22 @@ object GenerateUtils {
s"$binaryRowTerm.setDouble($index, $fieldValTerm)"
case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE =>
s"$binaryRowTerm.setTimestamp($index, $fieldValTerm, ${getPrecision(t)})"
- case DISTINCT_TYPE =>
- binaryRowFieldSetAccess(
- index,
- binaryRowTerm,
- t.asInstanceOf[DistinctType].getSourceType,
- fieldValTerm)
case _ =>
throw new CodeGenException(
- "Fail to find binary row field setter method of LogicalType " + t + ".")
+ "Fail to find binary row field setter method of DataType " + t + ".")
}
- def binaryRowSetNull(index: Int, rowTerm: String, t: LogicalType): String =
+ def binaryRowSetNull(index: Int, rowTerm: String, t: DataType): String =
binaryRowSetNull(index.toString, rowTerm, t)
- @tailrec
- def binaryRowSetNull(indexTerm: String, rowTerm: String, t: LogicalType): String =
+ def binaryRowSetNull(indexTerm: String, rowTerm: String, t: DataType): String =
t.getTypeRoot match {
// ordered by type root definition
- case DECIMAL if !DecimalData.isCompact(getPrecision(t)) =>
+ case DECIMAL if !Decimal.isCompact(getPrecision(t)) =>
s"$rowTerm.setDecimal($indexTerm, null, ${getPrecision(t)})"
case TIMESTAMP_WITHOUT_TIME_ZONE | TIMESTAMP_WITH_LOCAL_TIME_ZONE
- if !TimestampData.isCompact(getPrecision(t)) =>
+ if !Timestamp.isCompact(getPrecision(t)) =>
s"$rowTerm.setTimestamp($indexTerm, null, ${getPrecision(t)})"
- case DISTINCT_TYPE =>
- binaryRowSetNull(indexTerm, rowTerm, t.asInstanceOf[DistinctType].getSourceType)
case _ =>
s"$rowTerm.setNullAt($indexTerm)"
}
@@ -831,15 +731,14 @@ object GenerateUtils {
* @return
* the record initialization statement
*/
- @tailrec
def generateRecordStatement(
- t: LogicalType,
+ t: DataType,
clazz: Class[_],
recordTerm: String,
recordWriterTerm: Option[String] = None,
ctx: CodeGeneratorContext): String = t.getTypeRoot match {
// ordered by type root definition
- case ROW | STRUCTURED_TYPE if clazz == classOf[BinaryRowData] =>
+ case ROW if clazz == classOf[BinaryRow] =>
val writerTerm = recordWriterTerm.getOrElse(
throw new CodeGenException("No writer is specified when writing BinaryRowData record.")
)
@@ -851,13 +750,6 @@ object GenerateUtils {
|$recordTerm = new $typeTerm(${getFieldCount(t)});
|$writerTerm = new $binaryRowWriter($recordTerm);
|""".stripMargin.trim
- case DISTINCT_TYPE =>
- generateRecordStatement(
- t.asInstanceOf[DistinctType].getSourceType,
- clazz,
- recordTerm,
- recordWriterTerm,
- ctx)
case _ =>
val typeTerm = boxedTypeTermForType(t)
ctx.addReusableMember(s"$typeTerm $recordTerm = new $typeTerm();")
diff --git a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/GeneratedExpression.scala b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/GeneratedExpression.scala
index 6c25c728..38397c1a 100644
--- a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/GeneratedExpression.scala
+++ b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/GeneratedExpression.scala
@@ -17,7 +17,7 @@
*/
package org.apache.flink.table.store.codegen
-import org.apache.flink.table.types.logical.LogicalType
+import org.apache.flink.table.store.types.DataType
/**
* Describes a generated expression.
@@ -35,7 +35,7 @@ case class GeneratedExpression(
resultTerm: String,
nullTerm: String,
code: String,
- resultType: LogicalType
+ resultType: DataType
)
object GeneratedExpression {
diff --git a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ProjectionCodeGenerator.scala b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ProjectionCodeGenerator.scala
index 8a8b6548..05d33b87 100644
--- a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ProjectionCodeGenerator.scala
+++ b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/ProjectionCodeGenerator.scala
@@ -17,14 +17,14 @@
*/
package org.apache.flink.table.store.codegen
-import org.apache.flink.table.data.RowData
-import org.apache.flink.table.data.binary.BinaryRowData
import org.apache.flink.table.store.codegen.GeneratedExpression.{NEVER_NULL, NO_CODE}
import org.apache.flink.table.store.codegen.GenerateUtils.{generateRecordStatement, newName, DEFAULT_INPUT1_TERM, DEFAULT_OUT_RECORD_TERM, DEFAULT_OUT_RECORD_WRITER_TERM, ROW_DATA}
-import org.apache.flink.table.types.logical.RowType
+import org.apache.flink.table.store.data.{BinaryRow, InternalRow}
+import org.apache.flink.table.store.types.RowType
/**
- * CodeGenerator for projection, Take out some fields of [[RowData]] to generate a new [[RowData]].
+ * CodeGenerator for projection, Take out some fields of [[InternalRow]] to generate a new
+ * [[InternalRow]].
*/
object ProjectionCodeGenerator {
@@ -33,7 +33,7 @@ object ProjectionCodeGenerator {
inType: RowType,
outType: RowType,
inputMapping: Array[Int],
- outClass: Class[_ <: RowData] = classOf[BinaryRowData],
+ outClass: Class[_ <: InternalRow] = classOf[BinaryRow],
inputTerm: String = DEFAULT_INPUT1_TERM,
outRecordTerm: String = DEFAULT_OUT_RECORD_TERM,
outRecordWriterTerm: String = DEFAULT_OUT_RECORD_WRITER_TERM,
@@ -80,7 +80,7 @@ object ProjectionCodeGenerator {
inType: RowType,
outType: RowType,
inputMapping: Array[Int],
- outClass: Class[_ <: RowData] = classOf[BinaryRowData],
+ outClass: Class[_ <: InternalRow] = classOf[BinaryRow],
inputTerm: String = DEFAULT_INPUT1_TERM,
outRecordTerm: String = DEFAULT_OUT_RECORD_TERM,
outRecordWriterTerm: String = DEFAULT_OUT_RECORD_WRITER_TERM,
diff --git a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/SortCodeGenerator.scala b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/SortCodeGenerator.scala
index 137d632f..b5e322e5 100644
--- a/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/SortCodeGenerator.scala
+++ b/flink-table-store-codegen/src/main/scala/org.apache.flink.table.store.codegen/SortCodeGenerator.scala
@@ -17,12 +17,11 @@
*/
package org.apache.flink.table.store.codegen
-import org.apache.flink.table.data.{DecimalData, TimestampData}
-import org.apache.flink.table.data.binary.BinaryRowData
import org.apache.flink.table.store.codegen.GenerateUtils.{newName, ROW_DATA, SEGMENT}
+import org.apache.flink.table.store.data.{BinaryRow, Decimal, Timestamp}
+import org.apache.flink.table.store.types.{DataType, DecimalType, RowType, TimestampType}
+import org.apache.flink.table.store.types.DataTypeRoot._
import org.apache.flink.table.store.utils.{SortUtil, TypeUtils}
-import org.apache.flink.table.types.logical.{DecimalType, LogicalType, RowType, TimestampType}
-import org.apache.flink.table.types.logical.LogicalTypeRoot._
import scala.collection.mutable
@@ -276,7 +275,7 @@ class SortCodeGenerator(val input: RowType, val sortSpec: SortSpec) {
*/
val reverseKeys = new mutable.ArrayBuffer[String]
// If it is big endian, it would be better, no reverse.
- if (BinaryRowData.LITTLE_ENDIAN) {
+ if (BinaryRow.LITTLE_ENDIAN) {
var reverseOffset = 0
for (chunk <- chunks) {
val operator = BYTE_OPERATOR_MAPPING(chunk)
@@ -364,7 +363,7 @@ class SortCodeGenerator(val input: RowType, val sortSpec: SortSpec) {
swapKeys
}
- def getter(t: LogicalType, index: Int): String = {
+ def getter(t: DataType, index: Int): String = {
val prefix = prefixGetFromBinaryRow(t)
t match {
case dt: DecimalType =>
@@ -377,10 +376,10 @@ class SortCodeGenerator(val input: RowType, val sortSpec: SortSpec) {
}
/** For put${prefix}NormalizedKey() and compare$prefix() of [[SortUtil]]. */
- def prefixPutNormalizedKey(t: LogicalType): String = prefixGetFromBinaryRow(t)
+ def prefixPutNormalizedKey(t: DataType): String = prefixGetFromBinaryRow(t)
/** For get$prefix() of TypeGetterSetters. */
- def prefixGetFromBinaryRow(t: LogicalType): String = t.getTypeRoot match {
+ def prefixGetFromBinaryRow(t: DataType): String = t.getTypeRoot match {
case INTEGER => "Int"
case BIGINT => "Long"
case SMALLINT => "Short"
@@ -394,8 +393,6 @@ class SortCodeGenerator(val input: RowType, val sortSpec: SortSpec) {
case DATE => "Int"
case TIME_WITHOUT_TIME_ZONE => "Int"
case TIMESTAMP_WITHOUT_TIME_ZONE => "Timestamp"
- case INTERVAL_YEAR_MONTH => "Int"
- case INTERVAL_DAY_TIME => "Long"
case _ => null
}
@@ -409,19 +406,19 @@ class SortCodeGenerator(val input: RowType, val sortSpec: SortSpec) {
}
}
- def supportNormalizedKey(t: LogicalType): Boolean = {
+ def supportNormalizedKey(t: DataType): Boolean = {
t.getTypeRoot match {
case _ if TypeUtils.isPrimitive(t) => true
case VARCHAR | CHAR | VARBINARY | BINARY | DATE | TIME_WITHOUT_TIME_ZONE => true
case TIMESTAMP_WITHOUT_TIME_ZONE =>
// TODO: support normalize key for non-compact timestamp
- TimestampData.isCompact(t.asInstanceOf[TimestampType].getPrecision)
- case DECIMAL => DecimalData.isCompact(t.asInstanceOf[DecimalType].getPrecision)
+ Timestamp.isCompact(t.asInstanceOf[TimestampType].getPrecision)
+ case DECIMAL => Decimal.isCompact(t.asInstanceOf[DecimalType].getPrecision)
case _ => false
}
}
- def getNormalizeKeyLen(t: LogicalType): Int = {
+ def getNormalizeKeyLen(t: DataType): Int = {
t.getTypeRoot match {
case BOOLEAN => 1
case TINYINT => 1
@@ -431,13 +428,11 @@ class SortCodeGenerator(val input: RowType, val sortSpec: SortSpec) {
case DOUBLE => 8
case BIGINT => 8
case TIMESTAMP_WITHOUT_TIME_ZONE
- if TimestampData.isCompact(t.asInstanceOf[TimestampType].getPrecision) =>
+ if Timestamp.isCompact(t.asInstanceOf[TimestampType].getPrecision) =>
8
- case INTERVAL_YEAR_MONTH => 4
- case INTERVAL_DAY_TIME => 8
case DATE => 4
case TIME_WITHOUT_TIME_ZONE => 4
- case DECIMAL if DecimalData.isCompact(t.asInstanceOf[DecimalType].getPrecision) => 8
+ case DECIMAL if Decimal.isCompact(t.asInstanceOf[DecimalType].getPrecision) => 8
case VARCHAR | CHAR | VARBINARY | BINARY => Int.MaxValue
}
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/CodeGenerator.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/CodeGenerator.java
index 977419f0..83715d95 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/CodeGenerator.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/CodeGenerator.java
@@ -18,8 +18,8 @@
package org.apache.flink.table.store.codegen;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.store.types.DataType;
+import org.apache.flink.table.store.types.RowType;
import java.util.List;
@@ -37,7 +37,7 @@ public interface CodeGenerator {
* fields are compared in ascending order.
*/
GeneratedClass<NormalizedKeyComputer> generateNormalizedKeyComputer(
- List<LogicalType> fieldTypes, String name);
+ List<DataType> fieldTypes, String name);
/**
* Generate a {@link RecordComparator}.
@@ -47,5 +47,5 @@ public interface CodeGenerator {
* fields are compared in ascending order.
*/
GeneratedClass<RecordComparator> generateRecordComparator(
- List<LogicalType> fieldTypes, String name);
+ List<DataType> fieldTypes, String name);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/NormalizedKeyComputer.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/NormalizedKeyComputer.java
index 80bbc64c..07166953 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/NormalizedKeyComputer.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/NormalizedKeyComputer.java
@@ -18,8 +18,8 @@
package org.apache.flink.table.store.codegen;
-import org.apache.flink.core.memory.MemorySegment;
-import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.store.data.InternalRow;
+import org.apache.flink.table.store.memory.MemorySegment;
/**
* Normalized key computer for {@code SortBuffer}. For performance, subclasses are usually
@@ -28,7 +28,7 @@ import org.apache.flink.table.data.RowData;
public interface NormalizedKeyComputer {
/** Writes a normalized key for the given record into the target {@link MemorySegment}. */
- void putKey(RowData record, MemorySegment target, int offset);
+ void putKey(InternalRow record, MemorySegment target, int offset);
/** Compares two normalized keys in respective {@link MemorySegment}. */
int compareKey(MemorySegment segI, int offsetI, MemorySegment segJ, int offsetJ);
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/Projection.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/Projection.java
index 5b3abd7c..1f88d8e2 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/Projection.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/Projection.java
@@ -18,8 +18,8 @@
package org.apache.flink.table.store.codegen;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.binary.BinaryRowData;
+import org.apache.flink.table.store.data.BinaryRow;
+import org.apache.flink.table.store.data.InternalRow;
/**
* Interface for code generated projection, which will map a RowData to another BinaryRowData.
@@ -27,5 +27,5 @@ import org.apache.flink.table.data.binary.BinaryRowData;
*/
public interface Projection {
- BinaryRowData apply(RowData row);
+ BinaryRow apply(InternalRow row);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/RecordComparator.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/RecordComparator.java
index d2e7be8f..ad262187 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/RecordComparator.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/codegen/RecordComparator.java
@@ -18,7 +18,7 @@
package org.apache.flink.table.store.codegen;
-import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.store.data.InternalRow;
import java.io.Serializable;
import java.util.Comparator;
@@ -27,8 +27,8 @@ import java.util.Comparator;
* Record comparator for {@code BinaryInMemorySortBuffer}. For performance, subclasses are usually
* implemented through CodeGenerator. A new interface for helping JVM inline. Copied from Flink.
*/
-public interface RecordComparator extends Comparator<RowData>, Serializable {
+public interface RecordComparator extends Comparator<InternalRow>, Serializable {
@Override
- int compare(RowData o1, RowData o2);
+ int compare(InternalRow o1, InternalRow o2);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractBinaryWriter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractBinaryWriter.java
index c3ef534f..423d5dfc 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractBinaryWriter.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractBinaryWriter.java
@@ -19,26 +19,16 @@ package org.apache.flink.table.store.data;
import org.apache.flink.annotation.Internal;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
-import org.apache.flink.core.memory.MemorySegment;
-import org.apache.flink.core.memory.MemorySegmentFactory;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.data.binary.BinaryArrayData;
-import org.apache.flink.table.data.binary.BinaryFormat;
-import org.apache.flink.table.data.binary.BinaryMapData;
-import org.apache.flink.table.data.binary.BinaryRowData;
-import org.apache.flink.table.data.binary.BinarySegmentUtils;
-import org.apache.flink.table.data.binary.BinaryStringData;
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import static org.apache.flink.table.store.data.BinarySection.MAX_FIX_PART_DATA_SIZE;
+
/**
* Use the special format to write data to a {@link MemorySegment} (its capacity grows
* automatically).
@@ -68,18 +58,18 @@ abstract class AbstractBinaryWriter implements BinaryWriter {
protected abstract void setNullBit(int ordinal);
- /** See {@link BinarySegmentUtils#readStringData(MemorySegment[], int, int, long)}. */
+ /** See {@link MemorySegmentUtils#readBinaryString(MemorySegment[], int, int, long)}. */
@Override
- public void writeString(int pos, StringData input) {
- BinaryStringData string = (BinaryStringData) input;
+ public void writeString(int pos, BinaryString input) {
+ BinaryString string = input;
if (string.getSegments() == null) {
String javaObject = string.toString();
writeBytes(pos, javaObject.getBytes(StandardCharsets.UTF_8));
} else {
int len = string.getSizeInBytes();
if (len <= 7) {
- byte[] bytes = BinarySegmentUtils.allocateReuseBytes(len);
- BinarySegmentUtils.copyToBytes(
+ byte[] bytes = MemorySegmentUtils.allocateReuseBytes(len);
+ MemorySegmentUtils.copyToBytes(
string.getSegments(), string.getOffset(), bytes, 0, len);
writeBytesToFixLenPart(segment, getFieldOffset(pos), bytes, len);
} else {
@@ -90,7 +80,7 @@ abstract class AbstractBinaryWriter implements BinaryWriter {
private void writeBytes(int pos, byte[] bytes) {
int len = bytes.length;
- if (len <= BinaryFormat.MAX_FIX_PART_DATA_SIZE) {
+ if (len <= MAX_FIX_PART_DATA_SIZE) {
writeBytesToFixLenPart(segment, getFieldOffset(pos), bytes, len);
} else {
writeBytesToVarLenPart(pos, bytes, len);
@@ -98,27 +88,27 @@ abstract class AbstractBinaryWriter implements BinaryWriter {
}
@Override
- public void writeArray(int pos, ArrayData input, ArrayDataSerializer serializer) {
- BinaryArrayData binary = serializer.toBinaryArray(input);
+ public void writeArray(int pos, InternalArray input, ArrayDataSerializer serializer) {
+ BinaryArray binary = serializer.toBinaryArray(input);
writeSegmentsToVarLenPart(
pos, binary.getSegments(), binary.getOffset(), binary.getSizeInBytes());
}
@Override
- public void writeMap(int pos, MapData input, MapDataSerializer serializer) {
- BinaryMapData binary = serializer.toBinaryMap(input);
+ public void writeMap(int pos, InternalMap input, MapDataSerializer serializer) {
+ BinaryMap binary = serializer.toBinaryMap(input);
writeSegmentsToVarLenPart(
pos, binary.getSegments(), binary.getOffset(), binary.getSizeInBytes());
}
@Override
- public void writeRow(int pos, RowData input, RowDataSerializer serializer) {
- if (input instanceof BinaryFormat) {
- BinaryFormat row = (BinaryFormat) input;
+ public void writeRow(int pos, InternalRow input, RowDataSerializer serializer) {
+ if (input instanceof BinarySection) {
+ BinarySection row = (BinarySection) input;
writeSegmentsToVarLenPart(
pos, row.getSegments(), row.getOffset(), row.getSizeInBytes());
} else {
- BinaryRowData row = serializer.toBinaryRow(input);
+ BinaryRow row = serializer.toBinaryRow(input);
writeSegmentsToVarLenPart(
pos, row.getSegments(), row.getOffset(), row.getSizeInBytes());
}
@@ -127,7 +117,7 @@ abstract class AbstractBinaryWriter implements BinaryWriter {
@Override
public void writeBinary(int pos, byte[] bytes) {
int len = bytes.length;
- if (len <= BinaryFormat.MAX_FIX_PART_DATA_SIZE) {
+ if (len <= BinarySection.MAX_FIX_PART_DATA_SIZE) {
writeBytesToFixLenPart(segment, getFieldOffset(pos), bytes, len);
} else {
writeBytesToVarLenPart(pos, bytes, len);
@@ -135,10 +125,10 @@ abstract class AbstractBinaryWriter implements BinaryWriter {
}
@Override
- public void writeDecimal(int pos, DecimalData value, int precision) {
+ public void writeDecimal(int pos, Decimal value, int precision) {
assert value == null || (value.precision() == precision);
- if (DecimalData.isCompact(precision)) {
+ if (Decimal.isCompact(precision)) {
assert value != null;
writeLong(pos, value.toUnscaledLong());
} else {
@@ -170,8 +160,8 @@ abstract class AbstractBinaryWriter implements BinaryWriter {
}
@Override
- public void writeTimestamp(int pos, TimestampData value, int precision) {
- if (TimestampData.isCompact(precision)) {
+ public void writeTimestamp(int pos, Timestamp value, int precision) {
+ if (Timestamp.isCompact(precision)) {
writeLong(pos, value.getMillisecond());
} else {
// store the nanoOfMillisecond in fixed-length part as offset and nanoOfMillisecond
@@ -274,7 +264,7 @@ abstract class AbstractBinaryWriter implements BinaryWriter {
if (newCapacity - minCapacity < 0) {
newCapacity = minCapacity;
}
- segment = MemorySegmentFactory.wrap(Arrays.copyOf(segment.getArray(), newCapacity));
+ segment = MemorySegment.wrap(Arrays.copyOf(segment.getArray(), newCapacity));
afterGrow();
}
@@ -291,7 +281,7 @@ abstract class AbstractBinaryWriter implements BinaryWriter {
MemorySegment segment, int fieldOffset, byte[] bytes, int len) {
long firstByte = len | 0x80; // first bit is 1, other bits is len
long sevenBytes = 0L; // real data
- if (BinaryRowData.LITTLE_ENDIAN) {
+ if (BinaryRow.LITTLE_ENDIAN) {
for (int i = 0; i < len; i++) {
sevenBytes |= ((0x00000000000000FFL & bytes[i]) << (i * 8L));
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractPagedInputView.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractPagedInputView.java
index 0539b0b0..4e38c8d2 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractPagedInputView.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractPagedInputView.java
@@ -19,7 +19,7 @@
package org.apache.flink.table.store.data;
import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegment;
import java.io.EOFException;
import java.io.IOException;
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractPagedOutputView.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractPagedOutputView.java
index 49cad0af..322781f0 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractPagedOutputView.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractPagedOutputView.java
@@ -20,8 +20,8 @@ package org.apache.flink.table.store.data;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.core.memory.MemorySegment;
-import org.apache.flink.core.memory.MemorySegmentWritable;
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentWritable;
import java.io.IOException;
import java.io.UTFDataFormatException;
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractRowDataSerializer.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractRowDataSerializer.java
index be9184a4..48d6b269 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractRowDataSerializer.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/AbstractRowDataSerializer.java
@@ -18,21 +18,17 @@
package org.apache.flink.table.store.data;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.binary.BinaryRowData;
-
import java.io.IOException;
/** Row serializer, provided paged serialize paged method. */
-@Internal
-public abstract class AbstractRowDataSerializer<T extends RowData> extends PagedTypeSerializer<T> {
+public abstract class AbstractRowDataSerializer<T extends InternalRow>
+ extends PagedTypeSerializer<T> {
private static final long serialVersionUID = 1L;
/** Get the number of fields. */
public abstract int getArity();
- /** Convert a {@link RowData} to a {@link BinaryRowData}. */
- public abstract BinaryRowData toBinaryRow(T rowData) throws IOException;
+ /** Convert a {@link InternalRow} to a {@link BinaryRow}. */
+ public abstract BinaryRow toBinaryRow(T rowData) throws IOException;
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/ArrayDataSerializer.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/ArrayDataSerializer.java
index 7eac559c..6673e12f 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/ArrayDataSerializer.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/ArrayDataSerializer.java
@@ -27,39 +27,35 @@ import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.core.memory.MemorySegmentFactory;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.data.binary.BinaryArrayData;
-import org.apache.flink.table.data.binary.BinarySegmentUtils;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.utils.LogicalTypeUtils;
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+import org.apache.flink.table.store.types.DataType;
import org.apache.flink.util.InstantiationUtil;
import java.io.IOException;
import java.lang.reflect.Array;
import java.util.Arrays;
-/** Serializer for {@link ArrayData}. */
+/** Serializer for {@link InternalArray}. */
@Internal
-public class ArrayDataSerializer extends TypeSerializer<ArrayData> {
+public class ArrayDataSerializer extends TypeSerializer<InternalArray> {
private static final long serialVersionUID = 1L;
- private final LogicalType eleType;
+ private final DataType eleType;
private final TypeSerializer<Object> eleSer;
- private final ArrayData.ElementGetter elementGetter;
+ private final InternalArray.ElementGetter elementGetter;
- private transient BinaryArrayData reuseArray;
+ private transient BinaryArray reuseArray;
private transient BinaryArrayWriter reuseWriter;
- public ArrayDataSerializer(LogicalType eleType) {
+ public ArrayDataSerializer(DataType eleType) {
this(eleType, InternalSerializers.create(eleType));
}
- private ArrayDataSerializer(LogicalType eleType, TypeSerializer<Object> eleSer) {
+ private ArrayDataSerializer(DataType eleType, TypeSerializer<Object> eleSer) {
this.eleType = eleType;
this.eleSer = eleSer;
- this.elementGetter = ArrayData.createElementGetter(eleType);
+ this.elementGetter = InternalArray.createElementGetter(eleType);
}
@Override
@@ -68,65 +64,61 @@ public class ArrayDataSerializer extends TypeSerializer<ArrayData> {
}
@Override
- public TypeSerializer<ArrayData> duplicate() {
+ public TypeSerializer<InternalArray> duplicate() {
return new ArrayDataSerializer(eleType, eleSer.duplicate());
}
@Override
- public ArrayData createInstance() {
- return new BinaryArrayData();
+ public InternalArray createInstance() {
+ return new BinaryArray();
}
@Override
- public ArrayData copy(ArrayData from) {
- if (from instanceof GenericArrayData) {
- return copyGenericArray((GenericArrayData) from);
- } else if (from instanceof BinaryArrayData) {
- return ((BinaryArrayData) from).copy();
+ public InternalArray copy(InternalArray from) {
+ if (from instanceof GenericArray) {
+ return copyGenericArray((GenericArray) from);
+ } else if (from instanceof BinaryArray) {
+ return ((BinaryArray) from).copy();
} else {
return toBinaryArray(from);
}
}
@Override
- public ArrayData copy(ArrayData from, ArrayData reuse) {
+ public InternalArray copy(InternalArray from, InternalArray reuse) {
return copy(from);
}
- private GenericArrayData copyGenericArray(GenericArrayData array) {
+ private GenericArray copyGenericArray(GenericArray array) {
if (array.isPrimitiveArray()) {
switch (eleType.getTypeRoot()) {
case BOOLEAN:
- return new GenericArrayData(
- Arrays.copyOf(array.toBooleanArray(), array.size()));
+ return new GenericArray(Arrays.copyOf(array.toBooleanArray(), array.size()));
case TINYINT:
- return new GenericArrayData(Arrays.copyOf(array.toByteArray(), array.size()));
+ return new GenericArray(Arrays.copyOf(array.toByteArray(), array.size()));
case SMALLINT:
- return new GenericArrayData(Arrays.copyOf(array.toShortArray(), array.size()));
+ return new GenericArray(Arrays.copyOf(array.toShortArray(), array.size()));
case INTEGER:
- return new GenericArrayData(Arrays.copyOf(array.toIntArray(), array.size()));
+ return new GenericArray(Arrays.copyOf(array.toIntArray(), array.size()));
case BIGINT:
- return new GenericArrayData(Arrays.copyOf(array.toLongArray(), array.size()));
+ return new GenericArray(Arrays.copyOf(array.toLongArray(), array.size()));
case FLOAT:
- return new GenericArrayData(Arrays.copyOf(array.toFloatArray(), array.size()));
+ return new GenericArray(Arrays.copyOf(array.toFloatArray(), array.size()));
case DOUBLE:
- return new GenericArrayData(Arrays.copyOf(array.toDoubleArray(), array.size()));
+ return new GenericArray(Arrays.copyOf(array.toDoubleArray(), array.size()));
default:
throw new RuntimeException("Unknown type: " + eleType);
}
} else {
Object[] objectArray = array.toObjectArray();
Object[] newArray =
- (Object[])
- Array.newInstance(
- LogicalTypeUtils.toInternalConversionClass(eleType),
- array.size());
+ (Object[]) Array.newInstance(InternalRow.getDataClass(eleType), array.size());
for (int i = 0; i < array.size(); i++) {
if (objectArray[i] != null) {
newArray[i] = eleSer.copy(objectArray[i]);
}
}
- return new GenericArrayData(newArray);
+ return new GenericArray(newArray);
}
}
@@ -136,31 +128,31 @@ public class ArrayDataSerializer extends TypeSerializer<ArrayData> {
}
@Override
- public void serialize(ArrayData record, DataOutputView target) throws IOException {
- BinaryArrayData binaryArray = toBinaryArray(record);
+ public void serialize(InternalArray record, DataOutputView target) throws IOException {
+ BinaryArray binaryArray = toBinaryArray(record);
target.writeInt(binaryArray.getSizeInBytes());
- BinarySegmentUtils.copyToView(
+ MemorySegmentUtils.copyToView(
binaryArray.getSegments(),
binaryArray.getOffset(),
binaryArray.getSizeInBytes(),
target);
}
- public BinaryArrayData toBinaryArray(ArrayData from) {
- if (from instanceof BinaryArrayData) {
- return (BinaryArrayData) from;
+ public BinaryArray toBinaryArray(InternalArray from) {
+ if (from instanceof BinaryArray) {
+ return (BinaryArray) from;
}
int numElements = from.size();
if (reuseArray == null) {
- reuseArray = new BinaryArrayData();
+ reuseArray = new BinaryArray();
}
if (reuseWriter == null || reuseWriter.getNumElements() != numElements) {
reuseWriter =
new BinaryArrayWriter(
reuseArray,
numElements,
- BinaryArrayData.calculateFixLengthPartSize(eleType));
+ BinaryArray.calculateFixLengthPartSize(eleType));
} else {
reuseWriter.reset();
}
@@ -179,23 +171,22 @@ public class ArrayDataSerializer extends TypeSerializer<ArrayData> {
}
@Override
- public ArrayData deserialize(DataInputView source) throws IOException {
- return deserializeReuse(new BinaryArrayData(), source);
+ public InternalArray deserialize(DataInputView source) throws IOException {
+ return deserializeReuse(new BinaryArray(), source);
}
@Override
- public ArrayData deserialize(ArrayData reuse, DataInputView source) throws IOException {
+ public InternalArray deserialize(InternalArray reuse, DataInputView source) throws IOException {
return deserializeReuse(
- reuse instanceof BinaryArrayData ? (BinaryArrayData) reuse : new BinaryArrayData(),
- source);
+ reuse instanceof BinaryArray ? (BinaryArray) reuse : new BinaryArray(), source);
}
- private BinaryArrayData deserializeReuse(BinaryArrayData reuse, DataInputView source)
+ private BinaryArray deserializeReuse(BinaryArray reuse, DataInputView source)
throws IOException {
int length = source.readInt();
byte[] bytes = new byte[length];
source.readFully(bytes);
- reuse.pointTo(MemorySegmentFactory.wrap(bytes), 0, bytes.length);
+ reuse.pointTo(MemorySegment.wrap(bytes), 0, bytes.length);
return reuse;
}
@@ -231,16 +222,16 @@ public class ArrayDataSerializer extends TypeSerializer<ArrayData> {
}
@Override
- public TypeSerializerSnapshot<ArrayData> snapshotConfiguration() {
+ public TypeSerializerSnapshot<InternalArray> snapshotConfiguration() {
return new ArrayDataSerializerSnapshot(eleType, eleSer);
}
/** {@link TypeSerializerSnapshot} for {@link ArrayDataSerializer}. */
public static final class ArrayDataSerializerSnapshot
- implements TypeSerializerSnapshot<ArrayData> {
+ implements TypeSerializerSnapshot<InternalArray> {
private static final int CURRENT_VERSION = 3;
- private LogicalType previousType;
+ private DataType previousType;
private TypeSerializer previousEleSer;
@SuppressWarnings("unused")
@@ -248,7 +239,7 @@ public class ArrayDataSerializer extends TypeSerializer<ArrayData> {
// this constructor is used when restoring from a checkpoint/savepoint.
}
- ArrayDataSerializerSnapshot(LogicalType eleType, TypeSerializer eleSer) {
+ ArrayDataSerializerSnapshot(DataType eleType, TypeSerializer eleSer) {
this.previousType = eleType;
this.previousEleSer = eleSer;
}
@@ -280,13 +271,13 @@ public class ArrayDataSerializer extends TypeSerializer<ArrayData> {
}
@Override
- public TypeSerializer<ArrayData> restoreSerializer() {
+ public TypeSerializer<InternalArray> restoreSerializer() {
return new ArrayDataSerializer(previousType, previousEleSer);
}
@Override
- public TypeSerializerSchemaCompatibility<ArrayData> resolveSchemaCompatibility(
- TypeSerializer<ArrayData> newSerializer) {
+ public TypeSerializerSchemaCompatibility<InternalArray> resolveSchemaCompatibility(
+ TypeSerializer<InternalArray> newSerializer) {
if (!(newSerializer instanceof ArrayDataSerializer)) {
return TypeSerializerSchemaCompatibility.incompatible();
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryArray.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryArray.java
new file mode 100644
index 00000000..9f3ee57d
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryArray.java
@@ -0,0 +1,564 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+import org.apache.flink.table.store.types.DataType;
+
+import java.lang.reflect.Array;
+
+import static org.apache.flink.table.store.memory.MemorySegment.UNSAFE;
+
+/**
+ * A binary implementation of {@link InternalArray} which is backed by {@link MemorySegment}s.
+ *
+ * <p>For fields that hold fixed-length primitive types, such as long, double or int, they are
+ * stored compacted in bytes, just like the original java array.
+ *
+ * <p>The binary layout of {@link BinaryArray}:
+ *
+ * <pre>
+ * [size(int)] + [null bits(4-byte word boundaries)] + [values or offset&length] + [variable length part].
+ * </pre>
+ */
+public final class BinaryArray extends BinarySection implements InternalArray, DataSetters {
+
+ /** Offset for Arrays. */
+ private static final int BYTE_ARRAY_BASE_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
+
+ private static final int BOOLEAN_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(boolean[].class);
+ private static final int SHORT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(short[].class);
+ private static final int INT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(int[].class);
+ private static final int LONG_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(long[].class);
+ private static final int FLOAT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(float[].class);
+ private static final int DOUBLE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(double[].class);
+
+ public static int calculateHeaderInBytes(int numFields) {
+ return 4 + ((numFields + 31) / 32) * 4;
+ }
+
+ /**
+ * It store real value when type is primitive. It store the length and offset of variable-length
+ * part when type is string, map, etc.
+ */
+ public static int calculateFixLengthPartSize(DataType type) {
+ // ordered by type root definition
+ switch (type.getTypeRoot()) {
+ case BOOLEAN:
+ case TINYINT:
+ return 1;
+ case CHAR:
+ case VARCHAR:
+ case BINARY:
+ case VARBINARY:
+ case DECIMAL:
+ case BIGINT:
+ case DOUBLE:
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ case ARRAY:
+ case MULTISET:
+ case MAP:
+ case ROW:
+ // long and double are 8 bytes;
+ // otherwise it stores the length and offset of the variable-length part for types
+ // such as is string, map, etc.
+ return 8;
+ case SMALLINT:
+ return 2;
+ case INTEGER:
+ case FLOAT:
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ return 4;
+ default:
+ throw new IllegalArgumentException("Unsupported type: " + type);
+ }
+ }
+
+ // The number of elements in this array
+ private int size;
+
+ /** The position to start storing array elements. */
+ private int elementOffset;
+
+ public BinaryArray() {}
+
+ private void assertIndexIsValid(int ordinal) {
+ assert ordinal >= 0 : "ordinal (" + ordinal + ") should >= 0";
+ assert ordinal < size : "ordinal (" + ordinal + ") should < " + size;
+ }
+
+ private int getElementOffset(int ordinal, int elementSize) {
+ return elementOffset + ordinal * elementSize;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ @Override
+ public void pointTo(MemorySegment[] segments, int offset, int sizeInBytes) {
+ // Read the number of elements from the first 4 bytes.
+ final int size = MemorySegmentUtils.getInt(segments, offset);
+ assert size >= 0 : "size (" + size + ") should >= 0";
+
+ this.size = size;
+ this.segments = segments;
+ this.offset = offset;
+ this.sizeInBytes = sizeInBytes;
+ this.elementOffset = offset + calculateHeaderInBytes(this.size);
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.bitGet(segments, offset + 4, pos);
+ }
+
+ @Override
+ public void setNullAt(int pos) {
+ assertIndexIsValid(pos);
+ MemorySegmentUtils.bitSet(segments, offset + 4, pos);
+ }
+
+ public void setNotNullAt(int pos) {
+ assertIndexIsValid(pos);
+ MemorySegmentUtils.bitUnSet(segments, offset + 4, pos);
+ }
+
+ @Override
+ public long getLong(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getLong(segments, getElementOffset(pos, 8));
+ }
+
+ @Override
+ public void setLong(int pos, long value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setLong(segments, getElementOffset(pos, 8), value);
+ }
+
+ public void setNullLong(int pos) {
+ assertIndexIsValid(pos);
+ MemorySegmentUtils.bitSet(segments, offset + 4, pos);
+ MemorySegmentUtils.setLong(segments, getElementOffset(pos, 8), 0L);
+ }
+
+ @Override
+ public int getInt(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getInt(segments, getElementOffset(pos, 4));
+ }
+
+ @Override
+ public void setInt(int pos, int value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setInt(segments, getElementOffset(pos, 4), value);
+ }
+
+ public void setNullInt(int pos) {
+ assertIndexIsValid(pos);
+ MemorySegmentUtils.bitSet(segments, offset + 4, pos);
+ MemorySegmentUtils.setInt(segments, getElementOffset(pos, 4), 0);
+ }
+
+ @Override
+ public BinaryString getString(int pos) {
+ assertIndexIsValid(pos);
+ int fieldOffset = getElementOffset(pos, 8);
+ final long offsetAndSize = MemorySegmentUtils.getLong(segments, fieldOffset);
+ return MemorySegmentUtils.readBinaryString(segments, offset, fieldOffset, offsetAndSize);
+ }
+
+ @Override
+ public Decimal getDecimal(int pos, int precision, int scale) {
+ assertIndexIsValid(pos);
+ if (Decimal.isCompact(precision)) {
+ return Decimal.fromUnscaledLong(
+ MemorySegmentUtils.getLong(segments, getElementOffset(pos, 8)),
+ precision,
+ scale);
+ }
+
+ int fieldOffset = getElementOffset(pos, 8);
+ final long offsetAndSize = MemorySegmentUtils.getLong(segments, fieldOffset);
+ return MemorySegmentUtils.readDecimal(segments, offset, offsetAndSize, precision, scale);
+ }
+
+ @Override
+ public Timestamp getTimestamp(int pos, int precision) {
+ assertIndexIsValid(pos);
+
+ if (Timestamp.isCompact(precision)) {
+ return Timestamp.fromEpochMillis(
+ MemorySegmentUtils.getLong(segments, getElementOffset(pos, 8)));
+ }
+
+ int fieldOffset = getElementOffset(pos, 8);
+ final long offsetAndNanoOfMilli = MemorySegmentUtils.getLong(segments, fieldOffset);
+ return MemorySegmentUtils.readTimestampData(segments, offset, offsetAndNanoOfMilli);
+ }
+
+ @Override
+ public byte[] getBinary(int pos) {
+ assertIndexIsValid(pos);
+ int fieldOffset = getElementOffset(pos, 8);
+ final long offsetAndSize = MemorySegmentUtils.getLong(segments, fieldOffset);
+ return MemorySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndSize);
+ }
+
+ @Override
+ public InternalArray getArray(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.readArrayData(segments, offset, getLong(pos));
+ }
+
+ @Override
+ public InternalMap getMap(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.readMapData(segments, offset, getLong(pos));
+ }
+
+ @Override
+ public InternalRow getRow(int pos, int numFields) {
+ assertIndexIsValid(pos);
+ int fieldOffset = getElementOffset(pos, 8);
+ final long offsetAndSize = MemorySegmentUtils.getLong(segments, fieldOffset);
+ return MemorySegmentUtils.readRowData(segments, numFields, offset, offsetAndSize);
+ }
+
+ @Override
+ public boolean getBoolean(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getBoolean(segments, getElementOffset(pos, 1));
+ }
+
+ @Override
+ public void setBoolean(int pos, boolean value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setBoolean(segments, getElementOffset(pos, 1), value);
+ }
+
+ public void setNullBoolean(int pos) {
+ assertIndexIsValid(pos);
+ MemorySegmentUtils.bitSet(segments, offset + 4, pos);
+ MemorySegmentUtils.setBoolean(segments, getElementOffset(pos, 1), false);
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getByte(segments, getElementOffset(pos, 1));
+ }
+
+ @Override
+ public void setByte(int pos, byte value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setByte(segments, getElementOffset(pos, 1), value);
+ }
+
+ public void setNullByte(int pos) {
+ assertIndexIsValid(pos);
+ MemorySegmentUtils.bitSet(segments, offset + 4, pos);
+ MemorySegmentUtils.setByte(segments, getElementOffset(pos, 1), (byte) 0);
+ }
+
+ @Override
+ public short getShort(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getShort(segments, getElementOffset(pos, 2));
+ }
+
+ @Override
+ public void setShort(int pos, short value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setShort(segments, getElementOffset(pos, 2), value);
+ }
+
+ public void setNullShort(int pos) {
+ assertIndexIsValid(pos);
+ MemorySegmentUtils.bitSet(segments, offset + 4, pos);
+ MemorySegmentUtils.setShort(segments, getElementOffset(pos, 2), (short) 0);
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getFloat(segments, getElementOffset(pos, 4));
+ }
+
+ @Override
+ public void setFloat(int pos, float value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setFloat(segments, getElementOffset(pos, 4), value);
+ }
+
+ public void setNullFloat(int pos) {
+ assertIndexIsValid(pos);
+ MemorySegmentUtils.bitSet(segments, offset + 4, pos);
+ MemorySegmentUtils.setFloat(segments, getElementOffset(pos, 4), 0F);
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getDouble(segments, getElementOffset(pos, 8));
+ }
+
+ @Override
+ public void setDouble(int pos, double value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setDouble(segments, getElementOffset(pos, 8), value);
+ }
+
+ public void setNullDouble(int pos) {
+ assertIndexIsValid(pos);
+ MemorySegmentUtils.bitSet(segments, offset + 4, pos);
+ MemorySegmentUtils.setDouble(segments, getElementOffset(pos, 8), 0.0);
+ }
+
+ @Override
+ public void setDecimal(int pos, Decimal value, int precision) {
+ assertIndexIsValid(pos);
+
+ if (Decimal.isCompact(precision)) {
+ // compact format
+ setLong(pos, value.toUnscaledLong());
+ } else {
+ int fieldOffset = getElementOffset(pos, 8);
+ int cursor = (int) (MemorySegmentUtils.getLong(segments, fieldOffset) >>> 32);
+ assert cursor > 0 : "invalid cursor " + cursor;
+ // zero-out the bytes
+ MemorySegmentUtils.setLong(segments, offset + cursor, 0L);
+ MemorySegmentUtils.setLong(segments, offset + cursor + 8, 0L);
+
+ if (value == null) {
+ setNullAt(pos);
+ // keep the offset for future update
+ MemorySegmentUtils.setLong(segments, fieldOffset, ((long) cursor) << 32);
+ } else {
+
+ byte[] bytes = value.toUnscaledBytes();
+ assert (bytes.length <= 16);
+
+ // Write the bytes to the variable length portion.
+ MemorySegmentUtils.copyFromBytes(segments, offset + cursor, bytes, 0, bytes.length);
+ setLong(pos, ((long) cursor << 32) | ((long) bytes.length));
+ }
+ }
+ }
+
+ @Override
+ public void setTimestamp(int pos, Timestamp value, int precision) {
+ assertIndexIsValid(pos);
+
+ if (Timestamp.isCompact(precision)) {
+ setLong(pos, value.getMillisecond());
+ } else {
+ int fieldOffset = getElementOffset(pos, 8);
+ int cursor = (int) (MemorySegmentUtils.getLong(segments, fieldOffset) >>> 32);
+ assert cursor > 0 : "invalid cursor " + cursor;
+
+ if (value == null) {
+ setNullAt(pos);
+ // zero-out the bytes
+ MemorySegmentUtils.setLong(segments, offset + cursor, 0L);
+ // keep the offset for future update
+ MemorySegmentUtils.setLong(segments, fieldOffset, ((long) cursor) << 32);
+ } else {
+ // write millisecond to the variable length portion.
+ MemorySegmentUtils.setLong(segments, offset + cursor, value.getMillisecond());
+ // write nanoOfMillisecond to the fixed-length portion.
+ setLong(pos, ((long) cursor << 32) | (long) value.getNanoOfMillisecond());
+ }
+ }
+ }
+
+ public boolean anyNull() {
+ for (int i = offset + 4; i < elementOffset; i += 4) {
+ if (MemorySegmentUtils.getInt(segments, i) != 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void checkNoNull() {
+ if (anyNull()) {
+ throw new RuntimeException("Primitive array must not contain a null value.");
+ }
+ }
+
+ @Override
+ public boolean[] toBooleanArray() {
+ checkNoNull();
+ boolean[] values = new boolean[size];
+ MemorySegmentUtils.copyToUnsafe(
+ segments, elementOffset, values, BOOLEAN_ARRAY_OFFSET, size);
+ return values;
+ }
+
+ @Override
+ public byte[] toByteArray() {
+ checkNoNull();
+ byte[] values = new byte[size];
+ MemorySegmentUtils.copyToUnsafe(
+ segments, elementOffset, values, BYTE_ARRAY_BASE_OFFSET, size);
+ return values;
+ }
+
+ @Override
+ public short[] toShortArray() {
+ checkNoNull();
+ short[] values = new short[size];
+ MemorySegmentUtils.copyToUnsafe(
+ segments, elementOffset, values, SHORT_ARRAY_OFFSET, size * 2);
+ return values;
+ }
+
+ @Override
+ public int[] toIntArray() {
+ checkNoNull();
+ int[] values = new int[size];
+ MemorySegmentUtils.copyToUnsafe(
+ segments, elementOffset, values, INT_ARRAY_OFFSET, size * 4);
+ return values;
+ }
+
+ @Override
+ public long[] toLongArray() {
+ checkNoNull();
+ long[] values = new long[size];
+ MemorySegmentUtils.copyToUnsafe(
+ segments, elementOffset, values, LONG_ARRAY_OFFSET, size * 8);
+ return values;
+ }
+
+ @Override
+ public float[] toFloatArray() {
+ checkNoNull();
+ float[] values = new float[size];
+ MemorySegmentUtils.copyToUnsafe(
+ segments, elementOffset, values, FLOAT_ARRAY_OFFSET, size * 4);
+ return values;
+ }
+
+ @Override
+ public double[] toDoubleArray() {
+ checkNoNull();
+ double[] values = new double[size];
+ MemorySegmentUtils.copyToUnsafe(
+ segments, elementOffset, values, DOUBLE_ARRAY_OFFSET, size * 8);
+ return values;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> T[] toObjectArray(DataType elementType) {
+ Class<T> elementClass = (Class<T>) InternalRow.getDataClass(elementType);
+ InternalArray.ElementGetter elementGetter = InternalArray.createElementGetter(elementType);
+ T[] values = (T[]) Array.newInstance(elementClass, size);
+ for (int i = 0; i < size; i++) {
+ if (!isNullAt(i)) {
+ values[i] = (T) elementGetter.getElementOrNull(this, i);
+ }
+ }
+ return values;
+ }
+
+ public BinaryArray copy() {
+ return copy(new BinaryArray());
+ }
+
+ public BinaryArray copy(BinaryArray reuse) {
+ byte[] bytes = MemorySegmentUtils.copyToBytes(segments, offset, sizeInBytes);
+ reuse.pointTo(MemorySegment.wrap(bytes), 0, sizeInBytes);
+ return reuse;
+ }
+
+ @Override
+ public int hashCode() {
+ return MemorySegmentUtils.hashByWords(segments, offset, sizeInBytes);
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Construction Utilities
+ // ------------------------------------------------------------------------------------------
+
+ public static BinaryArray fromPrimitiveArray(boolean[] arr) {
+ return fromPrimitiveArray(arr, BOOLEAN_ARRAY_OFFSET, arr.length, 1);
+ }
+
+ public static BinaryArray fromPrimitiveArray(byte[] arr) {
+ return fromPrimitiveArray(arr, BYTE_ARRAY_BASE_OFFSET, arr.length, 1);
+ }
+
+ public static BinaryArray fromPrimitiveArray(short[] arr) {
+ return fromPrimitiveArray(arr, SHORT_ARRAY_OFFSET, arr.length, 2);
+ }
+
+ public static BinaryArray fromPrimitiveArray(int[] arr) {
+ return fromPrimitiveArray(arr, INT_ARRAY_OFFSET, arr.length, 4);
+ }
+
+ public static BinaryArray fromPrimitiveArray(long[] arr) {
+ return fromPrimitiveArray(arr, LONG_ARRAY_OFFSET, arr.length, 8);
+ }
+
+ public static BinaryArray fromPrimitiveArray(float[] arr) {
+ return fromPrimitiveArray(arr, FLOAT_ARRAY_OFFSET, arr.length, 4);
+ }
+
+ public static BinaryArray fromPrimitiveArray(double[] arr) {
+ return fromPrimitiveArray(arr, DOUBLE_ARRAY_OFFSET, arr.length, 8);
+ }
+
+ private static BinaryArray fromPrimitiveArray(
+ Object arr, int offset, int length, int elementSize) {
+ final long headerInBytes = calculateHeaderInBytes(length);
+ final long valueRegionInBytes = elementSize * length;
+
+ // must align by 8 bytes
+ long totalSizeInLongs = (headerInBytes + valueRegionInBytes + 7) / 8;
+ if (totalSizeInLongs > Integer.MAX_VALUE / 8) {
+ throw new UnsupportedOperationException(
+ "Cannot convert this array to unsafe format as " + "it's too big.");
+ }
+ long totalSize = totalSizeInLongs * 8;
+
+ final byte[] data = new byte[(int) totalSize];
+
+ UNSAFE.putInt(data, (long) BYTE_ARRAY_BASE_OFFSET, length);
+ UNSAFE.copyMemory(
+ arr, offset, data, BYTE_ARRAY_BASE_OFFSET + headerInBytes, valueRegionInBytes);
+
+ BinaryArray result = new BinaryArray();
+ result.pointTo(MemorySegment.wrap(data), 0, (int) totalSize);
+ return result;
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryArrayWriter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryArrayWriter.java
index 1c37ff8b..4ecfe261 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryArrayWriter.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryArrayWriter.java
@@ -18,31 +18,29 @@
package org.apache.flink.table.store.data;
import org.apache.flink.annotation.Internal;
-import org.apache.flink.core.memory.MemorySegmentFactory;
-import org.apache.flink.table.data.binary.BinaryArrayData;
-import org.apache.flink.table.data.binary.BinarySegmentUtils;
-import org.apache.flink.table.types.logical.DistinctType;
-import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+import org.apache.flink.table.store.types.DataType;
import java.io.Serializable;
-/** Writer for binary array. See {@link BinaryArrayData}. */
+/** Writer for binary array. See {@link BinaryArray}. */
@Internal
public final class BinaryArrayWriter extends AbstractBinaryWriter {
private final int nullBitsSizeInBytes;
- private final BinaryArrayData array;
+ private final BinaryArray array;
private final int numElements;
private final int fixedSize;
- public BinaryArrayWriter(BinaryArrayData array, int numElements, int elementSize) {
- this.nullBitsSizeInBytes = BinaryArrayData.calculateHeaderInBytes(numElements);
+ public BinaryArrayWriter(BinaryArray array, int numElements, int elementSize) {
+ this.nullBitsSizeInBytes = BinaryArray.calculateHeaderInBytes(numElements);
this.fixedSize =
roundNumberOfBytesToNearestWord(nullBitsSizeInBytes + elementSize * numElements);
this.cursor = fixedSize;
this.numElements = numElements;
- this.segment = MemorySegmentFactory.wrap(new byte[fixedSize]);
+ this.segment = MemorySegment.wrap(new byte[fixedSize]);
this.segment.putInt(0, numElements);
this.array = array;
}
@@ -59,7 +57,7 @@ public final class BinaryArrayWriter extends AbstractBinaryWriter {
@Override
public void setNullBit(int ordinal) {
- BinarySegmentUtils.bitSet(segment, 4, ordinal);
+ MemorySegmentUtils.bitSet(segment, 4, ordinal);
}
public void setNullBoolean(int ordinal) {
@@ -110,11 +108,11 @@ public final class BinaryArrayWriter extends AbstractBinaryWriter {
}
/**
- * @deprecated Use {@link #createNullSetter(LogicalType)} for avoiding logical types during
+ * @deprecated Use {@link #createNullSetter(DataType)} for avoiding logical types during
* runtime.
*/
@Deprecated
- public void setNullAt(int pos, LogicalType type) {
+ public void setNullAt(int pos, DataType type) {
switch (type.getTypeRoot()) {
case BOOLEAN:
setNullBoolean(pos);
@@ -128,13 +126,11 @@ public final class BinaryArrayWriter extends AbstractBinaryWriter {
case INTEGER:
case DATE:
case TIME_WITHOUT_TIME_ZONE:
- case INTERVAL_YEAR_MONTH:
setNullInt(pos);
break;
case BIGINT:
case TIMESTAMP_WITHOUT_TIME_ZONE:
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- case INTERVAL_DAY_TIME:
setNullLong(pos);
break;
case FLOAT:
@@ -227,7 +223,7 @@ public final class BinaryArrayWriter extends AbstractBinaryWriter {
*
* @param elementType the element type of the array
*/
- public static NullSetter createNullSetter(LogicalType elementType) {
+ public static NullSetter createNullSetter(DataType elementType) {
// ordered by type root definition
switch (elementType.getTypeRoot()) {
case CHAR:
@@ -238,13 +234,10 @@ public final class BinaryArrayWriter extends AbstractBinaryWriter {
case BIGINT:
case TIMESTAMP_WITHOUT_TIME_ZONE:
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- case INTERVAL_DAY_TIME:
case ARRAY:
case MULTISET:
case MAP:
case ROW:
- case STRUCTURED_TYPE:
- case RAW:
return BinaryArrayWriter::setNullLong;
case BOOLEAN:
return BinaryArrayWriter::setNullBoolean;
@@ -255,19 +248,11 @@ public final class BinaryArrayWriter extends AbstractBinaryWriter {
case INTEGER:
case DATE:
case TIME_WITHOUT_TIME_ZONE:
- case INTERVAL_YEAR_MONTH:
return BinaryArrayWriter::setNullInt;
case FLOAT:
return BinaryArrayWriter::setNullFloat;
case DOUBLE:
return BinaryArrayWriter::setNullDouble;
- case TIMESTAMP_WITH_TIME_ZONE:
- throw new UnsupportedOperationException();
- case DISTINCT_TYPE:
- return createNullSetter(((DistinctType) elementType).getSourceType());
- case NULL:
- case SYMBOL:
- case UNRESOLVED:
default:
throw new IllegalArgumentException();
}
@@ -276,7 +261,7 @@ public final class BinaryArrayWriter extends AbstractBinaryWriter {
/**
* Accessor for setting the elements of an array writer to {@code null} during runtime.
*
- * @see #createNullSetter(LogicalType)
+ * @see #createNullSetter(DataType)
*/
public interface NullSetter extends Serializable {
void setNull(BinaryArrayWriter writer, int pos);
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryMap.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryMap.java
new file mode 100644
index 00000000..0a0d77b3
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryMap.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+import org.apache.flink.table.store.types.DataType;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/**
+ * [4 byte(keyArray size in bytes)] + [Key BinaryArray] + [Value BinaryArray].
+ *
+ * <p>{@code BinaryMap} are influenced by Apache Spark UnsafeMapData.
+ */
+public final class BinaryMap extends BinarySection implements InternalMap {
+
+ private final BinaryArray keys;
+ private final BinaryArray values;
+
+ public BinaryMap() {
+ keys = new BinaryArray();
+ values = new BinaryArray();
+ }
+
+ public int size() {
+ return keys.size();
+ }
+
+ @Override
+ public void pointTo(MemorySegment[] segments, int offset, int sizeInBytes) {
+ // Read the numBytes of key array from the first 4 bytes.
+ final int keyArrayBytes = MemorySegmentUtils.getInt(segments, offset);
+ assert keyArrayBytes >= 0 : "keyArraySize (" + keyArrayBytes + ") should >= 0";
+ final int valueArrayBytes = sizeInBytes - keyArrayBytes - 4;
+ assert valueArrayBytes >= 0 : "valueArraySize (" + valueArrayBytes + ") should >= 0";
+
+ keys.pointTo(segments, offset + 4, keyArrayBytes);
+ values.pointTo(segments, offset + 4 + keyArrayBytes, valueArrayBytes);
+
+ assert keys.size() == values.size();
+
+ this.segments = segments;
+ this.offset = offset;
+ this.sizeInBytes = sizeInBytes;
+ }
+
+ public BinaryArray keyArray() {
+ return keys;
+ }
+
+ public BinaryArray valueArray() {
+ return values;
+ }
+
+ public Map<?, ?> toJavaMap(DataType keyType, DataType valueType) {
+ Object[] keyArray = keys.toObjectArray(keyType);
+ Object[] valueArray = values.toObjectArray(valueType);
+
+ Map<Object, Object> map = new HashMap<>();
+ for (int i = 0; i < keyArray.length; i++) {
+ map.put(keyArray[i], valueArray[i]);
+ }
+ return map;
+ }
+
+ public BinaryMap copy() {
+ return copy(new BinaryMap());
+ }
+
+ public BinaryMap copy(BinaryMap reuse) {
+ byte[] bytes = MemorySegmentUtils.copyToBytes(segments, offset, sizeInBytes);
+ reuse.pointTo(MemorySegment.wrap(bytes), 0, sizeInBytes);
+ return reuse;
+ }
+
+ @Override
+ public int hashCode() {
+ return MemorySegmentUtils.hashByWords(segments, offset, sizeInBytes);
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Construction Utilities
+ // ------------------------------------------------------------------------------------------
+
+ public static BinaryMap valueOf(BinaryArray key, BinaryArray value) {
+ checkArgument(key.segments.length == 1 && value.getSegments().length == 1);
+ byte[] bytes = new byte[4 + key.sizeInBytes + value.sizeInBytes];
+ MemorySegment segment = MemorySegment.wrap(bytes);
+ segment.putInt(0, key.sizeInBytes);
+ key.getSegments()[0].copyTo(key.getOffset(), segment, 4, key.sizeInBytes);
+ value.getSegments()[0].copyTo(
+ value.getOffset(), segment, 4 + key.sizeInBytes, value.sizeInBytes);
+ BinaryMap map = new BinaryMap();
+ map.pointTo(segment, 0, bytes.length);
+ return map;
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRow.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRow.java
new file mode 100644
index 00000000..43dfacb3
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRow.java
@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+import org.apache.flink.table.store.types.DataType;
+import org.apache.flink.table.store.types.DataTypeRoot;
+import org.apache.flink.table.store.types.DecimalType;
+import org.apache.flink.table.store.types.LocalZonedTimestampType;
+import org.apache.flink.table.store.types.RowKind;
+import org.apache.flink.table.store.types.TimestampType;
+
+import java.nio.ByteOrder;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/**
+ * An implementation of {@link InternalRow} which is backed by {@link MemorySegment} instead of
+ * Object. It can significantly reduce the serialization/deserialization of Java objects.
+ *
+ * <p>A Row has two part: Fixed-length part and variable-length part.
+ *
+ * <p>Fixed-length part contains 1 byte header and null bit set and field values. Null bit set is
+ * used for null tracking and is aligned to 8-byte word boundaries. `Field values` holds
+ * fixed-length primitive types and variable-length values which can be stored in 8 bytes inside. If
+ * it do not fit the variable-length field, then store the length and offset of variable-length
+ * part.
+ *
+ * <p>Fixed-length part will certainly fall into a MemorySegment, which will speed up the read and
+ * write of field. During the write phase, if the target memory segment has less space than fixed
+ * length part size, we will skip the space. So the number of fields in a single Row cannot exceed
+ * the capacity of a single MemorySegment, if there are too many fields, we suggest that user set a
+ * bigger pageSize of MemorySegment.
+ *
+ * <p>Variable-length part may fall into multiple MemorySegments.
+ */
+public final class BinaryRow extends BinarySection implements InternalRow, DataSetters {
+
+ public static final boolean LITTLE_ENDIAN =
+ (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN);
+ private static final long FIRST_BYTE_ZERO = LITTLE_ENDIAN ? ~0xFFL : ~(0xFFL << 56L);
+ public static final int HEADER_SIZE_IN_BITS = 8;
+
+ public static final BinaryRow EMPTY_ROW = new BinaryRow(0);
+
+ static {
+ int size = EMPTY_ROW.getFixedLengthPartSize();
+ byte[] bytes = new byte[size];
+ EMPTY_ROW.pointTo(MemorySegment.wrap(bytes), 0, size);
+ }
+
+ public static int calculateBitSetWidthInBytes(int arity) {
+ return ((arity + 63 + HEADER_SIZE_IN_BITS) / 64) * 8;
+ }
+
+ public static int calculateFixPartSizeInBytes(int arity) {
+ return calculateBitSetWidthInBytes(arity) + 8 * arity;
+ }
+
+ /**
+ * If it is a fixed-length field, we can call this BinaryRow's setXX method for in-place
+ * updates. If it is variable-length field, can't use this method, because the underlying data
+ * is stored continuously.
+ */
+ public static boolean isInFixedLengthPart(DataType type) {
+ switch (type.getTypeRoot()) {
+ case BOOLEAN:
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ case BIGINT:
+ case FLOAT:
+ case DOUBLE:
+ return true;
+ case DECIMAL:
+ return Decimal.isCompact(((DecimalType) type).getPrecision());
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ return Timestamp.isCompact(((TimestampType) type).getPrecision());
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return Timestamp.isCompact(((LocalZonedTimestampType) type).getPrecision());
+ default:
+ return false;
+ }
+ }
+
+ public static boolean isMutable(DataType type) {
+ return isInFixedLengthPart(type) || type.getTypeRoot() == DataTypeRoot.DECIMAL;
+ }
+
+ private final int arity;
+ private final int nullBitsSizeInBytes;
+
+ public BinaryRow(int arity) {
+ checkArgument(arity >= 0);
+ this.arity = arity;
+ this.nullBitsSizeInBytes = calculateBitSetWidthInBytes(arity);
+ }
+
+ private int getFieldOffset(int pos) {
+ return offset + nullBitsSizeInBytes + pos * 8;
+ }
+
+ private void assertIndexIsValid(int index) {
+ assert index >= 0 : "index (" + index + ") should >= 0";
+ assert index < arity : "index (" + index + ") should < " + arity;
+ }
+
+ public int getFixedLengthPartSize() {
+ return nullBitsSizeInBytes + 8 * arity;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return arity;
+ }
+
+ @Override
+ public RowKind getRowKind() {
+ byte kindValue = segments[0].get(offset);
+ return RowKind.fromByteValue(kindValue);
+ }
+
+ @Override
+ public void setRowKind(RowKind kind) {
+ segments[0].put(offset, kind.toByteValue());
+ }
+
+ public void setTotalSize(int sizeInBytes) {
+ this.sizeInBytes = sizeInBytes;
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.bitGet(segments[0], offset, pos + HEADER_SIZE_IN_BITS);
+ }
+
+ private void setNotNullAt(int i) {
+ assertIndexIsValid(i);
+ MemorySegmentUtils.bitUnSet(segments[0], offset, i + HEADER_SIZE_IN_BITS);
+ }
+
+ @Override
+ public void setNullAt(int i) {
+ assertIndexIsValid(i);
+ MemorySegmentUtils.bitSet(segments[0], offset, i + HEADER_SIZE_IN_BITS);
+ // We must set the fixed length part zero.
+ // 1.Only int/long/boolean...(Fix length type) will invoke this setNullAt.
+ // 2.Set to zero in order to equals and hash operation bytes calculation.
+ segments[0].putLong(getFieldOffset(i), 0);
+ }
+
+ @Override
+ public void setInt(int pos, int value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ segments[0].putInt(getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setLong(int pos, long value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ segments[0].putLong(getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setDouble(int pos, double value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ segments[0].putDouble(getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setDecimal(int pos, Decimal value, int precision) {
+ assertIndexIsValid(pos);
+
+ if (Decimal.isCompact(precision)) {
+ // compact format
+ setLong(pos, value.toUnscaledLong());
+ } else {
+ int fieldOffset = getFieldOffset(pos);
+ int cursor = (int) (segments[0].getLong(fieldOffset) >>> 32);
+ assert cursor > 0 : "invalid cursor " + cursor;
+ // zero-out the bytes
+ MemorySegmentUtils.setLong(segments, offset + cursor, 0L);
+ MemorySegmentUtils.setLong(segments, offset + cursor + 8, 0L);
+
+ if (value == null) {
+ setNullAt(pos);
+ // keep the offset for future update
+ segments[0].putLong(fieldOffset, ((long) cursor) << 32);
+ } else {
+
+ byte[] bytes = value.toUnscaledBytes();
+ assert bytes.length <= 16;
+
+ // Write the bytes to the variable length portion.
+ MemorySegmentUtils.copyFromBytes(segments, offset + cursor, bytes, 0, bytes.length);
+ setLong(pos, ((long) cursor << 32) | ((long) bytes.length));
+ }
+ }
+ }
+
+ @Override
+ public void setTimestamp(int pos, Timestamp value, int precision) {
+ assertIndexIsValid(pos);
+
+ if (Timestamp.isCompact(precision)) {
+ setLong(pos, value.getMillisecond());
+ } else {
+ int fieldOffset = getFieldOffset(pos);
+ int cursor = (int) (segments[0].getLong(fieldOffset) >>> 32);
+ assert cursor > 0 : "invalid cursor " + cursor;
+
+ if (value == null) {
+ setNullAt(pos);
+ // zero-out the bytes
+ MemorySegmentUtils.setLong(segments, offset + cursor, 0L);
+ // keep the offset for future update
+ segments[0].putLong(fieldOffset, ((long) cursor) << 32);
+ } else {
+ // write millisecond to the variable length portion.
+ MemorySegmentUtils.setLong(segments, offset + cursor, value.getMillisecond());
+ // write nanoOfMillisecond to the fixed-length portion.
+ setLong(pos, ((long) cursor << 32) | (long) value.getNanoOfMillisecond());
+ }
+ }
+ }
+
+ @Override
+ public void setBoolean(int pos, boolean value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ segments[0].putBoolean(getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setShort(int pos, short value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ segments[0].putShort(getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setByte(int pos, byte value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ segments[0].put(getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setFloat(int pos, float value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ segments[0].putFloat(getFieldOffset(pos), value);
+ }
+
+ @Override
+ public boolean getBoolean(int pos) {
+ assertIndexIsValid(pos);
+ return segments[0].getBoolean(getFieldOffset(pos));
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ assertIndexIsValid(pos);
+ return segments[0].get(getFieldOffset(pos));
+ }
+
+ @Override
+ public short getShort(int pos) {
+ assertIndexIsValid(pos);
+ return segments[0].getShort(getFieldOffset(pos));
+ }
+
+ @Override
+ public int getInt(int pos) {
+ assertIndexIsValid(pos);
+ return segments[0].getInt(getFieldOffset(pos));
+ }
+
+ @Override
+ public long getLong(int pos) {
+ assertIndexIsValid(pos);
+ return segments[0].getLong(getFieldOffset(pos));
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ assertIndexIsValid(pos);
+ return segments[0].getFloat(getFieldOffset(pos));
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ assertIndexIsValid(pos);
+ return segments[0].getDouble(getFieldOffset(pos));
+ }
+
+ @Override
+ public BinaryString getString(int pos) {
+ assertIndexIsValid(pos);
+ int fieldOffset = getFieldOffset(pos);
+ final long offsetAndLen = segments[0].getLong(fieldOffset);
+ return MemorySegmentUtils.readBinaryString(segments, offset, fieldOffset, offsetAndLen);
+ }
+
+ @Override
+ public Decimal getDecimal(int pos, int precision, int scale) {
+ assertIndexIsValid(pos);
+
+ if (Decimal.isCompact(precision)) {
+ return Decimal.fromUnscaledLong(
+ segments[0].getLong(getFieldOffset(pos)), precision, scale);
+ }
+
+ int fieldOffset = getFieldOffset(pos);
+ final long offsetAndSize = segments[0].getLong(fieldOffset);
+ return MemorySegmentUtils.readDecimal(segments, offset, offsetAndSize, precision, scale);
+ }
+
+ @Override
+ public Timestamp getTimestamp(int pos, int precision) {
+ assertIndexIsValid(pos);
+
+ if (Timestamp.isCompact(precision)) {
+ return Timestamp.fromEpochMillis(segments[0].getLong(getFieldOffset(pos)));
+ }
+
+ int fieldOffset = getFieldOffset(pos);
+ final long offsetAndNanoOfMilli = segments[0].getLong(fieldOffset);
+ return MemorySegmentUtils.readTimestampData(segments, offset, offsetAndNanoOfMilli);
+ }
+
+ @Override
+ public byte[] getBinary(int pos) {
+ assertIndexIsValid(pos);
+ int fieldOffset = getFieldOffset(pos);
+ final long offsetAndLen = segments[0].getLong(fieldOffset);
+ return MemorySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndLen);
+ }
+
+ @Override
+ public InternalArray getArray(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.readArrayData(segments, offset, getLong(pos));
+ }
+
+ @Override
+ public InternalMap getMap(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.readMapData(segments, offset, getLong(pos));
+ }
+
+ @Override
+ public InternalRow getRow(int pos, int numFields) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.readRowData(segments, numFields, offset, getLong(pos));
+ }
+
+ /** The bit is 1 when the field is null. Default is 0. */
+ public boolean anyNull() {
+ // Skip the header.
+ if ((segments[0].getLong(0) & FIRST_BYTE_ZERO) != 0) {
+ return true;
+ }
+ for (int i = 8; i < nullBitsSizeInBytes; i += 8) {
+ if (segments[0].getLong(i) != 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public boolean anyNull(int[] fields) {
+ for (int field : fields) {
+ if (isNullAt(field)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public BinaryRow copy() {
+ return copy(new BinaryRow(arity));
+ }
+
+ public BinaryRow copy(BinaryRow reuse) {
+ return copyInternal(reuse);
+ }
+
+ private BinaryRow copyInternal(BinaryRow reuse) {
+ byte[] bytes = MemorySegmentUtils.copyToBytes(segments, offset, sizeInBytes);
+ reuse.pointTo(MemorySegment.wrap(bytes), 0, sizeInBytes);
+ return reuse;
+ }
+
+ public void clear() {
+ segments = null;
+ offset = 0;
+ sizeInBytes = 0;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ // both BinaryRow and NestedRow have the same memory format
+ if (!(o instanceof BinaryRow || o instanceof NestedRow)) {
+ return false;
+ }
+ final BinarySection that = (BinarySection) o;
+ return sizeInBytes == that.sizeInBytes
+ && MemorySegmentUtils.equals(
+ segments, offset, that.segments, that.offset, sizeInBytes);
+ }
+
+ @Override
+ public int hashCode() {
+ return MemorySegmentUtils.hashByWords(segments, offset, sizeInBytes);
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRowDataSerializer.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRowSerializer.java
similarity index 74%
rename from flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRowDataSerializer.java
rename to flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRowSerializer.java
index 1f8892b1..24b7e333 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRowDataSerializer.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRowSerializer.java
@@ -18,25 +18,21 @@
package org.apache.flink.table.store.data;
-import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.core.memory.MemorySegment;
-import org.apache.flink.core.memory.MemorySegmentFactory;
-import org.apache.flink.core.memory.MemorySegmentWritable;
-import org.apache.flink.table.data.binary.BinaryRowData;
-import org.apache.flink.table.data.binary.BinarySegmentUtils;
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+import org.apache.flink.table.store.memory.MemorySegmentWritable;
import java.io.IOException;
import static org.apache.flink.util.Preconditions.checkArgument;
-/** Serializer for {@link BinaryRowData}. */
-@Internal
-public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRowData> {
+/** Serializer for {@link BinaryRow}. */
+public class BinaryRowSerializer extends AbstractRowDataSerializer<BinaryRow> {
private static final long serialVersionUID = 1L;
public static final int LENGTH_SIZE_IN_BYTES = 4;
@@ -44,9 +40,9 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
private final int numFields;
private final int fixedLengthPartSize;
- public BinaryRowDataSerializer(int numFields) {
+ public BinaryRowSerializer(int numFields) {
this.numFields = numFields;
- this.fixedLengthPartSize = BinaryRowData.calculateFixPartSizeInBytes(numFields);
+ this.fixedLengthPartSize = BinaryRow.calculateFixPartSizeInBytes(numFields);
}
@Override
@@ -55,22 +51,22 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
}
@Override
- public TypeSerializer<BinaryRowData> duplicate() {
- return new BinaryRowDataSerializer(numFields);
+ public TypeSerializer<BinaryRow> duplicate() {
+ return new BinaryRowSerializer(numFields);
}
@Override
- public BinaryRowData createInstance() {
- return new BinaryRowData(numFields);
+ public BinaryRow createInstance() {
+ return new BinaryRow(numFields);
}
@Override
- public BinaryRowData copy(BinaryRowData from) {
- return copy(from, new BinaryRowData(numFields));
+ public BinaryRow copy(BinaryRow from) {
+ return copy(from, new BinaryRow(numFields));
}
@Override
- public BinaryRowData copy(BinaryRowData from, BinaryRowData reuse) {
+ public BinaryRow copy(BinaryRow from, BinaryRow reuse) {
return from.copy(reuse);
}
@@ -80,36 +76,36 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
}
@Override
- public void serialize(BinaryRowData record, DataOutputView target) throws IOException {
+ public void serialize(BinaryRow record, DataOutputView target) throws IOException {
target.writeInt(record.getSizeInBytes());
if (target instanceof MemorySegmentWritable) {
serializeWithoutLength(record, (MemorySegmentWritable) target);
} else {
- BinarySegmentUtils.copyToView(
+ MemorySegmentUtils.copyToView(
record.getSegments(), record.getOffset(), record.getSizeInBytes(), target);
}
}
@Override
- public BinaryRowData deserialize(DataInputView source) throws IOException {
- BinaryRowData row = new BinaryRowData(numFields);
+ public BinaryRow deserialize(DataInputView source) throws IOException {
+ BinaryRow row = new BinaryRow(numFields);
int length = source.readInt();
byte[] bytes = new byte[length];
source.readFully(bytes);
- row.pointTo(MemorySegmentFactory.wrap(bytes), 0, length);
+ row.pointTo(MemorySegment.wrap(bytes), 0, length);
return row;
}
@Override
- public BinaryRowData deserialize(BinaryRowData reuse, DataInputView source) throws IOException {
+ public BinaryRow deserialize(BinaryRow reuse, DataInputView source) throws IOException {
MemorySegment[] segments = reuse.getSegments();
checkArgument(
segments == null || (segments.length == 1 && reuse.getOffset() == 0),
- "Reuse BinaryRowData should have no segments or only one segment and offset start at 0.");
+ "Reuse BinaryRow should have no segments or only one segment and offset start at 0.");
int length = source.readInt();
if (segments == null || segments[0].size() < length) {
- segments = new MemorySegment[] {MemorySegmentFactory.wrap(new byte[length])};
+ segments = new MemorySegment[] {MemorySegment.wrap(new byte[length])};
}
source.readFully(segments[0].getArray(), 0, length);
reuse.pointTo(segments, 0, length);
@@ -122,14 +118,14 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
}
@Override
- public BinaryRowData toBinaryRow(BinaryRowData rowData) throws IOException {
+ public BinaryRow toBinaryRow(BinaryRow rowData) throws IOException {
return rowData;
}
// ============================ Page related operations ===================================
@Override
- public int serializeToPages(BinaryRowData record, AbstractPagedOutputView headerLessView)
+ public int serializeToPages(BinaryRow record, AbstractPagedOutputView headerLessView)
throws IOException {
int skip = checkSkipWriteForFixLengthPart(headerLessView);
headerLessView.writeInt(record.getSizeInBytes());
@@ -137,7 +133,7 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
return skip;
}
- private static void serializeWithoutLength(BinaryRowData record, MemorySegmentWritable writable)
+ private static void serializeWithoutLength(BinaryRow record, MemorySegmentWritable writable)
throws IOException {
if (record.getSegments().length == 1) {
writable.write(record.getSegments()[0], record.getOffset(), record.getSizeInBytes());
@@ -146,7 +142,7 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
}
}
- public static void serializeWithoutLengthSlow(BinaryRowData record, MemorySegmentWritable out)
+ public static void serializeWithoutLengthSlow(BinaryRow record, MemorySegmentWritable out)
throws IOException {
int remainSize = record.getSizeInBytes();
int posInSegOfRecord = record.getOffset();
@@ -167,20 +163,20 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
}
@Override
- public BinaryRowData deserializeFromPages(AbstractPagedInputView headerLessView)
+ public BinaryRow deserializeFromPages(AbstractPagedInputView headerLessView)
throws IOException {
return deserializeFromPages(createInstance(), headerLessView);
}
@Override
- public BinaryRowData deserializeFromPages(
- BinaryRowData reuse, AbstractPagedInputView headerLessView) throws IOException {
+ public BinaryRow deserializeFromPages(BinaryRow reuse, AbstractPagedInputView headerLessView)
+ throws IOException {
checkSkipReadForFixLengthPart(headerLessView);
return deserialize(reuse, headerLessView);
}
@Override
- public BinaryRowData mapFromPages(BinaryRowData reuse, AbstractPagedInputView headerLessView)
+ public BinaryRow mapFromPages(BinaryRow reuse, AbstractPagedInputView headerLessView)
throws IOException {
checkSkipReadForFixLengthPart(headerLessView);
pointTo(headerLessView.readInt(), reuse, headerLessView);
@@ -211,10 +207,10 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
* Point row to memory segments with offset(in the AbstractPagedInputView) and length.
*
* @param length row length.
- * @param reuse reuse BinaryRowData object.
+ * @param reuse reuse BinaryRow object.
* @param headerLessView source memory segments container.
*/
- public void pointTo(int length, BinaryRowData reuse, AbstractPagedInputView headerLessView)
+ public void pointTo(int length, BinaryRow reuse, AbstractPagedInputView headerLessView)
throws IOException {
if (length < 0) {
throw new IOException(
@@ -240,7 +236,7 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
}
private void pointToMultiSegments(
- BinaryRowData reuse,
+ BinaryRow reuse,
AbstractPagedInputView source,
int sizeInBytes,
int remainLength,
@@ -270,7 +266,7 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
/**
* We need skip bytes to write when the remain bytes of current segment is not enough to write
- * binary row fixed part. See {@link BinaryRowData}.
+ * binary row fixed part. See {@link BinaryRow}.
*/
private int checkSkipWriteForFixLengthPart(AbstractPagedOutputView out) throws IOException {
// skip if there is no enough size.
@@ -284,7 +280,7 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
/**
* We need skip bytes to read when the remain bytes of current segment is not enough to write
- * binary row fixed part. See {@link BinaryRowData}.
+ * binary row fixed part. See {@link BinaryRow}.
*/
public void checkSkipReadForFixLengthPart(AbstractPagedInputView source) throws IOException {
// skip if there is no enough size.
@@ -313,8 +309,8 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
@Override
public boolean equals(Object obj) {
- return obj instanceof BinaryRowDataSerializer
- && numFields == ((BinaryRowDataSerializer) obj).numFields;
+ return obj instanceof BinaryRowSerializer
+ && numFields == ((BinaryRowSerializer) obj).numFields;
}
@Override
@@ -323,23 +319,23 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
}
@Override
- public TypeSerializerSnapshot<BinaryRowData> snapshotConfiguration() {
- return new BinaryRowDataSerializerSnapshot(numFields);
+ public TypeSerializerSnapshot<BinaryRow> snapshotConfiguration() {
+ return new BinaryRowSerializerSnapshot(numFields);
}
- /** {@link TypeSerializerSnapshot} for {@link BinaryRowDataSerializer}. */
- public static final class BinaryRowDataSerializerSnapshot
- implements TypeSerializerSnapshot<BinaryRowData> {
+ /** {@link TypeSerializerSnapshot} for {@link BinaryRowSerializer}. */
+ public static final class BinaryRowSerializerSnapshot
+ implements TypeSerializerSnapshot<BinaryRow> {
private static final int CURRENT_VERSION = 3;
private int previousNumFields;
@SuppressWarnings("unused")
- public BinaryRowDataSerializerSnapshot() {
+ public BinaryRowSerializerSnapshot() {
// this constructor is used when restoring from a checkpoint/savepoint.
}
- BinaryRowDataSerializerSnapshot(int numFields) {
+ BinaryRowSerializerSnapshot(int numFields) {
this.previousNumFields = numFields;
}
@@ -360,19 +356,18 @@ public class BinaryRowDataSerializer extends AbstractRowDataSerializer<BinaryRow
}
@Override
- public TypeSerializer<BinaryRowData> restoreSerializer() {
- return new BinaryRowDataSerializer(previousNumFields);
+ public TypeSerializer<BinaryRow> restoreSerializer() {
+ return new BinaryRowSerializer(previousNumFields);
}
@Override
- public TypeSerializerSchemaCompatibility<BinaryRowData> resolveSchemaCompatibility(
- TypeSerializer<BinaryRowData> newSerializer) {
- if (!(newSerializer instanceof BinaryRowDataSerializer)) {
+ public TypeSerializerSchemaCompatibility<BinaryRow> resolveSchemaCompatibility(
+ TypeSerializer<BinaryRow> newSerializer) {
+ if (!(newSerializer instanceof BinaryRowSerializer)) {
return TypeSerializerSchemaCompatibility.incompatible();
}
- BinaryRowDataSerializer newBinaryRowSerializer =
- (BinaryRowDataSerializer) newSerializer;
+ BinaryRowSerializer newBinaryRowSerializer = (BinaryRowSerializer) newSerializer;
if (previousNumFields != newBinaryRowSerializer.numFields) {
return TypeSerializerSchemaCompatibility.incompatible();
} else {
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRowWriter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRowWriter.java
index d2108375..114b0047 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRowWriter.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryRowWriter.java
@@ -18,29 +18,28 @@
package org.apache.flink.table.store.data;
import org.apache.flink.annotation.Internal;
-import org.apache.flink.core.memory.MemorySegmentFactory;
-import org.apache.flink.table.data.binary.BinaryRowData;
-import org.apache.flink.table.data.binary.BinarySegmentUtils;
-import org.apache.flink.types.RowKind;
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+import org.apache.flink.table.store.types.RowKind;
-/** Writer for {@link BinaryRowData}. */
+/** Writer for {@link BinaryRow}. */
@Internal
public final class BinaryRowWriter extends AbstractBinaryWriter {
private final int nullBitsSizeInBytes;
- private final BinaryRowData row;
+ private final BinaryRow row;
private final int fixedSize;
- public BinaryRowWriter(BinaryRowData row) {
+ public BinaryRowWriter(BinaryRow row) {
this(row, 0);
}
- public BinaryRowWriter(BinaryRowData row, int initialSize) {
- this.nullBitsSizeInBytes = BinaryRowData.calculateBitSetWidthInBytes(row.getArity());
+ public BinaryRowWriter(BinaryRow row, int initialSize) {
+ this.nullBitsSizeInBytes = BinaryRow.calculateBitSetWidthInBytes(row.getFieldCount());
this.fixedSize = row.getFixedLengthPartSize();
this.cursor = fixedSize;
- this.segment = MemorySegmentFactory.wrap(new byte[fixedSize + initialSize]);
+ this.segment = MemorySegment.wrap(new byte[fixedSize + initialSize]);
this.row = row;
this.row.pointTo(segment, 0, segment.size());
}
@@ -63,7 +62,7 @@ public final class BinaryRowWriter extends AbstractBinaryWriter {
@Override
public void setNullBit(int pos) {
- BinarySegmentUtils.bitSet(segment, 0, pos + BinaryRowData.HEADER_SIZE_IN_BITS);
+ MemorySegmentUtils.bitSet(segment, 0, pos + BinaryRow.HEADER_SIZE_IN_BITS);
}
public void writeRowKind(RowKind kind) {
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinarySection.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinarySection.java
new file mode 100644
index 00000000..b20889a0
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinarySection.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+import org.apache.flink.util.Preconditions;
+
+/** Describe a section of memory. */
+public abstract class BinarySection {
+
+ /**
+ * It decides whether to put data in FixLenPart or VarLenPart. See more in {@link BinaryRow}.
+ *
+ * <p>If len is less than 8, its binary format is: 1-bit mark(1) = 1, 7-bits len, and 7-bytes
+ * data. Data is stored in fix-length part.
+ *
+ * <p>If len is greater or equal to 8, its binary format is: 1-bit mark(1) = 0, 31-bits offset
+ * to the data, and 4-bytes length of data. Data is stored in variable-length part.
+ */
+ public static final int MAX_FIX_PART_DATA_SIZE = 7;
+
+ /**
+ * To get the mark in highest bit of long. Form: 10000000 00000000 ... (8 bytes)
+ *
+ * <p>This is used to decide whether the data is stored in fixed-length part or variable-length
+ * part. see {@link #MAX_FIX_PART_DATA_SIZE} for more information.
+ */
+ public static final long HIGHEST_FIRST_BIT = 0x80L << 56;
+
+ /**
+ * To get the 7 bits length in second bit to eighth bit out of a long. Form: 01111111 00000000
+ * ... (8 bytes)
+ *
+ * <p>This is used to get the length of the data which is stored in this long. see {@link
+ * #MAX_FIX_PART_DATA_SIZE} for more information.
+ */
+ public static final long HIGHEST_SECOND_TO_EIGHTH_BIT = 0x7FL << 56;
+
+ protected MemorySegment[] segments;
+ protected int offset;
+ protected int sizeInBytes;
+
+ public BinarySection() {}
+
+ public BinarySection(MemorySegment[] segments, int offset, int sizeInBytes) {
+ Preconditions.checkArgument(segments != null);
+ this.segments = segments;
+ this.offset = offset;
+ this.sizeInBytes = sizeInBytes;
+ }
+
+ public final void pointTo(MemorySegment segment, int offset, int sizeInBytes) {
+ pointTo(new MemorySegment[] {segment}, offset, sizeInBytes);
+ }
+
+ public void pointTo(MemorySegment[] segments, int offset, int sizeInBytes) {
+ Preconditions.checkArgument(segments != null);
+ this.segments = segments;
+ this.offset = offset;
+ this.sizeInBytes = sizeInBytes;
+ }
+
+ public MemorySegment[] getSegments() {
+ return segments;
+ }
+
+ public int getOffset() {
+ return offset;
+ }
+
+ public int getSizeInBytes() {
+ return sizeInBytes;
+ }
+
+ public byte[] toBytes() {
+ return MemorySegmentUtils.getBytes(segments, offset, sizeInBytes);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ final BinarySection that = (BinarySection) o;
+ return sizeInBytes == that.sizeInBytes
+ && MemorySegmentUtils.equals(
+ segments, offset, that.segments, that.offset, sizeInBytes);
+ }
+
+ @Override
+ public int hashCode() {
+ return MemorySegmentUtils.hash(segments, offset, sizeInBytes);
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryString.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryString.java
new file mode 100644
index 00000000..bc41e971
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryString.java
@@ -0,0 +1,1007 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+import static org.apache.flink.table.store.memory.MemorySegmentUtils.allocateReuseBytes;
+import static org.apache.flink.table.store.memory.MemorySegmentUtils.allocateReuseChars;
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/** A string which is backed by {@link MemorySegment}s. */
+public final class BinaryString extends BinarySection implements Comparable<BinaryString> {
+
+ public static final BinaryString EMPTY_UTF8 = BinaryString.fromBytes(encodeUTF8(""));
+
+ public BinaryString(MemorySegment[] segments, int offset, int sizeInBytes) {
+ super(segments, offset, sizeInBytes);
+ this.segments = segments;
+ this.offset = offset;
+ this.sizeInBytes = sizeInBytes;
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Construction Utilities
+ // ------------------------------------------------------------------------------------------
+
+ public static BinaryString fromAddress(MemorySegment[] segments, int offset, int numBytes) {
+ return new BinaryString(segments, offset, numBytes);
+ }
+
+ @Nullable
+ public static BinaryString fromString(String str) {
+ if (str == null) {
+ return null;
+ }
+ return fromBytes(encodeUTF8(str));
+ }
+
+ /** Creates a {@link BinaryString} instance from the given UTF-8 bytes. */
+ public static BinaryString fromBytes(byte[] bytes) {
+ return fromBytes(bytes, 0, bytes.length);
+ }
+
+ /**
+ * Creates a {@link BinaryString} instance from the given UTF-8 bytes with offset and number of
+ * bytes.
+ */
+ public static BinaryString fromBytes(byte[] bytes, int offset, int numBytes) {
+ return new BinaryString(new MemorySegment[] {MemorySegment.wrap(bytes)}, offset, numBytes);
+ }
+
+ /** Creates a {@link BinaryString} instance that contains `length` spaces. */
+ public static BinaryString blankString(int length) {
+ byte[] spaces = new byte[length];
+ Arrays.fill(spaces, (byte) ' ');
+ return fromBytes(spaces);
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Public Interfaces
+ // ------------------------------------------------------------------------------------------
+
+ @Override
+ public String toString() {
+ byte[] bytes = allocateReuseBytes(sizeInBytes);
+ MemorySegmentUtils.copyToBytes(segments, offset, bytes, 0, sizeInBytes);
+ return decodeUTF8(bytes, 0, sizeInBytes);
+ }
+
+ /**
+ * Compares two strings lexicographically. Since UTF-8 uses groups of six bits, it is sometimes
+ * useful to use octal notation which uses 3-bit groups. With a calculator which can convert
+ * between hexadecimal and octal it can be easier to manually create or interpret UTF-8 compared
+ * with using binary. So we just compare the binary.
+ */
+ @Override
+ public int compareTo(@Nonnull BinaryString other) {
+ if (segments.length == 1 && other.segments.length == 1) {
+ int len = Math.min(sizeInBytes, other.sizeInBytes);
+ MemorySegment seg1 = segments[0];
+ MemorySegment seg2 = other.segments[0];
+
+ for (int i = 0; i < len; i++) {
+ int res = (seg1.get(offset + i) & 0xFF) - (seg2.get(other.offset + i) & 0xFF);
+ if (res != 0) {
+ return res;
+ }
+ }
+ return sizeInBytes - other.sizeInBytes;
+ }
+
+ // if there are multi segments.
+ return compareMultiSegments(other);
+ }
+
+ /** Find the boundaries of segments, and then compare MemorySegment. */
+ private int compareMultiSegments(BinaryString other) {
+
+ if (sizeInBytes == 0 || other.sizeInBytes == 0) {
+ return sizeInBytes - other.sizeInBytes;
+ }
+
+ int len = Math.min(sizeInBytes, other.sizeInBytes);
+
+ MemorySegment seg1 = segments[0];
+ MemorySegment seg2 = other.segments[0];
+
+ int segmentSize = segments[0].size();
+ int otherSegmentSize = other.segments[0].size();
+
+ int sizeOfFirst1 = segmentSize - offset;
+ int sizeOfFirst2 = otherSegmentSize - other.offset;
+
+ int varSegIndex1 = 1;
+ int varSegIndex2 = 1;
+
+ // find the first segment of this string.
+ while (sizeOfFirst1 <= 0) {
+ sizeOfFirst1 += segmentSize;
+ seg1 = segments[varSegIndex1++];
+ }
+
+ while (sizeOfFirst2 <= 0) {
+ sizeOfFirst2 += otherSegmentSize;
+ seg2 = other.segments[varSegIndex2++];
+ }
+
+ int offset1 = segmentSize - sizeOfFirst1;
+ int offset2 = otherSegmentSize - sizeOfFirst2;
+
+ int needCompare = Math.min(Math.min(sizeOfFirst1, sizeOfFirst2), len);
+
+ while (needCompare > 0) {
+ // compare in one segment.
+ for (int i = 0; i < needCompare; i++) {
+ int res = (seg1.get(offset1 + i) & 0xFF) - (seg2.get(offset2 + i) & 0xFF);
+ if (res != 0) {
+ return res;
+ }
+ }
+ if (needCompare == len) {
+ break;
+ }
+ len -= needCompare;
+ // next segment
+ if (sizeOfFirst1 < sizeOfFirst2) { // I am smaller
+ seg1 = segments[varSegIndex1++];
+ offset1 = 0;
+ offset2 += needCompare;
+ sizeOfFirst1 = segmentSize;
+ sizeOfFirst2 -= needCompare;
+ } else if (sizeOfFirst1 > sizeOfFirst2) { // other is smaller
+ seg2 = other.segments[varSegIndex2++];
+ offset2 = 0;
+ offset1 += needCompare;
+ sizeOfFirst2 = otherSegmentSize;
+ sizeOfFirst1 -= needCompare;
+ } else { // same, should go ahead both.
+ seg1 = segments[varSegIndex1++];
+ seg2 = other.segments[varSegIndex2++];
+ offset1 = 0;
+ offset2 = 0;
+ sizeOfFirst1 = segmentSize;
+ sizeOfFirst2 = otherSegmentSize;
+ }
+ needCompare = Math.min(Math.min(sizeOfFirst1, sizeOfFirst2), len);
+ }
+
+ checkArgument(needCompare == len);
+
+ return sizeInBytes - other.sizeInBytes;
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Public methods on BinaryString
+ // ------------------------------------------------------------------------------------------
+
+ /** Returns the number of UTF-8 code points in the string. */
+ public int numChars() {
+ if (inFirstSegment()) {
+ int len = 0;
+ for (int i = 0; i < sizeInBytes; i += numBytesForFirstByte(getByteOneSegment(i))) {
+ len++;
+ }
+ return len;
+ } else {
+ return numCharsMultiSegs();
+ }
+ }
+
+ private int numCharsMultiSegs() {
+ int len = 0;
+ int segSize = segments[0].size();
+ BinaryString.SegmentAndOffset index = firstSegmentAndOffset(segSize);
+ int i = 0;
+ while (i < sizeInBytes) {
+ int charBytes = numBytesForFirstByte(index.value());
+ i += charBytes;
+ len++;
+ index.skipBytes(charBytes, segSize);
+ }
+ return len;
+ }
+
+ /**
+ * Returns the {@code byte} value at the specified index. An index ranges from {@code 0} to
+ * {@code sizeInBytes - 1}.
+ *
+ * @param index the index of the {@code byte} value.
+ * @return the {@code byte} value at the specified index of this UTF-8 bytes.
+ * @exception IndexOutOfBoundsException if the {@code index} argument is negative or not less
+ * than the length of this UTF-8 bytes.
+ */
+ public byte byteAt(int index) {
+ int globalOffset = offset + index;
+ int size = segments[0].size();
+ if (globalOffset < size) {
+ return segments[0].get(globalOffset);
+ } else {
+ return segments[globalOffset / size].get(globalOffset % size);
+ }
+ }
+
+ /** Copy a new {@code BinaryString}. */
+ public BinaryString copy() {
+ byte[] copy = MemorySegmentUtils.copyToBytes(segments, offset, sizeInBytes);
+ return BinaryString.fromBytes(copy);
+ }
+
+ /**
+ * Returns a binary string that is a substring of this binary string. The substring begins at
+ * the specified {@code beginIndex} and extends to the character at index {@code endIndex - 1}.
+ *
+ * <p>Examples:
+ *
+ * <blockquote>
+ *
+ * <pre>
+ * fromString("hamburger").substring(4, 8) returns binary string "urge"
+ * fromString("smiles").substring(1, 5) returns binary string "mile"
+ * </pre>
+ *
+ * </blockquote>
+ *
+ * @param beginIndex the beginning index, inclusive.
+ * @param endIndex the ending index, exclusive.
+ * @return the specified substring, return EMPTY_UTF8 when index out of bounds instead of
+ * StringIndexOutOfBoundsException.
+ */
+ public BinaryString substring(int beginIndex, int endIndex) {
+ if (endIndex <= beginIndex || beginIndex >= sizeInBytes) {
+ return EMPTY_UTF8;
+ }
+ if (inFirstSegment()) {
+ MemorySegment segment = segments[0];
+ int i = 0;
+ int c = 0;
+ while (i < sizeInBytes && c < beginIndex) {
+ i += numBytesForFirstByte(segment.get(i + offset));
+ c += 1;
+ }
+
+ int j = i;
+ while (i < sizeInBytes && c < endIndex) {
+ i += numBytesForFirstByte(segment.get(i + offset));
+ c += 1;
+ }
+
+ if (i > j) {
+ byte[] bytes = new byte[i - j];
+ segment.get(offset + j, bytes, 0, i - j);
+ return fromBytes(bytes);
+ } else {
+ return EMPTY_UTF8;
+ }
+ } else {
+ return substringMultiSegs(beginIndex, endIndex);
+ }
+ }
+
+ private BinaryString substringMultiSegs(final int start, final int until) {
+ int segSize = segments[0].size();
+ BinaryString.SegmentAndOffset index = firstSegmentAndOffset(segSize);
+ int i = 0;
+ int c = 0;
+ while (i < sizeInBytes && c < start) {
+ int charSize = numBytesForFirstByte(index.value());
+ i += charSize;
+ index.skipBytes(charSize, segSize);
+ c += 1;
+ }
+
+ int j = i;
+ while (i < sizeInBytes && c < until) {
+ int charSize = numBytesForFirstByte(index.value());
+ i += charSize;
+ index.skipBytes(charSize, segSize);
+ c += 1;
+ }
+
+ if (i > j) {
+ return fromBytes(MemorySegmentUtils.copyToBytes(segments, offset + j, i - j));
+ } else {
+ return EMPTY_UTF8;
+ }
+ }
+
+ /**
+ * Returns true if and only if this BinaryString contains the specified sequence of bytes
+ * values.
+ *
+ * @param s the sequence to search for
+ * @return true if this BinaryString contains {@code s}, false otherwise
+ */
+ public boolean contains(final BinaryString s) {
+ if (s.sizeInBytes == 0) {
+ return true;
+ }
+ int find =
+ MemorySegmentUtils.find(
+ segments, offset, sizeInBytes, s.segments, s.offset, s.sizeInBytes);
+ return find != -1;
+ }
+
+ /**
+ * Tests if this BinaryString starts with the specified prefix.
+ *
+ * @param prefix the prefix.
+ * @return {@code true} if the bytes represented by the argument is a prefix of the bytes
+ * represented by this string; {@code false} otherwise. Note also that {@code true} will be
+ * returned if the argument is an empty BinaryString or is equal to this {@code
+ * BinaryString} object as determined by the {@link #equals(Object)} method.
+ */
+ public boolean startsWith(final BinaryString prefix) {
+ return matchAt(prefix, 0);
+ }
+
+ /**
+ * Tests if this BinaryString ends with the specified suffix.
+ *
+ * @param suffix the suffix.
+ * @return {@code true} if the bytes represented by the argument is a suffix of the bytes
+ * represented by this object; {@code false} otherwise. Note that the result will be {@code
+ * true} if the argument is the empty string or is equal to this {@code BinaryString} object
+ * as determined by the {@link #equals(Object)} method.
+ */
+ public boolean endsWith(final BinaryString suffix) {
+ return matchAt(suffix, sizeInBytes - suffix.sizeInBytes);
+ }
+
+ /**
+ * Returns a string whose value is this string, with any leading and trailing whitespace
+ * removed.
+ *
+ * @return A string whose value is this string, with any leading and trailing white space
+ * removed, or this string if it has no leading or trailing white space.
+ */
+ public BinaryString trim() {
+ if (inFirstSegment()) {
+ int s = 0;
+ int e = this.sizeInBytes - 1;
+ // skip all of the space (0x20) in the left side
+ while (s < this.sizeInBytes && getByteOneSegment(s) == 0x20) {
+ s++;
+ }
+ // skip all of the space (0x20) in the right side
+ while (e >= s && getByteOneSegment(e) == 0x20) {
+ e--;
+ }
+ if (s > e) {
+ // empty string
+ return EMPTY_UTF8;
+ } else {
+ return copyBinaryStringInOneSeg(s, e - s + 1);
+ }
+ } else {
+ return trimMultiSegs();
+ }
+ }
+
+ private BinaryString trimMultiSegs() {
+ int s = 0;
+ int e = this.sizeInBytes - 1;
+ int segSize = segments[0].size();
+ BinaryString.SegmentAndOffset front = firstSegmentAndOffset(segSize);
+ // skip all of the space (0x20) in the left side
+ while (s < this.sizeInBytes && front.value() == 0x20) {
+ s++;
+ front.nextByte(segSize);
+ }
+ BinaryString.SegmentAndOffset behind = lastSegmentAndOffset(segSize);
+ // skip all of the space (0x20) in the right side
+ while (e >= s && behind.value() == 0x20) {
+ e--;
+ behind.previousByte(segSize);
+ }
+ if (s > e) {
+ // empty string
+ return EMPTY_UTF8;
+ } else {
+ return copyBinaryString(s, e);
+ }
+ }
+
+ /**
+ * Returns the index within this string of the first occurrence of the specified substring,
+ * starting at the specified index.
+ *
+ * @param str the substring to search for.
+ * @param fromIndex the index from which to start the search.
+ * @return the index of the first occurrence of the specified substring, starting at the
+ * specified index, or {@code -1} if there is no such occurrence.
+ */
+ public int indexOf(BinaryString str, int fromIndex) {
+ if (str.sizeInBytes == 0) {
+ return 0;
+ }
+ if (inFirstSegment()) {
+ // position in byte
+ int byteIdx = 0;
+ // position is char
+ int charIdx = 0;
+ while (byteIdx < sizeInBytes && charIdx < fromIndex) {
+ byteIdx += numBytesForFirstByte(getByteOneSegment(byteIdx));
+ charIdx++;
+ }
+ do {
+ if (byteIdx + str.sizeInBytes > sizeInBytes) {
+ return -1;
+ }
+ if (MemorySegmentUtils.equals(
+ segments, offset + byteIdx, str.segments, str.offset, str.sizeInBytes)) {
+ return charIdx;
+ }
+ byteIdx += numBytesForFirstByte(getByteOneSegment(byteIdx));
+ charIdx++;
+ } while (byteIdx < sizeInBytes);
+
+ return -1;
+ } else {
+ return indexOfMultiSegs(str, fromIndex);
+ }
+ }
+
+ private int indexOfMultiSegs(BinaryString str, int fromIndex) {
+ // position in byte
+ int byteIdx = 0;
+ // position is char
+ int charIdx = 0;
+ int segSize = segments[0].size();
+ BinaryString.SegmentAndOffset index = firstSegmentAndOffset(segSize);
+ while (byteIdx < sizeInBytes && charIdx < fromIndex) {
+ int charBytes = numBytesForFirstByte(index.value());
+ byteIdx += charBytes;
+ charIdx++;
+ index.skipBytes(charBytes, segSize);
+ }
+ do {
+ if (byteIdx + str.sizeInBytes > sizeInBytes) {
+ return -1;
+ }
+ if (MemorySegmentUtils.equals(
+ segments, offset + byteIdx, str.segments, str.offset, str.sizeInBytes)) {
+ return charIdx;
+ }
+ int charBytes = numBytesForFirstByte(index.segment.get(index.offset));
+ byteIdx += charBytes;
+ charIdx++;
+ index.skipBytes(charBytes, segSize);
+ } while (byteIdx < sizeInBytes);
+
+ return -1;
+ }
+
+ /**
+ * Converts all of the characters in this {@code BinaryString} to upper case.
+ *
+ * @return the {@code BinaryString}, converted to uppercase.
+ */
+ public BinaryString toUpperCase() {
+ if (sizeInBytes == 0) {
+ return EMPTY_UTF8;
+ }
+ int size = segments[0].size();
+ BinaryString.SegmentAndOffset segmentAndOffset = startSegmentAndOffset(size);
+ byte[] bytes = new byte[sizeInBytes];
+ bytes[0] = (byte) Character.toTitleCase(segmentAndOffset.value());
+ for (int i = 0; i < sizeInBytes; i++) {
+ byte b = segmentAndOffset.value();
+ if (numBytesForFirstByte(b) != 1) {
+ // fallback
+ return javaToUpperCase();
+ }
+ int upper = Character.toUpperCase((int) b);
+ if (upper > 127) {
+ // fallback
+ return javaToUpperCase();
+ }
+ bytes[i] = (byte) upper;
+ segmentAndOffset.nextByte(size);
+ }
+ return fromBytes(bytes);
+ }
+
+ private BinaryString javaToUpperCase() {
+ return fromString(toString().toUpperCase());
+ }
+
+ /**
+ * Converts all of the characters in this {@code BinaryString} to lower case.
+ *
+ * @return the {@code BinaryString}, converted to lowercase.
+ */
+ public BinaryString toLowerCase() {
+ if (sizeInBytes == 0) {
+ return EMPTY_UTF8;
+ }
+ int size = segments[0].size();
+ BinaryString.SegmentAndOffset segmentAndOffset = startSegmentAndOffset(size);
+ byte[] bytes = new byte[sizeInBytes];
+ bytes[0] = (byte) Character.toTitleCase(segmentAndOffset.value());
+ for (int i = 0; i < sizeInBytes; i++) {
+ byte b = segmentAndOffset.value();
+ if (numBytesForFirstByte(b) != 1) {
+ // fallback
+ return javaToLowerCase();
+ }
+ int lower = Character.toLowerCase((int) b);
+ if (lower > 127) {
+ // fallback
+ return javaToLowerCase();
+ }
+ bytes[i] = (byte) lower;
+ segmentAndOffset.nextByte(size);
+ }
+ return fromBytes(bytes);
+ }
+
+ private BinaryString javaToLowerCase() {
+ return fromString(toString().toLowerCase());
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Internal methods on BinaryString
+ // ------------------------------------------------------------------------------------------
+
+ byte getByteOneSegment(int i) {
+ return segments[0].get(offset + i);
+ }
+
+ boolean inFirstSegment() {
+ return sizeInBytes + offset <= segments[0].size();
+ }
+
+ private boolean matchAt(final BinaryString s, int pos) {
+ return (inFirstSegment() && s.inFirstSegment())
+ ? matchAtOneSeg(s, pos)
+ : matchAtVarSeg(s, pos);
+ }
+
+ private boolean matchAtOneSeg(final BinaryString s, int pos) {
+ return s.sizeInBytes + pos <= sizeInBytes
+ && pos >= 0
+ && segments[0].equalTo(s.segments[0], offset + pos, s.offset, s.sizeInBytes);
+ }
+
+ private boolean matchAtVarSeg(final BinaryString s, int pos) {
+ return s.sizeInBytes + pos <= sizeInBytes
+ && pos >= 0
+ && MemorySegmentUtils.equals(
+ segments, offset + pos, s.segments, s.offset, s.sizeInBytes);
+ }
+
+ BinaryString copyBinaryStringInOneSeg(int start, int len) {
+ byte[] newBytes = new byte[len];
+ segments[0].get(offset + start, newBytes, 0, len);
+ return fromBytes(newBytes);
+ }
+
+ BinaryString copyBinaryString(int start, int end) {
+ int len = end - start + 1;
+ byte[] newBytes = new byte[len];
+ MemorySegmentUtils.copyToBytes(segments, offset + start, newBytes, 0, len);
+ return fromBytes(newBytes);
+ }
+
+ BinaryString.SegmentAndOffset firstSegmentAndOffset(int segSize) {
+ int segIndex = offset / segSize;
+ return new BinaryString.SegmentAndOffset(segIndex, offset % segSize);
+ }
+
+ BinaryString.SegmentAndOffset lastSegmentAndOffset(int segSize) {
+ int lastOffset = offset + sizeInBytes - 1;
+ int segIndex = lastOffset / segSize;
+ return new BinaryString.SegmentAndOffset(segIndex, lastOffset % segSize);
+ }
+
+ private BinaryString.SegmentAndOffset startSegmentAndOffset(int segSize) {
+ return inFirstSegment()
+ ? new BinaryString.SegmentAndOffset(0, offset)
+ : firstSegmentAndOffset(segSize);
+ }
+
+ /** CurrentSegment and positionInSegment. */
+ class SegmentAndOffset {
+ int segIndex;
+ MemorySegment segment;
+ int offset;
+
+ private SegmentAndOffset(int segIndex, int offset) {
+ this.segIndex = segIndex;
+ this.segment = segments[segIndex];
+ this.offset = offset;
+ }
+
+ private void assignSegment() {
+ segment = segIndex >= 0 && segIndex < segments.length ? segments[segIndex] : null;
+ }
+
+ void previousByte(int segSize) {
+ offset--;
+ if (offset == -1) {
+ segIndex--;
+ assignSegment();
+ offset = segSize - 1;
+ }
+ }
+
+ void nextByte(int segSize) {
+ offset++;
+ checkAdvance(segSize);
+ }
+
+ private void checkAdvance(int segSize) {
+ if (offset == segSize) {
+ advance();
+ }
+ }
+
+ private void advance() {
+ segIndex++;
+ assignSegment();
+ offset = 0;
+ }
+
+ void skipBytes(int n, int segSize) {
+ int remaining = segSize - this.offset;
+ if (remaining > n) {
+ this.offset += n;
+ } else {
+ while (true) {
+ int toSkip = Math.min(remaining, n);
+ n -= toSkip;
+ if (n <= 0) {
+ this.offset += toSkip;
+ checkAdvance(segSize);
+ return;
+ }
+ advance();
+ remaining = segSize - this.offset;
+ }
+ }
+ }
+
+ byte value() {
+ return this.segment.get(this.offset);
+ }
+ }
+
+ /**
+ * Returns the number of bytes for a code point with the first byte as `b`.
+ *
+ * @param b The first byte of a code point
+ */
+ static int numBytesForFirstByte(final byte b) {
+ if (b >= 0) {
+ // 1 byte, 7 bits: 0xxxxxxx
+ return 1;
+ } else if ((b >> 5) == -2 && (b & 0x1e) != 0) {
+ // 2 bytes, 11 bits: 110xxxxx 10xxxxxx
+ return 2;
+ } else if ((b >> 4) == -2) {
+ // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx
+ return 3;
+ } else if ((b >> 3) == -2) {
+ // 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+ return 4;
+ } else {
+ // Skip the first byte disallowed in UTF-8
+ // Handling errors quietly, same semantics to java String.
+ return 1;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // UTF-8 encoding and decoding (High-speed version, 30% faster+)
+ // ------------------------------------------------------------------------
+
+ private static final int MAX_BYTES_PER_CHAR = 3;
+
+ /** This method must have the same result with JDK's String.getBytes. */
+ public static byte[] encodeUTF8(String str) {
+ byte[] bytes = allocateReuseBytes(str.length() * MAX_BYTES_PER_CHAR);
+ int len = encodeUTF8(str, bytes);
+ return Arrays.copyOf(bytes, len);
+ }
+
+ public static int encodeUTF8(String str, byte[] bytes) {
+ int offset = 0;
+ int len = str.length();
+ int sl = offset + len;
+ int dp = 0;
+ int dlASCII = dp + Math.min(len, bytes.length);
+
+ // ASCII only optimized loop
+ while (dp < dlASCII && str.charAt(offset) < '\u0080') {
+ bytes[dp++] = (byte) str.charAt(offset++);
+ }
+
+ while (offset < sl) {
+ char c = str.charAt(offset++);
+ if (c < 0x80) {
+ // Have at most seven bits
+ bytes[dp++] = (byte) c;
+ } else if (c < 0x800) {
+ // 2 bytes, 11 bits
+ bytes[dp++] = (byte) (0xc0 | (c >> 6));
+ bytes[dp++] = (byte) (0x80 | (c & 0x3f));
+ } else if (Character.isSurrogate(c)) {
+ final int uc;
+ int ip = offset - 1;
+ if (Character.isHighSurrogate(c)) {
+ if (sl - ip < 2) {
+ uc = -1;
+ } else {
+ char d = str.charAt(ip + 1);
+ if (Character.isLowSurrogate(d)) {
+ uc = Character.toCodePoint(c, d);
+ } else {
+ // for some illegal character
+ // the jdk will ignore the origin character and cast it to '?'
+ // this acts the same with jdk
+ return defaultEncodeUTF8(str, bytes);
+ }
+ }
+ } else {
+ if (Character.isLowSurrogate(c)) {
+ // for some illegal character
+ // the jdk will ignore the origin character and cast it to '?'
+ // this acts the same with jdk
+ return defaultEncodeUTF8(str, bytes);
+ } else {
+ uc = c;
+ }
+ }
+
+ if (uc < 0) {
+ bytes[dp++] = (byte) '?';
+ } else {
+ bytes[dp++] = (byte) (0xf0 | ((uc >> 18)));
+ bytes[dp++] = (byte) (0x80 | ((uc >> 12) & 0x3f));
+ bytes[dp++] = (byte) (0x80 | ((uc >> 6) & 0x3f));
+ bytes[dp++] = (byte) (0x80 | (uc & 0x3f));
+ offset++; // 2 chars
+ }
+ } else {
+ // 3 bytes, 16 bits
+ bytes[dp++] = (byte) (0xe0 | ((c >> 12)));
+ bytes[dp++] = (byte) (0x80 | ((c >> 6) & 0x3f));
+ bytes[dp++] = (byte) (0x80 | (c & 0x3f));
+ }
+ }
+ return dp;
+ }
+
+ public static int defaultEncodeUTF8(String str, byte[] bytes) {
+ try {
+ byte[] buffer = str.getBytes("UTF-8");
+ System.arraycopy(buffer, 0, bytes, 0, buffer.length);
+ return buffer.length;
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException("encodeUTF8 error", e);
+ }
+ }
+
+ public static String decodeUTF8(byte[] input, int offset, int byteLen) {
+ char[] chars = allocateReuseChars(byteLen);
+ int len = decodeUTF8Strict(input, offset, byteLen, chars);
+ if (len < 0) {
+ return defaultDecodeUTF8(input, offset, byteLen);
+ }
+ return new String(chars, 0, len);
+ }
+
+ public static int decodeUTF8Strict(byte[] sa, int sp, int len, char[] da) {
+ final int sl = sp + len;
+ int dp = 0;
+ int dlASCII = Math.min(len, da.length);
+
+ // ASCII only optimized loop
+ while (dp < dlASCII && sa[sp] >= 0) {
+ da[dp++] = (char) sa[sp++];
+ }
+
+ while (sp < sl) {
+ int b1 = sa[sp++];
+ if (b1 >= 0) {
+ // 1 byte, 7 bits: 0xxxxxxx
+ da[dp++] = (char) b1;
+ } else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) {
+ // 2 bytes, 11 bits: 110xxxxx 10xxxxxx
+ if (sp < sl) {
+ int b2 = sa[sp++];
+ if ((b2 & 0xc0) != 0x80) { // isNotContinuation(b2)
+ return -1;
+ } else {
+ da[dp++] = (char) (((b1 << 6) ^ b2) ^ (((byte) 0xC0 << 6) ^ ((byte) 0x80)));
+ }
+ continue;
+ }
+ return -1;
+ } else if ((b1 >> 4) == -2) {
+ // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx
+ if (sp + 1 < sl) {
+ int b2 = sa[sp++];
+ int b3 = sa[sp++];
+ if ((b1 == (byte) 0xe0 && (b2 & 0xe0) == 0x80)
+ || (b2 & 0xc0) != 0x80
+ || (b3 & 0xc0) != 0x80) { // isMalformed3(b1, b2, b3)
+ return -1;
+ } else {
+ char c =
+ (char)
+ ((b1 << 12)
+ ^ (b2 << 6)
+ ^ (b3
+ ^ (((byte) 0xE0 << 12)
+ ^ ((byte) 0x80 << 6)
+ ^ ((byte) 0x80))));
+ if (Character.isSurrogate(c)) {
+ return -1;
+ } else {
+ da[dp++] = c;
+ }
+ }
+ continue;
+ }
+ return -1;
+ } else if ((b1 >> 3) == -2) {
+ // 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+ if (sp + 2 < sl) {
+ int b2 = sa[sp++];
+ int b3 = sa[sp++];
+ int b4 = sa[sp++];
+ int uc =
+ ((b1 << 18)
+ ^ (b2 << 12)
+ ^ (b3 << 6)
+ ^ (b4
+ ^ (((byte) 0xF0 << 18)
+ ^ ((byte) 0x80 << 12)
+ ^ ((byte) 0x80 << 6)
+ ^ ((byte) 0x80))));
+ // isMalformed4 and shortest form check
+ if (((b2 & 0xc0) != 0x80 || (b3 & 0xc0) != 0x80 || (b4 & 0xc0) != 0x80)
+ || !Character.isSupplementaryCodePoint(uc)) {
+ return -1;
+ } else {
+ da[dp++] = Character.highSurrogate(uc);
+ da[dp++] = Character.lowSurrogate(uc);
+ }
+ continue;
+ }
+ return -1;
+ } else {
+ return -1;
+ }
+ }
+ return dp;
+ }
+
+ public static String decodeUTF8(MemorySegment input, int offset, int byteLen) {
+ char[] chars = allocateReuseChars(byteLen);
+ int len = decodeUTF8Strict(input, offset, byteLen, chars);
+ if (len < 0) {
+ byte[] bytes = allocateReuseBytes(byteLen);
+ input.get(offset, bytes, 0, byteLen);
+ return defaultDecodeUTF8(bytes, 0, byteLen);
+ }
+ return new String(chars, 0, len);
+ }
+
+ public static int decodeUTF8Strict(MemorySegment segment, int sp, int len, char[] da) {
+ final int sl = sp + len;
+ int dp = 0;
+ int dlASCII = Math.min(len, da.length);
+
+ // ASCII only optimized loop
+ while (dp < dlASCII && segment.get(sp) >= 0) {
+ da[dp++] = (char) segment.get(sp++);
+ }
+
+ while (sp < sl) {
+ int b1 = segment.get(sp++);
+ if (b1 >= 0) {
+ // 1 byte, 7 bits: 0xxxxxxx
+ da[dp++] = (char) b1;
+ } else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) {
+ // 2 bytes, 11 bits: 110xxxxx 10xxxxxx
+ if (sp < sl) {
+ int b2 = segment.get(sp++);
+ if ((b2 & 0xc0) != 0x80) { // isNotContinuation(b2)
+ return -1;
+ } else {
+ da[dp++] = (char) (((b1 << 6) ^ b2) ^ (((byte) 0xC0 << 6) ^ ((byte) 0x80)));
+ }
+ continue;
+ }
+ return -1;
+ } else if ((b1 >> 4) == -2) {
+ // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx
+ if (sp + 1 < sl) {
+ int b2 = segment.get(sp++);
+ int b3 = segment.get(sp++);
+ if ((b1 == (byte) 0xe0 && (b2 & 0xe0) == 0x80)
+ || (b2 & 0xc0) != 0x80
+ || (b3 & 0xc0) != 0x80) { // isMalformed3(b1, b2, b3)
+ return -1;
+ } else {
+ char c =
+ (char)
+ ((b1 << 12)
+ ^ (b2 << 6)
+ ^ (b3
+ ^ (((byte) 0xE0 << 12)
+ ^ ((byte) 0x80 << 6)
+ ^ ((byte) 0x80))));
+ if (Character.isSurrogate(c)) {
+ return -1;
+ } else {
+ da[dp++] = c;
+ }
+ }
+ continue;
+ }
+ return -1;
+ } else if ((b1 >> 3) == -2) {
+ // 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+ if (sp + 2 < sl) {
+ int b2 = segment.get(sp++);
+ int b3 = segment.get(sp++);
+ int b4 = segment.get(sp++);
+ int uc =
+ ((b1 << 18)
+ ^ (b2 << 12)
+ ^ (b3 << 6)
+ ^ (b4
+ ^ (((byte) 0xF0 << 18)
+ ^ ((byte) 0x80 << 12)
+ ^ ((byte) 0x80 << 6)
+ ^ ((byte) 0x80))));
+ // isMalformed4 and shortest form check
+ if (((b2 & 0xc0) != 0x80 || (b3 & 0xc0) != 0x80 || (b4 & 0xc0) != 0x80)
+ || !Character.isSupplementaryCodePoint(uc)) {
+ return -1;
+ } else {
+ da[dp++] = Character.highSurrogate(uc);
+ da[dp++] = Character.lowSurrogate(uc);
+ }
+ continue;
+ }
+ return -1;
+ } else {
+ return -1;
+ }
+ }
+ return dp;
+ }
+
+ public static String defaultDecodeUTF8(byte[] bytes, int offset, int len) {
+ return new String(bytes, offset, len, StandardCharsets.UTF_8);
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/StringDataSerializer.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryStringSerializer.java
similarity index 56%
rename from flink-table-store-common/src/main/java/org/apache/flink/table/store/data/StringDataSerializer.java
rename to flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryStringSerializer.java
index 7c3d4de2..e49434f3 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/StringDataSerializer.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryStringSerializer.java
@@ -18,27 +18,23 @@
package org.apache.flink.table.store.data;
-import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.SimpleTypeSerializerSnapshot;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.binary.BinarySegmentUtils;
-import org.apache.flink.table.data.binary.BinaryStringData;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
import java.io.IOException;
-/** Serializer for {@link StringData}. */
-@Internal
-public final class StringDataSerializer extends TypeSerializerSingleton<StringData> {
+/** Serializer for {@link BinaryString}. */
+public final class BinaryStringSerializer extends TypeSerializerSingleton<BinaryString> {
private static final long serialVersionUID = 1L;
- public static final StringDataSerializer INSTANCE = new StringDataSerializer();
+ public static final BinaryStringSerializer INSTANCE = new BinaryStringSerializer();
- private StringDataSerializer() {}
+ private BinaryStringSerializer() {}
@Override
public boolean isImmutableType() {
@@ -46,20 +42,20 @@ public final class StringDataSerializer extends TypeSerializerSingleton<StringDa
}
@Override
- public StringData createInstance() {
- return StringData.fromString("");
+ public BinaryString createInstance() {
+ return BinaryString.fromString("");
}
@Override
- public StringData copy(StringData from) {
- // BinaryStringData is the only implementation of StringData
- return ((BinaryStringData) from).copy();
+ public BinaryString copy(BinaryString from) {
+ // BinaryString is the only implementation of BinaryString
+ return ((BinaryString) from).copy();
}
@Override
- public StringData copy(StringData from, StringData reuse) {
- // BinaryStringData is the only implementation of StringData
- return ((BinaryStringData) from).copy();
+ public BinaryString copy(BinaryString from, BinaryString reuse) {
+ // BinaryString is the only implementation of BinaryString
+ return ((BinaryString) from).copy();
}
@Override
@@ -68,29 +64,26 @@ public final class StringDataSerializer extends TypeSerializerSingleton<StringDa
}
@Override
- public void serialize(StringData record, DataOutputView target) throws IOException {
- // BinaryStringData is the only implementation of StringData
- BinaryStringData string = (BinaryStringData) record;
- string.ensureMaterialized();
+ public void serialize(BinaryString string, DataOutputView target) throws IOException {
target.writeInt(string.getSizeInBytes());
- BinarySegmentUtils.copyToView(
+ MemorySegmentUtils.copyToView(
string.getSegments(), string.getOffset(), string.getSizeInBytes(), target);
}
@Override
- public StringData deserialize(DataInputView source) throws IOException {
+ public BinaryString deserialize(DataInputView source) throws IOException {
return deserializeInternal(source);
}
- public static StringData deserializeInternal(DataInputView source) throws IOException {
+ public static BinaryString deserializeInternal(DataInputView source) throws IOException {
int length = source.readInt();
byte[] bytes = new byte[length];
source.readFully(bytes);
- return StringData.fromBytes(bytes);
+ return BinaryString.fromBytes(bytes);
}
@Override
- public StringData deserialize(StringData record, DataInputView source) throws IOException {
+ public BinaryString deserialize(BinaryString record, DataInputView source) throws IOException {
return deserialize(source);
}
@@ -102,16 +95,16 @@ public final class StringDataSerializer extends TypeSerializerSingleton<StringDa
}
@Override
- public TypeSerializerSnapshot<StringData> snapshotConfiguration() {
- return new StringDataSerializerSnapshot();
+ public TypeSerializerSnapshot<BinaryString> snapshotConfiguration() {
+ return new BinaryStringSerializerSnapshot();
}
/** Serializer configuration snapshot for compatibility and format evolution. */
@SuppressWarnings("WeakerAccess")
- public static final class StringDataSerializerSnapshot
- extends SimpleTypeSerializerSnapshot<StringData> {
+ public static final class BinaryStringSerializerSnapshot
+ extends SimpleTypeSerializerSnapshot<BinaryString> {
- public StringDataSerializerSnapshot() {
+ public BinaryStringSerializerSnapshot() {
super(() -> INSTANCE);
}
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryWriter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryWriter.java
index e91179cf..005b4ae3 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryWriter.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/BinaryWriter.java
@@ -19,21 +19,14 @@ package org.apache.flink.table.store.data;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.types.logical.DecimalType;
-import org.apache.flink.table.types.logical.DistinctType;
-import org.apache.flink.table.types.logical.LocalZonedTimestampType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.flink.table.store.types.DataType;
+import org.apache.flink.table.store.types.DecimalType;
+import org.apache.flink.table.store.types.LocalZonedTimestampType;
+import org.apache.flink.table.store.types.TimestampType;
import java.io.Serializable;
-import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getPrecision;
+import static org.apache.flink.table.store.types.DataTypeChecks.getPrecision;
/**
* Writer to write a composite data format, like row, array. 1. Invoke {@link #reset()}. 2. Write
@@ -63,19 +56,19 @@ public interface BinaryWriter {
void writeDouble(int pos, double value);
- void writeString(int pos, StringData value);
+ void writeString(int pos, BinaryString value);
void writeBinary(int pos, byte[] bytes);
- void writeDecimal(int pos, DecimalData value, int precision);
+ void writeDecimal(int pos, Decimal value, int precision);
- void writeTimestamp(int pos, TimestampData value, int precision);
+ void writeTimestamp(int pos, Timestamp value, int precision);
- void writeArray(int pos, ArrayData value, ArrayDataSerializer serializer);
+ void writeArray(int pos, InternalArray value, ArrayDataSerializer serializer);
- void writeMap(int pos, MapData value, MapDataSerializer serializer);
+ void writeMap(int pos, InternalMap value, MapDataSerializer serializer);
- void writeRow(int pos, RowData value, RowDataSerializer serializer);
+ void writeRow(int pos, InternalRow value, RowDataSerializer serializer);
/** Finally, complete write to set real size to binary. */
void complete();
@@ -83,16 +76,12 @@ public interface BinaryWriter {
// --------------------------------------------------------------------------------------------
/**
- * @deprecated Use {@link #createValueSetter(LogicalType)} for avoiding logical types during
+ * @deprecated Use {@code #createValueSetter(DataType)} for avoiding logical types during
* runtime.
*/
@Deprecated
static void write(
- BinaryWriter writer,
- int pos,
- Object o,
- LogicalType type,
- TypeSerializer<?> serializer) {
+ BinaryWriter writer, int pos, Object o, DataType type, TypeSerializer<?> serializer) {
switch (type.getTypeRoot()) {
case BOOLEAN:
writer.writeBoolean(pos, (boolean) o);
@@ -106,20 +95,18 @@ public interface BinaryWriter {
case INTEGER:
case DATE:
case TIME_WITHOUT_TIME_ZONE:
- case INTERVAL_YEAR_MONTH:
writer.writeInt(pos, (int) o);
break;
case BIGINT:
- case INTERVAL_DAY_TIME:
writer.writeLong(pos, (long) o);
break;
case TIMESTAMP_WITHOUT_TIME_ZONE:
TimestampType timestampType = (TimestampType) type;
- writer.writeTimestamp(pos, (TimestampData) o, timestampType.getPrecision());
+ writer.writeTimestamp(pos, (Timestamp) o, timestampType.getPrecision());
break;
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
LocalZonedTimestampType lzTs = (LocalZonedTimestampType) type;
- writer.writeTimestamp(pos, (TimestampData) o, lzTs.getPrecision());
+ writer.writeTimestamp(pos, (Timestamp) o, lzTs.getPrecision());
break;
case FLOAT:
writer.writeFloat(pos, (float) o);
@@ -129,22 +116,21 @@ public interface BinaryWriter {
break;
case CHAR:
case VARCHAR:
- writer.writeString(pos, (StringData) o);
+ writer.writeString(pos, (BinaryString) o);
break;
case DECIMAL:
DecimalType decimalType = (DecimalType) type;
- writer.writeDecimal(pos, (DecimalData) o, decimalType.getPrecision());
+ writer.writeDecimal(pos, (Decimal) o, decimalType.getPrecision());
break;
case ARRAY:
- writer.writeArray(pos, (ArrayData) o, (ArrayDataSerializer) serializer);
+ writer.writeArray(pos, (InternalArray) o, (ArrayDataSerializer) serializer);
break;
case MAP:
case MULTISET:
- writer.writeMap(pos, (MapData) o, (MapDataSerializer) serializer);
+ writer.writeMap(pos, (InternalMap) o, (MapDataSerializer) serializer);
break;
case ROW:
- case STRUCTURED_TYPE:
- writer.writeRow(pos, (RowData) o, (RowDataSerializer) serializer);
+ writer.writeRow(pos, (InternalRow) o, (RowDataSerializer) serializer);
break;
case BINARY:
case VARBINARY:
@@ -160,12 +146,12 @@ public interface BinaryWriter {
*
* @param elementType the element type of the array
*/
- static ValueSetter createValueSetter(LogicalType elementType) {
+ static ValueSetter createValueSetter(DataType elementType) {
// ordered by type root definition
switch (elementType.getTypeRoot()) {
case CHAR:
case VARCHAR:
- return (writer, pos, value) -> writer.writeString(pos, (StringData) value);
+ return (writer, pos, value) -> writer.writeString(pos, (BinaryString) value);
case BOOLEAN:
return (writer, pos, value) -> writer.writeBoolean(pos, (boolean) value);
case BINARY:
@@ -174,7 +160,7 @@ public interface BinaryWriter {
case DECIMAL:
final int decimalPrecision = getPrecision(elementType);
return (writer, pos, value) ->
- writer.writeDecimal(pos, (DecimalData) value, decimalPrecision);
+ writer.writeDecimal(pos, (Decimal) value, decimalPrecision);
case TINYINT:
return (writer, pos, value) -> writer.writeByte(pos, (byte) value);
case SMALLINT:
@@ -182,10 +168,8 @@ public interface BinaryWriter {
case INTEGER:
case DATE:
case TIME_WITHOUT_TIME_ZONE:
- case INTERVAL_YEAR_MONTH:
return (writer, pos, value) -> writer.writeInt(pos, (int) value);
case BIGINT:
- case INTERVAL_DAY_TIME:
return (writer, pos, value) -> writer.writeLong(pos, (long) value);
case FLOAT:
return (writer, pos, value) -> writer.writeFloat(pos, (float) value);
@@ -195,29 +179,26 @@ public interface BinaryWriter {
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
final int timestampPrecision = getPrecision(elementType);
return (writer, pos, value) ->
- writer.writeTimestamp(pos, (TimestampData) value, timestampPrecision);
- case TIMESTAMP_WITH_TIME_ZONE:
- throw new UnsupportedOperationException();
+ writer.writeTimestamp(pos, (Timestamp) value, timestampPrecision);
case ARRAY:
- final TypeSerializer<ArrayData> arraySerializer =
+ final TypeSerializer<InternalArray> arraySerializer =
InternalSerializers.create(elementType);
return (writer, pos, value) ->
writer.writeArray(
- pos, (ArrayData) value, (ArrayDataSerializer) arraySerializer);
+ pos, (InternalArray) value, (ArrayDataSerializer) arraySerializer);
case MULTISET:
case MAP:
- final TypeSerializer<MapData> mapSerializer =
+ final TypeSerializer<InternalMap> mapSerializer =
InternalSerializers.create(elementType);
return (writer, pos, value) ->
- writer.writeMap(pos, (MapData) value, (MapDataSerializer) mapSerializer);
+ writer.writeMap(
+ pos, (InternalMap) value, (MapDataSerializer) mapSerializer);
case ROW:
- case STRUCTURED_TYPE:
- final TypeSerializer<RowData> rowSerializer =
+ final TypeSerializer<InternalRow> rowSerializer =
InternalSerializers.create(elementType);
return (writer, pos, value) ->
- writer.writeRow(pos, (RowData) value, (RowDataSerializer) rowSerializer);
- case DISTINCT_TYPE:
- return createValueSetter(((DistinctType) elementType).getSourceType());
+ writer.writeRow(
+ pos, (InternalRow) value, (RowDataSerializer) rowSerializer);
default:
throw new IllegalArgumentException();
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DataGetters.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DataGetters.java
new file mode 100644
index 00000000..71136f33
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DataGetters.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+/** Getters to get data. */
+public interface DataGetters {
+
+ /** Returns true if the element is null at the given position. */
+ boolean isNullAt(int pos);
+
+ /** Returns the boolean value at the given position. */
+ boolean getBoolean(int pos);
+
+ /** Returns the byte value at the given position. */
+ byte getByte(int pos);
+
+ /** Returns the short value at the given position. */
+ short getShort(int pos);
+
+ /** Returns the integer value at the given position. */
+ int getInt(int pos);
+
+ /** Returns the long value at the given position. */
+ long getLong(int pos);
+
+ /** Returns the float value at the given position. */
+ float getFloat(int pos);
+
+ /** Returns the double value at the given position. */
+ double getDouble(int pos);
+
+ /** Returns the string value at the given position. */
+ BinaryString getString(int pos);
+
+ /**
+ * Returns the decimal value at the given position.
+ *
+ * <p>The precision and scale are required to determine whether the decimal value was stored in
+ * a compact representation (see {@link Decimal}).
+ */
+ Decimal getDecimal(int pos, int precision, int scale);
+
+ /**
+ * Returns the timestamp value at the given position.
+ *
+ * <p>The precision is required to determine whether the timestamp value was stored in a compact
+ * representation (see {@link Timestamp}).
+ */
+ Timestamp getTimestamp(int pos, int precision);
+
+ /** Returns the binary value at the given position. */
+ byte[] getBinary(int pos);
+
+ /** Returns the array value at the given position. */
+ InternalArray getArray(int pos);
+
+ /** Returns the map value at the given position. */
+ InternalMap getMap(int pos);
+
+ /**
+ * Returns the row value at the given position.
+ *
+ * <p>The number of fields is required to correctly extract the row.
+ */
+ InternalRow getRow(int pos, int numFields);
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DataSetters.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DataSetters.java
new file mode 100644
index 00000000..ec729814
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DataSetters.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+/**
+ * Provide type specialized setters to reduce if/else and eliminate box and unbox. This is mainly
+ * used on the binary format such as {@link BinaryRow}.
+ */
+public interface DataSetters {
+
+ void setNullAt(int pos);
+
+ void setBoolean(int pos, boolean value);
+
+ void setByte(int pos, byte value);
+
+ void setShort(int pos, short value);
+
+ void setInt(int pos, int value);
+
+ void setLong(int pos, long value);
+
+ void setFloat(int pos, float value);
+
+ void setDouble(int pos, double value);
+
+ /**
+ * Set the decimal column value.
+ *
+ * <p>Note: Precision is compact: can call {@link #setNullAt} when decimal is null. Precision is
+ * not compact: can not call {@link #setNullAt} when decimal is null, must call {@code
+ * setDecimal(pos, null, precision)} because we need update var-length-part.
+ */
+ void setDecimal(int pos, Decimal value, int precision);
+
+ /**
+ * Set Timestamp value.
+ *
+ * <p>Note: If precision is compact: can call {@link #setNullAt} when TimestampData value is
+ * null. Otherwise: can not call {@link #setNullAt} when TimestampData value is null, must call
+ * {@code setTimestamp(pos, null, precision)} because we need to update var-length-part.
+ */
+ void setTimestamp(int pos, Timestamp value, int precision);
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/Decimal.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/Decimal.java
new file mode 100644
index 00000000..648e269d
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/Decimal.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.types.DecimalType;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/**
+ * An internal data structure representing data of {@link DecimalType}.
+ *
+ * <p>This data structure is immutable and might store decimal values in a compact representation
+ * (as a long value) if values are small enough.
+ */
+public final class Decimal implements Comparable<Decimal> {
+
+ // member fields and static fields are package-visible,
+ // in order to be accessible for DecimalUtils
+
+ static final int MAX_COMPACT_PRECISION = 18;
+
+ /** Maximum number of decimal digits an Int can represent. (1e9 < Int.MaxValue < 1e10) */
+ static final int MAX_INT_DIGITS = 9;
+
+ /** Maximum number of decimal digits a Long can represent. (1e18 < Long.MaxValue < 1e19) */
+ static final int MAX_LONG_DIGITS = 18;
+
+ static final long[] POW10 = new long[MAX_COMPACT_PRECISION + 1];
+
+ static {
+ POW10[0] = 1;
+ for (int i = 1; i < POW10.length; i++) {
+ POW10[i] = 10 * POW10[i - 1];
+ }
+ }
+
+ // The semantics of the fields are as follows:
+ // - `precision` and `scale` represent the precision and scale of SQL decimal type
+ // - If `decimalVal` is set, it represents the whole decimal value
+ // - Otherwise, the decimal value is longVal/(10^scale).
+ //
+ // Note that the (precision, scale) must be correct.
+ // if precision > MAX_COMPACT_PRECISION,
+ // `decimalVal` represents the value. `longVal` is undefined
+ // otherwise, (longVal, scale) represents the value
+ // `decimalVal` may be set and cached
+
+ final int precision;
+ final int scale;
+
+ final long longVal;
+ BigDecimal decimalVal;
+
+ // this constructor does not perform any sanity check.
+ Decimal(int precision, int scale, long longVal, BigDecimal decimalVal) {
+ this.precision = precision;
+ this.scale = scale;
+ this.longVal = longVal;
+ this.decimalVal = decimalVal;
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Public Interfaces
+ // ------------------------------------------------------------------------------------------
+
+ /**
+ * Returns the <i>precision</i> of this {@link Decimal}.
+ *
+ * <p>The precision is the number of digits in the unscaled value.
+ */
+ public int precision() {
+ return precision;
+ }
+
+ /** Returns the <i>scale</i> of this {@link Decimal}. */
+ public int scale() {
+ return scale;
+ }
+
+ /** Converts this {@link Decimal} into an instance of {@link BigDecimal}. */
+ public BigDecimal toBigDecimal() {
+ BigDecimal bd = decimalVal;
+ if (bd == null) {
+ decimalVal = bd = BigDecimal.valueOf(longVal, scale);
+ }
+ return bd;
+ }
+
+ /**
+ * Returns a long describing the <i>unscaled value</i> of this {@link Decimal}.
+ *
+ * @throws ArithmeticException if this {@link Decimal} does not exactly fit in a long.
+ */
+ public long toUnscaledLong() {
+ if (isCompact()) {
+ return longVal;
+ } else {
+ return toBigDecimal().unscaledValue().longValueExact();
+ }
+ }
+
+ /**
+ * Returns a byte array describing the <i>unscaled value</i> of this {@link Decimal}.
+ *
+ * @return the unscaled byte array of this {@link Decimal}.
+ */
+ public byte[] toUnscaledBytes() {
+ return toBigDecimal().unscaledValue().toByteArray();
+ }
+
+ /** Returns whether the decimal value is small enough to be stored in a long. */
+ public boolean isCompact() {
+ return precision <= MAX_COMPACT_PRECISION;
+ }
+
+ /** Returns a copy of this {@link Decimal} object. */
+ public Decimal copy() {
+ return new Decimal(precision, scale, longVal, decimalVal);
+ }
+
+ @Override
+ public int hashCode() {
+ return toBigDecimal().hashCode();
+ }
+
+ @Override
+ public int compareTo(@Nonnull Decimal that) {
+ if (this.isCompact() && that.isCompact() && this.scale == that.scale) {
+ return Long.compare(this.longVal, that.longVal);
+ }
+ return this.toBigDecimal().compareTo(that.toBigDecimal());
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (!(o instanceof Decimal)) {
+ return false;
+ }
+ Decimal that = (Decimal) o;
+ return this.compareTo(that) == 0;
+ }
+
+ @Override
+ public String toString() {
+ return toBigDecimal().toPlainString();
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Constructor Utilities
+ // ------------------------------------------------------------------------------------------
+
+ /**
+ * Creates an instance of {@link Decimal} from a {@link BigDecimal} and the given precision and
+ * scale.
+ *
+ * <p>The returned decimal value may be rounded to have the desired scale. The precision will be
+ * checked. If the precision overflows, null will be returned.
+ */
+ public static @Nullable Decimal fromBigDecimal(BigDecimal bd, int precision, int scale) {
+ bd = bd.setScale(scale, RoundingMode.HALF_UP);
+ if (bd.precision() > precision) {
+ return null;
+ }
+
+ long longVal = -1;
+ if (precision <= MAX_COMPACT_PRECISION) {
+ longVal = bd.movePointRight(scale).longValueExact();
+ }
+ return new Decimal(precision, scale, longVal, bd);
+ }
+
+ /**
+ * Creates an instance of {@link Decimal} from an unscaled long value and the given precision
+ * and scale.
+ */
+ public static Decimal fromUnscaledLong(long unscaledLong, int precision, int scale) {
+ checkArgument(precision > 0 && precision <= MAX_LONG_DIGITS);
+ return new Decimal(precision, scale, unscaledLong, null);
+ }
+
+ /**
+ * Creates an instance of {@link Decimal} from an unscaled byte array value and the given
+ * precision and scale.
+ */
+ public static Decimal fromUnscaledBytes(byte[] unscaledBytes, int precision, int scale) {
+ BigDecimal bd = new BigDecimal(new BigInteger(unscaledBytes), scale);
+ return fromBigDecimal(bd, precision, scale);
+ }
+
+ /**
+ * Creates an instance of {@link Decimal} for a zero value with the given precision and scale.
+ *
+ * <p>The precision will be checked. If the precision overflows, null will be returned.
+ */
+ public static @Nullable Decimal zero(int precision, int scale) {
+ if (precision <= MAX_COMPACT_PRECISION) {
+ return new Decimal(precision, scale, 0, null);
+ } else {
+ return fromBigDecimal(BigDecimal.ZERO, precision, scale);
+ }
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Utilities
+ // ------------------------------------------------------------------------------------------
+
+ /** Returns whether the decimal value is small enough to be stored in a long. */
+ public static boolean isCompact(int precision) {
+ return precision <= MAX_COMPACT_PRECISION;
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DecimalDataSerializer.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DecimalSerializer.java
similarity index 68%
rename from flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DecimalDataSerializer.java
rename to flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DecimalSerializer.java
index ce5e921f..e382312f 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DecimalDataSerializer.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/DecimalSerializer.java
@@ -24,20 +24,19 @@ import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.table.data.DecimalData;
import java.io.IOException;
-/** Serializer for {@link DecimalData}. */
+/** Serializer for {@link Decimal}. */
@Internal
-public final class DecimalDataSerializer extends TypeSerializer<DecimalData> {
+public final class DecimalSerializer extends TypeSerializer<Decimal> {
private static final long serialVersionUID = 1L;
private final int precision;
private final int scale;
- public DecimalDataSerializer(int precision, int scale) {
+ public DecimalSerializer(int precision, int scale) {
this.precision = precision;
this.scale = scale;
}
@@ -48,17 +47,17 @@ public final class DecimalDataSerializer extends TypeSerializer<DecimalData> {
}
@Override
- public DecimalData createInstance() {
- return DecimalData.zero(precision, scale);
+ public Decimal createInstance() {
+ return Decimal.zero(precision, scale);
}
@Override
- public DecimalData copy(DecimalData from) {
+ public Decimal copy(Decimal from) {
return from.copy();
}
@Override
- public DecimalData copy(DecimalData from, DecimalData reuse) {
+ public Decimal copy(Decimal from, Decimal reuse) {
return copy(from);
}
@@ -68,8 +67,8 @@ public final class DecimalDataSerializer extends TypeSerializer<DecimalData> {
}
@Override
- public void serialize(DecimalData record, DataOutputView target) throws IOException {
- if (DecimalData.isCompact(precision)) {
+ public void serialize(Decimal record, DataOutputView target) throws IOException {
+ if (Decimal.isCompact(precision)) {
assert record.isCompact();
target.writeLong(record.toUnscaledLong());
} else {
@@ -80,26 +79,26 @@ public final class DecimalDataSerializer extends TypeSerializer<DecimalData> {
}
@Override
- public DecimalData deserialize(DataInputView source) throws IOException {
- if (DecimalData.isCompact(precision)) {
+ public Decimal deserialize(DataInputView source) throws IOException {
+ if (Decimal.isCompact(precision)) {
long longVal = source.readLong();
- return DecimalData.fromUnscaledLong(longVal, precision, scale);
+ return Decimal.fromUnscaledLong(longVal, precision, scale);
} else {
int length = source.readInt();
byte[] bytes = new byte[length];
source.readFully(bytes);
- return DecimalData.fromUnscaledBytes(bytes, precision, scale);
+ return Decimal.fromUnscaledBytes(bytes, precision, scale);
}
}
@Override
- public DecimalData deserialize(DecimalData record, DataInputView source) throws IOException {
+ public Decimal deserialize(Decimal record, DataInputView source) throws IOException {
return deserialize(source);
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
- if (DecimalData.isCompact(precision)) {
+ if (Decimal.isCompact(precision)) {
target.writeLong(source.readLong());
} else {
int len = source.readInt();
@@ -109,8 +108,8 @@ public final class DecimalDataSerializer extends TypeSerializer<DecimalData> {
}
@Override
- public DecimalDataSerializer duplicate() {
- return new DecimalDataSerializer(precision, scale);
+ public DecimalSerializer duplicate() {
+ return new DecimalSerializer(precision, scale);
}
@Override
@@ -122,7 +121,7 @@ public final class DecimalDataSerializer extends TypeSerializer<DecimalData> {
return false;
}
- DecimalDataSerializer that = (DecimalDataSerializer) o;
+ DecimalSerializer that = (DecimalSerializer) o;
return precision == that.precision && scale == that.scale;
}
@@ -135,13 +134,12 @@ public final class DecimalDataSerializer extends TypeSerializer<DecimalData> {
}
@Override
- public TypeSerializerSnapshot<DecimalData> snapshotConfiguration() {
+ public TypeSerializerSnapshot<Decimal> snapshotConfiguration() {
return new DecimalSerializerSnapshot(precision, scale);
}
- /** {@link TypeSerializerSnapshot} for {@link DecimalDataSerializer}. */
- public static final class DecimalSerializerSnapshot
- implements TypeSerializerSnapshot<DecimalData> {
+ /** {@link TypeSerializerSnapshot} for {@link DecimalSerializer}. */
+ public static final class DecimalSerializerSnapshot implements TypeSerializerSnapshot<Decimal> {
private static final int CURRENT_VERSION = 3;
@@ -177,20 +175,20 @@ public final class DecimalDataSerializer extends TypeSerializer<DecimalData> {
}
@Override
- public TypeSerializer<DecimalData> restoreSerializer() {
- return new DecimalDataSerializer(previousPrecision, previousScale);
+ public TypeSerializer<Decimal> restoreSerializer() {
+ return new DecimalSerializer(previousPrecision, previousScale);
}
@Override
- public TypeSerializerSchemaCompatibility<DecimalData> resolveSchemaCompatibility(
- TypeSerializer<DecimalData> newSerializer) {
- if (!(newSerializer instanceof DecimalDataSerializer)) {
+ public TypeSerializerSchemaCompatibility<Decimal> resolveSchemaCompatibility(
+ TypeSerializer<Decimal> newSerializer) {
+ if (!(newSerializer instanceof DecimalSerializer)) {
return TypeSerializerSchemaCompatibility.incompatible();
}
- DecimalDataSerializer newDecimalDataSerializer = (DecimalDataSerializer) newSerializer;
- if (previousPrecision != newDecimalDataSerializer.precision
- || previousScale != newDecimalDataSerializer.scale) {
+ DecimalSerializer newDecimalSerializer = (DecimalSerializer) newSerializer;
+ if (previousPrecision != newDecimalSerializer.precision
+ || previousScale != newDecimalSerializer.scale) {
return TypeSerializerSchemaCompatibility.incompatible();
} else {
return TypeSerializerSchemaCompatibility.compatibleAsIs();
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/GenericArray.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/GenericArray.java
new file mode 100644
index 00000000..ecb42bd0
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/GenericArray.java
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.types.ArrayType;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * An internal data structure representing data of {@link ArrayType}.
+ *
+ * <p>Note: All elements of this data structure must be internal data structures and must be of the
+ * same type. See {@link InternalRow} for more information about internal data structures.
+ *
+ * <p>{@link GenericArray} is a generic implementation of {@link InternalArray} which wraps regular
+ * Java arrays.
+ */
+public final class GenericArray implements InternalArray {
+
+ private final Object array;
+ private final int size;
+ private final boolean isPrimitiveArray;
+
+ /**
+ * Creates an instance of {@link GenericArray} using the given Java array.
+ *
+ * <p>Note: All elements of the array must be internal data structures.
+ */
+ public GenericArray(Object[] array) {
+ this(array, array.length, false);
+ }
+
+ public GenericArray(int[] primitiveArray) {
+ this(primitiveArray, primitiveArray.length, true);
+ }
+
+ public GenericArray(long[] primitiveArray) {
+ this(primitiveArray, primitiveArray.length, true);
+ }
+
+ public GenericArray(float[] primitiveArray) {
+ this(primitiveArray, primitiveArray.length, true);
+ }
+
+ public GenericArray(double[] primitiveArray) {
+ this(primitiveArray, primitiveArray.length, true);
+ }
+
+ public GenericArray(short[] primitiveArray) {
+ this(primitiveArray, primitiveArray.length, true);
+ }
+
+ public GenericArray(byte[] primitiveArray) {
+ this(primitiveArray, primitiveArray.length, true);
+ }
+
+ public GenericArray(boolean[] primitiveArray) {
+ this(primitiveArray, primitiveArray.length, true);
+ }
+
+ private GenericArray(Object array, int size, boolean isPrimitiveArray) {
+ this.array = array;
+ this.size = size;
+ this.isPrimitiveArray = isPrimitiveArray;
+ }
+
+ /**
+ * Returns true if this is a primitive array.
+ *
+ * <p>A primitive array is an array whose elements are of primitive type.
+ */
+ public boolean isPrimitiveArray() {
+ return isPrimitiveArray;
+ }
+
+ /**
+ * Converts this {@link GenericArray} into an array of Java {@link Object}.
+ *
+ * <p>The method will convert a primitive array into an object array. But it will not convert
+ * internal data structures into external data structures (e.g. {@link BinaryString} to {@link
+ * String}).
+ */
+ public Object[] toObjectArray() {
+ if (isPrimitiveArray) {
+ Class<?> arrayClass = array.getClass();
+ if (int[].class.equals(arrayClass)) {
+ return ArrayUtils.toObject((int[]) array);
+ } else if (long[].class.equals(arrayClass)) {
+ return ArrayUtils.toObject((long[]) array);
+ } else if (float[].class.equals(arrayClass)) {
+ return ArrayUtils.toObject((float[]) array);
+ } else if (double[].class.equals(arrayClass)) {
+ return ArrayUtils.toObject((double[]) array);
+ } else if (short[].class.equals(arrayClass)) {
+ return ArrayUtils.toObject((short[]) array);
+ } else if (byte[].class.equals(arrayClass)) {
+ return ArrayUtils.toObject((byte[]) array);
+ } else if (boolean[].class.equals(arrayClass)) {
+ return ArrayUtils.toObject((boolean[]) array);
+ }
+ throw new RuntimeException("Unsupported primitive array: " + arrayClass);
+ } else {
+ return (Object[]) array;
+ }
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ return !isPrimitiveArray && ((Object[]) array)[pos] == null;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ GenericArray that = (GenericArray) o;
+ return size == that.size
+ && isPrimitiveArray == that.isPrimitiveArray
+ && Objects.deepEquals(array, that.array);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hash(size, isPrimitiveArray);
+ result = 31 * result + Arrays.deepHashCode(new Object[] {array});
+ return result;
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Read-only accessor methods
+ // ------------------------------------------------------------------------------------------
+
+ @Override
+ public boolean getBoolean(int pos) {
+ return isPrimitiveArray ? ((boolean[]) array)[pos] : (boolean) getObject(pos);
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ return isPrimitiveArray ? ((byte[]) array)[pos] : (byte) getObject(pos);
+ }
+
+ @Override
+ public short getShort(int pos) {
+ return isPrimitiveArray ? ((short[]) array)[pos] : (short) getObject(pos);
+ }
+
+ @Override
+ public int getInt(int pos) {
+ return isPrimitiveArray ? ((int[]) array)[pos] : (int) getObject(pos);
+ }
+
+ @Override
+ public long getLong(int pos) {
+ return isPrimitiveArray ? ((long[]) array)[pos] : (long) getObject(pos);
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ return isPrimitiveArray ? ((float[]) array)[pos] : (float) getObject(pos);
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ return isPrimitiveArray ? ((double[]) array)[pos] : (double) getObject(pos);
+ }
+
+ @Override
+ public byte[] getBinary(int pos) {
+ return (byte[]) getObject(pos);
+ }
+
+ @Override
+ public BinaryString getString(int pos) {
+ return (BinaryString) getObject(pos);
+ }
+
+ @Override
+ public Decimal getDecimal(int pos, int precision, int scale) {
+ return (Decimal) getObject(pos);
+ }
+
+ @Override
+ public Timestamp getTimestamp(int pos, int precision) {
+ return (Timestamp) getObject(pos);
+ }
+
+ @Override
+ public InternalRow getRow(int pos, int numFields) {
+ return (InternalRow) getObject(pos);
+ }
+
+ @Override
+ public InternalArray getArray(int pos) {
+ return (InternalArray) getObject(pos);
+ }
+
+ @Override
+ public InternalMap getMap(int pos) {
+ return (InternalMap) getObject(pos);
+ }
+
+ private Object getObject(int pos) {
+ return ((Object[]) array)[pos];
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Conversion Utilities
+ // ------------------------------------------------------------------------------------------
+
+ private boolean anyNull() {
+ for (Object element : (Object[]) array) {
+ if (element == null) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void checkNoNull() {
+ if (anyNull()) {
+ throw new RuntimeException("Primitive array must not contain a null value.");
+ }
+ }
+
+ @Override
+ public boolean[] toBooleanArray() {
+ if (isPrimitiveArray) {
+ return (boolean[]) array;
+ }
+ checkNoNull();
+ return ArrayUtils.toPrimitive((Boolean[]) array);
+ }
+
+ @Override
+ public byte[] toByteArray() {
+ if (isPrimitiveArray) {
+ return (byte[]) array;
+ }
+ checkNoNull();
+ return ArrayUtils.toPrimitive((Byte[]) array);
+ }
+
+ @Override
+ public short[] toShortArray() {
+ if (isPrimitiveArray) {
+ return (short[]) array;
+ }
+ checkNoNull();
+ return ArrayUtils.toPrimitive((Short[]) array);
+ }
+
+ @Override
+ public int[] toIntArray() {
+ if (isPrimitiveArray) {
+ return (int[]) array;
+ }
+ checkNoNull();
+ return ArrayUtils.toPrimitive((Integer[]) array);
+ }
+
+ @Override
+ public long[] toLongArray() {
+ if (isPrimitiveArray) {
+ return (long[]) array;
+ }
+ checkNoNull();
+ return ArrayUtils.toPrimitive((Long[]) array);
+ }
+
+ @Override
+ public float[] toFloatArray() {
+ if (isPrimitiveArray) {
+ return (float[]) array;
+ }
+ checkNoNull();
+ return ArrayUtils.toPrimitive((Float[]) array);
+ }
+
+ @Override
+ public double[] toDoubleArray() {
+ if (isPrimitiveArray) {
+ return (double[]) array;
+ }
+ checkNoNull();
+ return ArrayUtils.toPrimitive((Double[]) array);
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/GenericMap.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/GenericMap.java
new file mode 100644
index 00000000..eab09b9e
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/GenericMap.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.types.MapType;
+import org.apache.flink.table.store.types.MultisetType;
+
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * An internal data structure representing data of {@link MapType} or {@link MultisetType}.
+ *
+ * <p>{@link GenericMap} is a generic implementation of {@link InternalMap} which wraps regular Java
+ * maps.
+ *
+ * <p>Note: All keys and values of this data structure must be internal data structures. All keys
+ * must be of the same type; same for values. See {@link InternalRow} for more information about
+ * internal data structures.
+ *
+ * <p>Both keys and values can contain null for representing nullability.
+ */
+public final class GenericMap implements InternalMap {
+
+ private final Map<?, ?> map;
+
+ /**
+ * Creates an instance of {@link GenericMap} using the given Java map.
+ *
+ * <p>Note: All keys and values of the map must be internal data structures.
+ */
+ public GenericMap(Map<?, ?> map) {
+ this.map = map;
+ }
+
+ /**
+ * Returns the value to which the specified key is mapped, or {@code null} if this map contains
+ * no mapping for the key. The returned value is in internal data structure.
+ */
+ public Object get(Object key) {
+ return map.get(key);
+ }
+
+ @Override
+ public int size() {
+ return map.size();
+ }
+
+ @Override
+ public InternalArray keyArray() {
+ Object[] keys = map.keySet().toArray();
+ return new GenericArray(keys);
+ }
+
+ @Override
+ public InternalArray valueArray() {
+ Object[] values = map.values().toArray();
+ return new GenericArray(values);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (!(o instanceof GenericMap)) {
+ return false;
+ }
+ // deepEquals for values of byte[]
+ return deepEquals(map, ((GenericMap) o).map);
+ }
+
+ private static <K, V> boolean deepEquals(Map<K, V> m1, Map<?, ?> m2) {
+ // copied from HashMap.equals but with deepEquals comparison
+ if (m1.size() != m2.size()) {
+ return false;
+ }
+ try {
+ for (Map.Entry<K, V> e : m1.entrySet()) {
+ K key = e.getKey();
+ V value = e.getValue();
+ if (value == null) {
+ if (!(m2.get(key) == null && m2.containsKey(key))) {
+ return false;
+ }
+ } else {
+ if (!Objects.deepEquals(value, m2.get(key))) {
+ return false;
+ }
+ }
+ }
+ } catch (ClassCastException | NullPointerException unused) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 0;
+ for (Object key : map.keySet()) {
+ // only include key because values can contain byte[]
+ result += 31 * Objects.hashCode(key);
+ }
+ return result;
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/GenericRow.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/GenericRow.java
new file mode 100644
index 00000000..6fecfc5d
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/GenericRow.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.types.RowKind;
+import org.apache.flink.table.store.types.RowType;
+
+import java.util.Arrays;
+import java.util.Objects;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * An internal data structure representing data of {@link RowType}.
+ *
+ * <p>{@link GenericRow} is a generic implementation of {@link InternalRow} which is backed by an
+ * array of Java {@link Object}. A {@link GenericRow} can have an arbitrary number of fields of
+ * different types. The fields in a row can be accessed by position (0-based) using either the
+ * generic {@link #getField(int)} or type-specific getters (such as {@link #getInt(int)}). A field
+ * can be updated by the generic {@link #setField(int, Object)}.
+ *
+ * <p>Note: All fields of this data structure must be internal data structures. See {@link
+ * InternalRow} for more information about internal data structures.
+ *
+ * <p>The fields in {@link GenericRow} can be null for representing nullability.
+ */
+public final class GenericRow implements InternalRow {
+
+ /** The array to store the actual internal format values. */
+ private final Object[] fields;
+
+ /** The kind of change that a row describes in a changelog. */
+ private RowKind kind;
+
+ /**
+ * Creates an instance of {@link GenericRow} with given kind and number of fields.
+ *
+ * <p>Initially, all fields are set to null.
+ *
+ * <p>Note: All fields of the row must be internal data structures.
+ *
+ * @param kind kind of change that this row describes in a changelog
+ * @param arity number of fields
+ */
+ public GenericRow(RowKind kind, int arity) {
+ this.fields = new Object[arity];
+ this.kind = kind;
+ }
+
+ /**
+ * Creates an instance of {@link GenericRow} with given number of fields.
+ *
+ * <p>Initially, all fields are set to null. By default, the row describes a {@link
+ * RowKind#INSERT} in a changelog.
+ *
+ * <p>Note: All fields of the row must be internal data structures.
+ *
+ * @param arity number of fields
+ */
+ public GenericRow(int arity) {
+ this.fields = new Object[arity];
+ this.kind = RowKind.INSERT; // INSERT as default
+ }
+
+ /**
+ * Sets the field value at the given position.
+ *
+ * <p>Note: The given field value must be an internal data structures. Otherwise the {@link
+ * GenericRow} is corrupted and may throw exception when processing. See {@link InternalRow} for
+ * more information about internal data structures.
+ *
+ * <p>The field value can be null for representing nullability.
+ */
+ public void setField(int pos, Object value) {
+ this.fields[pos] = value;
+ }
+
+ /**
+ * Returns the field value at the given position.
+ *
+ * <p>Note: The returned value is in internal data structure. See {@link InternalRow} for more
+ * information about internal data structures.
+ *
+ * <p>The returned field value can be null for representing nullability.
+ */
+ public Object getField(int pos) {
+ return this.fields[pos];
+ }
+
+ @Override
+ public int getFieldCount() {
+ return fields.length;
+ }
+
+ @Override
+ public RowKind getRowKind() {
+ return kind;
+ }
+
+ @Override
+ public void setRowKind(RowKind kind) {
+ checkNotNull(kind);
+ this.kind = kind;
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ return this.fields[pos] == null;
+ }
+
+ @Override
+ public boolean getBoolean(int pos) {
+ return (boolean) this.fields[pos];
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ return (byte) this.fields[pos];
+ }
+
+ @Override
+ public short getShort(int pos) {
+ return (short) this.fields[pos];
+ }
+
+ @Override
+ public int getInt(int pos) {
+ return (int) this.fields[pos];
+ }
+
+ @Override
+ public long getLong(int pos) {
+ return (long) this.fields[pos];
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ return (float) this.fields[pos];
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ return (double) this.fields[pos];
+ }
+
+ @Override
+ public BinaryString getString(int pos) {
+ return (BinaryString) this.fields[pos];
+ }
+
+ @Override
+ public Decimal getDecimal(int pos, int precision, int scale) {
+ return (Decimal) this.fields[pos];
+ }
+
+ @Override
+ public Timestamp getTimestamp(int pos, int precision) {
+ return (Timestamp) this.fields[pos];
+ }
+
+ @Override
+ public byte[] getBinary(int pos) {
+ return (byte[]) this.fields[pos];
+ }
+
+ @Override
+ public InternalArray getArray(int pos) {
+ return (InternalArray) this.fields[pos];
+ }
+
+ @Override
+ public InternalMap getMap(int pos) {
+ return (InternalMap) this.fields[pos];
+ }
+
+ @Override
+ public InternalRow getRow(int pos, int numFields) {
+ return (InternalRow) this.fields[pos];
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof GenericRow)) {
+ return false;
+ }
+ GenericRow that = (GenericRow) o;
+ return kind == that.kind && Arrays.deepEquals(fields, that.fields);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hash(kind);
+ result = 31 * result + Arrays.deepHashCode(fields);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(kind.shortString()).append("(");
+ for (int i = 0; i < fields.length; i++) {
+ if (i != 0) {
+ sb.append(",");
+ }
+ sb.append(arrayAwareToString(fields[i]));
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+
+ private static String arrayAwareToString(Object o) {
+ final String arrayString = Arrays.deepToString(new Object[] {o});
+ return arrayString.substring(1, arrayString.length() - 1);
+ }
+
+ // ----------------------------------------------------------------------------------------
+ // Utilities
+ // ----------------------------------------------------------------------------------------
+
+ /**
+ * Creates an instance of {@link GenericRow} with given field values.
+ *
+ * <p>By default, the row describes a {@link RowKind#INSERT} in a changelog.
+ *
+ * <p>Note: All fields of the row must be internal data structures.
+ */
+ public static GenericRow of(Object... values) {
+ GenericRow row = new GenericRow(values.length);
+
+ for (int i = 0; i < values.length; ++i) {
+ row.setField(i, values[i]);
+ }
+
+ return row;
+ }
+
+ /**
+ * Creates an instance of {@link GenericRow} with given kind and field values.
+ *
+ * <p>Note: All fields of the row must be internal data structures.
+ */
+ public static GenericRow ofKind(RowKind kind, Object... values) {
+ GenericRow row = new GenericRow(kind, values.length);
+
+ for (int i = 0; i < values.length; ++i) {
+ row.setField(i, values[i]);
+ }
+
+ return row;
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalArray.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalArray.java
new file mode 100644
index 00000000..27e93984
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalArray.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.types.ArrayType;
+import org.apache.flink.table.store.types.DataType;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+
+import static org.apache.flink.table.store.types.DataTypeChecks.getFieldCount;
+import static org.apache.flink.table.store.types.DataTypeChecks.getPrecision;
+import static org.apache.flink.table.store.types.DataTypeChecks.getScale;
+
+/**
+ * Base interface of an internal data structure representing data of {@link ArrayType}.
+ *
+ * <p>Note: All elements of this data structure must be internal data structures and must be of the
+ * same type. See {@link InternalRow} for more information about internal data structures.
+ *
+ * <p>Use {@link GenericArray} to construct instances of this interface from regular Java arrays.
+ */
+public interface InternalArray extends DataGetters {
+
+ /** Returns the number of elements in this array. */
+ int size();
+
+ // ------------------------------------------------------------------------------------------
+ // Conversion Utilities
+ // ------------------------------------------------------------------------------------------
+
+ boolean[] toBooleanArray();
+
+ byte[] toByteArray();
+
+ short[] toShortArray();
+
+ int[] toIntArray();
+
+ long[] toLongArray();
+
+ float[] toFloatArray();
+
+ double[] toDoubleArray();
+
+ // ------------------------------------------------------------------------------------------
+ // Access Utilities
+ // ------------------------------------------------------------------------------------------
+
+ /**
+ * Creates an accessor for getting elements in an internal array data structure at the given
+ * position.
+ *
+ * @param elementType the element type of the array
+ */
+ static ElementGetter createElementGetter(DataType elementType) {
+ final ElementGetter elementGetter;
+ // ordered by type root definition
+ switch (elementType.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ elementGetter = InternalArray::getString;
+ break;
+ case BOOLEAN:
+ elementGetter = InternalArray::getBoolean;
+ break;
+ case BINARY:
+ case VARBINARY:
+ elementGetter = InternalArray::getBinary;
+ break;
+ case DECIMAL:
+ final int decimalPrecision = getPrecision(elementType);
+ final int decimalScale = getScale(elementType);
+ elementGetter =
+ (array, pos) -> array.getDecimal(pos, decimalPrecision, decimalScale);
+ break;
+ case TINYINT:
+ elementGetter = InternalArray::getByte;
+ break;
+ case SMALLINT:
+ elementGetter = InternalArray::getShort;
+ break;
+ case INTEGER:
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ elementGetter = InternalArray::getInt;
+ break;
+ case BIGINT:
+ elementGetter = InternalArray::getLong;
+ break;
+ case FLOAT:
+ elementGetter = InternalArray::getFloat;
+ break;
+ case DOUBLE:
+ elementGetter = InternalArray::getDouble;
+ break;
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ final int timestampPrecision = getPrecision(elementType);
+ elementGetter = (array, pos) -> array.getTimestamp(pos, timestampPrecision);
+ break;
+ case ARRAY:
+ elementGetter = InternalArray::getArray;
+ break;
+ case MULTISET:
+ case MAP:
+ elementGetter = InternalArray::getMap;
+ break;
+ case ROW:
+ final int rowFieldCount = getFieldCount(elementType);
+ elementGetter = (array, pos) -> array.getRow(pos, rowFieldCount);
+ break;
+ default:
+ throw new IllegalArgumentException();
+ }
+ if (!elementType.isNullable()) {
+ return elementGetter;
+ }
+ return (array, pos) -> {
+ if (array.isNullAt(pos)) {
+ return null;
+ }
+ return elementGetter.getElementOrNull(array, pos);
+ };
+ }
+
+ /**
+ * Accessor for getting the elements of an array during runtime.
+ *
+ * @see #createElementGetter(DataType)
+ */
+ interface ElementGetter extends Serializable {
+ @Nullable
+ Object getElementOrNull(InternalArray array, int pos);
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalMap.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalMap.java
new file mode 100644
index 00000000..c0b13413
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalMap.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.types.MapType;
+import org.apache.flink.table.store.types.MultisetType;
+
+/**
+ * Base interface of an internal data structure representing data of {@link MapType} or {@link
+ * MultisetType}.
+ *
+ * <p>Note: All keys and values of this data structure must be internal data structures. All keys
+ * must be of the same type; same for values. See {@link InternalRow} for more information about
+ * internal data structures.
+ *
+ * <p>Use {@link GenericMap} to construct instances of this interface from regular Java maps.
+ */
+public interface InternalMap {
+
+ /** Returns the number of key-value mappings in this map. */
+ int size();
+
+ /**
+ * Returns an array view of the keys contained in this map.
+ *
+ * <p>A key-value pair has the same index in the key array and value array.
+ */
+ InternalArray keyArray();
+
+ /**
+ * Returns an array view of the values contained in this map.
+ *
+ * <p>A key-value pair has the same index in the key array and value array.
+ */
+ InternalArray valueArray();
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalRow.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalRow.java
new file mode 100644
index 00000000..287643fb
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalRow.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.types.DataType;
+import org.apache.flink.table.store.types.DataTypeChecks;
+import org.apache.flink.table.store.types.RowKind;
+import org.apache.flink.table.store.types.RowType;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+
+import static org.apache.flink.table.store.types.DataTypeChecks.getPrecision;
+import static org.apache.flink.table.store.types.DataTypeChecks.getScale;
+
+/**
+ * Base interface for an internal data structure representing data of {@link RowType}.
+ *
+ * <p>The mappings from SQL data types to the internal data structures are listed in the following
+ * table:
+ *
+ * <pre>
+ * +--------------------------------+-----------------------------------------+
+ * | SQL Data Types | Internal Data Structures |
+ * +--------------------------------+-----------------------------------------+
+ * | BOOLEAN | boolean |
+ * +--------------------------------+-----------------------------------------+
+ * | CHAR / VARCHAR / STRING | {@link BinaryString} |
+ * +--------------------------------+-----------------------------------------+
+ * | BINARY / VARBINARY / BYTES | byte[] |
+ * +--------------------------------+-----------------------------------------+
+ * | DECIMAL | {@link Decimal} |
+ * +--------------------------------+-----------------------------------------+
+ * | TINYINT | byte |
+ * +--------------------------------+-----------------------------------------+
+ * | SMALLINT | short |
+ * +--------------------------------+-----------------------------------------+
+ * | INT | int |
+ * +--------------------------------+-----------------------------------------+
+ * | BIGINT | long |
+ * +--------------------------------+-----------------------------------------+
+ * | FLOAT | float |
+ * +--------------------------------+-----------------------------------------+
+ * | DOUBLE | double |
+ * +--------------------------------+-----------------------------------------+
+ * | DATE | int (number of days since epoch) |
+ * +--------------------------------+-----------------------------------------+
+ * | TIME | int (number of milliseconds of the day) |
+ * +--------------------------------+-----------------------------------------+
+ * | TIMESTAMP | {@link Timestamp} |
+ * +--------------------------------+-----------------------------------------+
+ * | TIMESTAMP WITH LOCAL TIME ZONE | {@link Timestamp} |
+ * +--------------------------------+-----------------------------------------+
+ * | ROW | {@link InternalRow} |
+ * +--------------------------------+-----------------------------------------+
+ * | ARRAY | {@link InternalArray} |
+ * +--------------------------------+-----------------------------------------+
+ * | MAP / MULTISET | {@link InternalMap} |
+ * +--------------------------------+-----------------------------------------+
+ * </pre>
+ *
+ * <p>Nullability is always handled by the container data structure.
+ */
+public interface InternalRow extends DataGetters {
+
+ /**
+ * Returns the number of fields in this row.
+ *
+ * <p>The number does not include {@link RowKind}. It is kept separately.
+ */
+ int getFieldCount();
+
+ /**
+ * Returns the kind of change that this row describes in a changelog.
+ *
+ * @see RowKind
+ */
+ RowKind getRowKind();
+
+ /**
+ * Sets the kind of change that this row describes in a changelog.
+ *
+ * @see RowKind
+ */
+ void setRowKind(RowKind kind);
+
+ // ------------------------------------------------------------------------------------------
+ // Access Utilities
+ // ------------------------------------------------------------------------------------------
+
+ /** Returns the data class for the given {@link DataType}. */
+ static Class<?> getDataClass(DataType type) {
+ // ordered by type root definition
+ switch (type.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ return BinaryString.class;
+ case BOOLEAN:
+ return Boolean.class;
+ case BINARY:
+ case VARBINARY:
+ return byte[].class;
+ case DECIMAL:
+ return Decimal.class;
+ case TINYINT:
+ return Byte.class;
+ case SMALLINT:
+ return Short.class;
+ case INTEGER:
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ return Integer.class;
+ case BIGINT:
+ return Long.class;
+ case FLOAT:
+ return Float.class;
+ case DOUBLE:
+ return Double.class;
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return Timestamp.class;
+ case ARRAY:
+ return InternalArray.class;
+ case MULTISET:
+ case MAP:
+ return InternalMap.class;
+ case ROW:
+ return InternalRow.class;
+ default:
+ throw new IllegalArgumentException("Illegal type: " + type);
+ }
+ }
+
+ /**
+ * Creates an accessor for getting elements in an internal row data structure at the given
+ * position.
+ *
+ * @param fieldType the element type of the row
+ * @param fieldPos the element position of the row
+ */
+ static FieldGetter createFieldGetter(DataType fieldType, int fieldPos) {
+ final FieldGetter fieldGetter;
+ // ordered by type root definition
+ switch (fieldType.getTypeRoot()) {
+ case CHAR:
+ case VARCHAR:
+ fieldGetter = row -> row.getString(fieldPos);
+ break;
+ case BOOLEAN:
+ fieldGetter = row -> row.getBoolean(fieldPos);
+ break;
+ case BINARY:
+ case VARBINARY:
+ fieldGetter = row -> row.getBinary(fieldPos);
+ break;
+ case DECIMAL:
+ final int decimalPrecision = getPrecision(fieldType);
+ final int decimalScale = getScale(fieldType);
+ fieldGetter = row -> row.getDecimal(fieldPos, decimalPrecision, decimalScale);
+ break;
+ case TINYINT:
+ fieldGetter = row -> row.getByte(fieldPos);
+ break;
+ case SMALLINT:
+ fieldGetter = row -> row.getShort(fieldPos);
+ break;
+ case INTEGER:
+ case DATE:
+ case TIME_WITHOUT_TIME_ZONE:
+ fieldGetter = row -> row.getInt(fieldPos);
+ break;
+ case BIGINT:
+ fieldGetter = row -> row.getLong(fieldPos);
+ break;
+ case FLOAT:
+ fieldGetter = row -> row.getFloat(fieldPos);
+ break;
+ case DOUBLE:
+ fieldGetter = row -> row.getDouble(fieldPos);
+ break;
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ final int timestampPrecision = getPrecision(fieldType);
+ fieldGetter = row -> row.getTimestamp(fieldPos, timestampPrecision);
+ break;
+ case ARRAY:
+ fieldGetter = row -> row.getArray(fieldPos);
+ break;
+ case MULTISET:
+ case MAP:
+ fieldGetter = row -> row.getMap(fieldPos);
+ break;
+ case ROW:
+ final int rowFieldCount = DataTypeChecks.getFieldCount(fieldType);
+ fieldGetter = row -> row.getRow(fieldPos, rowFieldCount);
+ break;
+ default:
+ throw new IllegalArgumentException();
+ }
+ if (!fieldType.isNullable()) {
+ return fieldGetter;
+ }
+ return row -> {
+ if (row.isNullAt(fieldPos)) {
+ return null;
+ }
+ return fieldGetter.getFieldOrNull(row);
+ };
+ }
+
+ /** Accessor for getting the field of a row during runtime. */
+ interface FieldGetter extends Serializable {
+ @Nullable
+ Object getFieldOrNull(InternalRow row);
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalSerializers.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalSerializers.java
index 0a8e2f98..23516962 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalSerializers.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/InternalSerializers.java
@@ -28,26 +28,26 @@ import org.apache.flink.api.common.typeutils.base.IntSerializer;
import org.apache.flink.api.common.typeutils.base.LongSerializer;
import org.apache.flink.api.common.typeutils.base.ShortSerializer;
import org.apache.flink.api.common.typeutils.base.array.BytePrimitiveArraySerializer;
-import org.apache.flink.table.types.logical.ArrayType;
-import org.apache.flink.table.types.logical.IntType;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.MapType;
-import org.apache.flink.table.types.logical.MultisetType;
-import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.store.types.ArrayType;
+import org.apache.flink.table.store.types.DataType;
+import org.apache.flink.table.store.types.IntType;
+import org.apache.flink.table.store.types.MapType;
+import org.apache.flink.table.store.types.MultisetType;
+import org.apache.flink.table.store.types.RowType;
-import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getPrecision;
-import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getScale;
+import static org.apache.flink.table.store.types.DataTypeChecks.getFieldTypes;
+import static org.apache.flink.table.store.types.DataTypeChecks.getPrecision;
+import static org.apache.flink.table.store.types.DataTypeChecks.getScale;
-/** {@link TypeSerializer} of {@link LogicalType} for internal data structures. */
+/** {@link TypeSerializer} of {@link DataType} for internal data structures. */
@Internal
public final class InternalSerializers {
/**
- * Creates a {@link TypeSerializer} for internal data structures of the given {@link
- * LogicalType}.
+ * Creates a {@link TypeSerializer} for internal data structures of the given {@link DataType}.
*/
@SuppressWarnings("unchecked")
- public static <T> TypeSerializer<T> create(LogicalType type) {
+ public static <T> TypeSerializer<T> create(DataType type) {
return (TypeSerializer<T>) createInternal(type);
}
@@ -58,19 +58,19 @@ public final class InternalSerializers {
return (RowDataSerializer) createInternal(type);
}
- private static TypeSerializer<?> createInternal(LogicalType type) {
+ private static TypeSerializer<?> createInternal(DataType type) {
// ordered by type root definition
switch (type.getTypeRoot()) {
case CHAR:
case VARCHAR:
- return StringDataSerializer.INSTANCE;
+ return BinaryStringSerializer.INSTANCE;
case BOOLEAN:
return BooleanSerializer.INSTANCE;
case BINARY:
case VARBINARY:
return BytePrimitiveArraySerializer.INSTANCE;
case DECIMAL:
- return new DecimalDataSerializer(getPrecision(type), getScale(type));
+ return new DecimalSerializer(getPrecision(type), getScale(type));
case TINYINT:
return ByteSerializer.INSTANCE;
case SMALLINT:
@@ -78,10 +78,8 @@ public final class InternalSerializers {
case INTEGER:
case DATE:
case TIME_WITHOUT_TIME_ZONE:
- case INTERVAL_YEAR_MONTH:
return IntSerializer.INSTANCE;
case BIGINT:
- case INTERVAL_DAY_TIME:
return LongSerializer.INSTANCE;
case FLOAT:
return FloatSerializer.INSTANCE;
@@ -89,9 +87,7 @@ public final class InternalSerializers {
return DoubleSerializer.INSTANCE;
case TIMESTAMP_WITHOUT_TIME_ZONE:
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- return new TimestampDataSerializer(getPrecision(type));
- case TIMESTAMP_WITH_TIME_ZONE:
- throw new UnsupportedOperationException();
+ return new TimestampSerializer(getPrecision(type));
case ARRAY:
return new ArrayDataSerializer(((ArrayType) type).getElementType());
case MULTISET:
@@ -101,8 +97,7 @@ public final class InternalSerializers {
MapType mapType = (MapType) type;
return new MapDataSerializer(mapType.getKeyType(), mapType.getValueType());
case ROW:
- case STRUCTURED_TYPE:
- return new RowDataSerializer(type.getChildren().toArray(new LogicalType[0]));
+ return new RowDataSerializer(getFieldTypes(type).toArray(new DataType[0]));
default:
throw new UnsupportedOperationException(
"Unsupported type '" + type + "' to get internal serializer");
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/JoinedRow.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/JoinedRow.java
new file mode 100644
index 00000000..d69806d1
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/JoinedRow.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.types.RowKind;
+
+import javax.annotation.Nullable;
+
+import java.util.Objects;
+
+/**
+ * An implementation of {@link InternalRow} which is backed by two concatenated {@link InternalRow}.
+ *
+ * <p>This implementation is mutable to allow for performant changes in hot code paths.
+ */
+public class JoinedRow implements InternalRow {
+
+ private RowKind rowKind = RowKind.INSERT;
+ private InternalRow row1;
+ private InternalRow row2;
+
+ /**
+ * Creates a new {@link JoinedRow} of kind {@link RowKind#INSERT}, but without backing rows.
+ *
+ * <p>Note that it must be ensured that the backing rows are set to non-{@code null} values
+ * before accessing data from this {@link JoinedRow}.
+ */
+ public JoinedRow() {}
+
+ /**
+ * Creates a new {@link JoinedRow} of kind {@link RowKind#INSERT} backed by {@param row1} and
+ * {@param row2}.
+ *
+ * <p>Note that it must be ensured that the backing rows are set to non-{@code null} values
+ * before accessing data from this {@link JoinedRow}.
+ */
+ public JoinedRow(@Nullable InternalRow row1, @Nullable InternalRow row2) {
+ this(RowKind.INSERT, row1, row2);
+ }
+
+ /**
+ * Creates a new {@link JoinedRow} of kind {@param rowKind} backed by {@param row1} and {@param
+ * row2}.
+ *
+ * <p>Note that it must be ensured that the backing rows are set to non-{@code null} values
+ * before accessing data from this {@link JoinedRow}.
+ */
+ public JoinedRow(RowKind rowKind, @Nullable InternalRow row1, @Nullable InternalRow row2) {
+ this.rowKind = rowKind;
+ this.row1 = row1;
+ this.row2 = row2;
+ }
+
+ /**
+ * Replaces the {@link InternalRow} backing this {@link JoinedRow}.
+ *
+ * <p>This method replaces the backing rows in place and does not return a new object. This is
+ * done for performance reasons.
+ */
+ public JoinedRow replace(InternalRow row1, InternalRow row2) {
+ this.row1 = row1;
+ this.row2 = row2;
+ return this;
+ }
+
+ // ---------------------------------------------------------------------------------------------
+
+ @Override
+ public int getFieldCount() {
+ return row1.getFieldCount() + row2.getFieldCount();
+ }
+
+ @Override
+ public RowKind getRowKind() {
+ return rowKind;
+ }
+
+ @Override
+ public void setRowKind(RowKind kind) {
+ this.rowKind = kind;
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.isNullAt(pos);
+ } else {
+ return row2.isNullAt(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public boolean getBoolean(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getBoolean(pos);
+ } else {
+ return row2.getBoolean(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getByte(pos);
+ } else {
+ return row2.getByte(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public short getShort(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getShort(pos);
+ } else {
+ return row2.getShort(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public int getInt(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getInt(pos);
+ } else {
+ return row2.getInt(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public long getLong(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getLong(pos);
+ } else {
+ return row2.getLong(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getFloat(pos);
+ } else {
+ return row2.getFloat(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getDouble(pos);
+ } else {
+ return row2.getDouble(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public BinaryString getString(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getString(pos);
+ } else {
+ return row2.getString(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public Decimal getDecimal(int pos, int precision, int scale) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getDecimal(pos, precision, scale);
+ } else {
+ return row2.getDecimal(pos - row1.getFieldCount(), precision, scale);
+ }
+ }
+
+ @Override
+ public Timestamp getTimestamp(int pos, int precision) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getTimestamp(pos, precision);
+ } else {
+ return row2.getTimestamp(pos - row1.getFieldCount(), precision);
+ }
+ }
+
+ @Override
+ public byte[] getBinary(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getBinary(pos);
+ } else {
+ return row2.getBinary(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public InternalArray getArray(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getArray(pos);
+ } else {
+ return row2.getArray(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public InternalMap getMap(int pos) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getMap(pos);
+ } else {
+ return row2.getMap(pos - row1.getFieldCount());
+ }
+ }
+
+ @Override
+ public InternalRow getRow(int pos, int numFields) {
+ if (pos < row1.getFieldCount()) {
+ return row1.getRow(pos, numFields);
+ } else {
+ return row2.getRow(pos - row1.getFieldCount(), numFields);
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ JoinedRow that = (JoinedRow) o;
+ return Objects.equals(rowKind, that.rowKind)
+ && Objects.equals(this.row1, that.row1)
+ && Objects.equals(this.row2, that.row2);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(rowKind, row1, row2);
+ }
+
+ @Override
+ public String toString() {
+ return rowKind.shortString() + "{" + "row1=" + row1 + ", row2=" + row2 + '}';
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/MapDataSerializer.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/MapDataSerializer.java
index 979831c1..f25f1ccf 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/MapDataSerializer.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/MapDataSerializer.java
@@ -27,36 +27,32 @@ import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.core.memory.MemorySegmentFactory;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.binary.BinaryArrayData;
-import org.apache.flink.table.data.binary.BinaryMapData;
-import org.apache.flink.table.data.binary.BinarySegmentUtils;
-import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+import org.apache.flink.table.store.types.DataType;
import org.apache.flink.util.InstantiationUtil;
import java.io.IOException;
-/** Serializer for {@link MapData}. */
+/** Serializer for {@link InternalMap}. */
@Internal
-public class MapDataSerializer extends TypeSerializer<MapData> {
+public class MapDataSerializer extends TypeSerializer<InternalMap> {
- private final LogicalType keyType;
- private final LogicalType valueType;
+ private final DataType keyType;
+ private final DataType valueType;
private final TypeSerializer keySerializer;
private final TypeSerializer valueSerializer;
- private final ArrayData.ElementGetter keyGetter;
- private final ArrayData.ElementGetter valueGetter;
+ private final InternalArray.ElementGetter keyGetter;
+ private final InternalArray.ElementGetter valueGetter;
- private transient BinaryArrayData reuseKeyArray;
- private transient BinaryArrayData reuseValueArray;
+ private transient BinaryArray reuseKeyArray;
+ private transient BinaryArray reuseValueArray;
private transient BinaryArrayWriter reuseKeyWriter;
private transient BinaryArrayWriter reuseValueWriter;
- public MapDataSerializer(LogicalType keyType, LogicalType valueType) {
+ public MapDataSerializer(DataType keyType, DataType valueType) {
this(
keyType,
valueType,
@@ -65,8 +61,8 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
}
private MapDataSerializer(
- LogicalType keyType,
- LogicalType valueType,
+ DataType keyType,
+ DataType valueType,
TypeSerializer keySerializer,
TypeSerializer valueSerializer) {
this.keyType = keyType;
@@ -75,8 +71,8 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
- this.keyGetter = ArrayData.createElementGetter(keyType);
- this.valueGetter = ArrayData.createElementGetter(valueType);
+ this.keyGetter = InternalArray.createElementGetter(keyType);
+ this.valueGetter = InternalArray.createElementGetter(valueType);
}
@Override
@@ -85,14 +81,14 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
}
@Override
- public TypeSerializer<MapData> duplicate() {
+ public TypeSerializer<InternalMap> duplicate() {
return new MapDataSerializer(
keyType, valueType, keySerializer.duplicate(), valueSerializer.duplicate());
}
@Override
- public MapData createInstance() {
- return new BinaryMapData();
+ public InternalMap createInstance() {
+ return new BinaryMap();
}
/**
@@ -100,16 +96,16 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
* HashMap, problems maybe occur.
*/
@Override
- public MapData copy(MapData from) {
- if (from instanceof BinaryMapData) {
- return ((BinaryMapData) from).copy();
+ public InternalMap copy(InternalMap from) {
+ if (from instanceof BinaryMap) {
+ return ((BinaryMap) from).copy();
} else {
return toBinaryMap(from);
}
}
@Override
- public MapData copy(MapData from, MapData reuse) {
+ public InternalMap copy(InternalMap from, InternalMap reuse) {
return copy(from);
}
@@ -119,31 +115,31 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
}
@Override
- public void serialize(MapData record, DataOutputView target) throws IOException {
- BinaryMapData binaryMap = toBinaryMap(record);
+ public void serialize(InternalMap record, DataOutputView target) throws IOException {
+ BinaryMap binaryMap = toBinaryMap(record);
target.writeInt(binaryMap.getSizeInBytes());
- BinarySegmentUtils.copyToView(
+ MemorySegmentUtils.copyToView(
binaryMap.getSegments(), binaryMap.getOffset(), binaryMap.getSizeInBytes(), target);
}
- public BinaryMapData toBinaryMap(MapData from) {
- if (from instanceof BinaryMapData) {
- return (BinaryMapData) from;
+ public BinaryMap toBinaryMap(InternalMap from) {
+ if (from instanceof BinaryMap) {
+ return (BinaryMap) from;
}
int numElements = from.size();
if (reuseKeyArray == null) {
- reuseKeyArray = new BinaryArrayData();
+ reuseKeyArray = new BinaryArray();
}
if (reuseValueArray == null) {
- reuseValueArray = new BinaryArrayData();
+ reuseValueArray = new BinaryArray();
}
if (reuseKeyWriter == null || reuseKeyWriter.getNumElements() != numElements) {
reuseKeyWriter =
new BinaryArrayWriter(
reuseKeyArray,
numElements,
- BinaryArrayData.calculateFixLengthPartSize(keyType));
+ BinaryArray.calculateFixLengthPartSize(keyType));
} else {
reuseKeyWriter.reset();
}
@@ -152,13 +148,13 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
new BinaryArrayWriter(
reuseValueArray,
numElements,
- BinaryArrayData.calculateFixLengthPartSize(valueType));
+ BinaryArray.calculateFixLengthPartSize(valueType));
} else {
reuseValueWriter.reset();
}
- ArrayData keyArray = from.keyArray();
- ArrayData valueArray = from.valueArray();
+ InternalArray keyArray = from.keyArray();
+ InternalArray valueArray = from.valueArray();
for (int i = 0; i < from.size(); i++) {
Object key = keyGetter.getElementOrNull(keyArray, i);
Object value = valueGetter.getElementOrNull(valueArray, i);
@@ -177,27 +173,25 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
reuseKeyWriter.complete();
reuseValueWriter.complete();
- return BinaryMapData.valueOf(reuseKeyArray, reuseValueArray);
+ return BinaryMap.valueOf(reuseKeyArray, reuseValueArray);
}
@Override
- public MapData deserialize(DataInputView source) throws IOException {
- return deserializeReuse(new BinaryMapData(), source);
+ public InternalMap deserialize(DataInputView source) throws IOException {
+ return deserializeReuse(new BinaryMap(), source);
}
@Override
- public MapData deserialize(MapData reuse, DataInputView source) throws IOException {
+ public InternalMap deserialize(InternalMap reuse, DataInputView source) throws IOException {
return deserializeReuse(
- reuse instanceof BinaryMapData ? (BinaryMapData) reuse : new BinaryMapData(),
- source);
+ reuse instanceof BinaryMap ? (BinaryMap) reuse : new BinaryMap(), source);
}
- private BinaryMapData deserializeReuse(BinaryMapData reuse, DataInputView source)
- throws IOException {
+ private BinaryMap deserializeReuse(BinaryMap reuse, DataInputView source) throws IOException {
int length = source.readInt();
byte[] bytes = new byte[length];
source.readFully(bytes);
- reuse.pointTo(MemorySegmentFactory.wrap(bytes), 0, bytes.length);
+ reuse.pointTo(MemorySegment.wrap(bytes), 0, bytes.length);
return reuse;
}
@@ -240,16 +234,17 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
}
@Override
- public TypeSerializerSnapshot<MapData> snapshotConfiguration() {
+ public TypeSerializerSnapshot<InternalMap> snapshotConfiguration() {
return new MapDataSerializerSnapshot(keyType, valueType, keySerializer, valueSerializer);
}
/** {@link TypeSerializerSnapshot} for {@link ArrayDataSerializer}. */
- public static final class MapDataSerializerSnapshot implements TypeSerializerSnapshot<MapData> {
+ public static final class MapDataSerializerSnapshot
+ implements TypeSerializerSnapshot<InternalMap> {
private static final int CURRENT_VERSION = 3;
- private LogicalType previousKeyType;
- private LogicalType previousValueType;
+ private DataType previousKeyType;
+ private DataType previousValueType;
private TypeSerializer previousKeySerializer;
private TypeSerializer previousValueSerializer;
@@ -260,10 +255,7 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
}
MapDataSerializerSnapshot(
- LogicalType keyT,
- LogicalType valueT,
- TypeSerializer keySer,
- TypeSerializer valueSer) {
+ DataType keyT, DataType valueT, TypeSerializer keySer, TypeSerializer valueSer) {
this.previousKeyType = keyT;
this.previousValueType = valueT;
@@ -304,7 +296,7 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
}
@Override
- public TypeSerializer<MapData> restoreSerializer() {
+ public TypeSerializer<InternalMap> restoreSerializer() {
return new MapDataSerializer(
previousKeyType,
previousValueType,
@@ -313,8 +305,8 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
}
@Override
- public TypeSerializerSchemaCompatibility<MapData> resolveSchemaCompatibility(
- TypeSerializer<MapData> newSerializer) {
+ public TypeSerializerSchemaCompatibility<InternalMap> resolveSchemaCompatibility(
+ TypeSerializer<InternalMap> newSerializer) {
if (!(newSerializer instanceof MapDataSerializer)) {
return TypeSerializerSchemaCompatibility.incompatible();
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/NestedRow.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/NestedRow.java
new file mode 100644
index 00000000..a0f1de4b
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/NestedRow.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentUtils;
+import org.apache.flink.table.store.types.RowKind;
+
+import static org.apache.flink.table.store.data.BinaryRow.calculateBitSetWidthInBytes;
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/**
+ * Its memory storage structure is exactly the same with {@link BinaryRow}. The only different is
+ * that, as {@link NestedRow} is used to store row value in the variable-length part of {@link
+ * BinaryRow}, every field (including both fixed-length part and variable-length part) of {@link
+ * NestedRow} has a possibility to cross the boundary of a segment, while the fixed-length part of
+ * {@link BinaryRow} must fit into its first memory segment.
+ */
+public final class NestedRow extends BinarySection implements InternalRow, DataSetters {
+
+ private final int arity;
+ private final int nullBitsSizeInBytes;
+
+ public NestedRow(int arity) {
+ checkArgument(arity >= 0);
+ this.arity = arity;
+ this.nullBitsSizeInBytes = calculateBitSetWidthInBytes(arity);
+ }
+
+ private int getFieldOffset(int pos) {
+ return offset + nullBitsSizeInBytes + pos * 8;
+ }
+
+ private void assertIndexIsValid(int index) {
+ assert index >= 0 : "index (" + index + ") should >= 0";
+ assert index < arity : "index (" + index + ") should < " + arity;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return arity;
+ }
+
+ @Override
+ public RowKind getRowKind() {
+ byte kindValue = MemorySegmentUtils.getByte(segments, offset);
+ return RowKind.fromByteValue(kindValue);
+ }
+
+ @Override
+ public void setRowKind(RowKind kind) {
+ MemorySegmentUtils.setByte(segments, offset, kind.toByteValue());
+ }
+
+ private void setNotNullAt(int i) {
+ assertIndexIsValid(i);
+ MemorySegmentUtils.bitUnSet(segments, offset, i + 8);
+ }
+
+ /** See {@link BinaryRow#setNullAt(int)}. */
+ @Override
+ public void setNullAt(int i) {
+ assertIndexIsValid(i);
+ MemorySegmentUtils.bitSet(segments, offset, i + 8);
+ MemorySegmentUtils.setLong(segments, getFieldOffset(i), 0);
+ }
+
+ @Override
+ public void setInt(int pos, int value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setInt(segments, getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setLong(int pos, long value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setLong(segments, getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setDouble(int pos, double value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setDouble(segments, getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setDecimal(int pos, Decimal value, int precision) {
+ assertIndexIsValid(pos);
+
+ if (Decimal.isCompact(precision)) {
+ // compact format
+ setLong(pos, value.toUnscaledLong());
+ } else {
+ int fieldOffset = getFieldOffset(pos);
+ int cursor = (int) (MemorySegmentUtils.getLong(segments, fieldOffset) >>> 32);
+ assert cursor > 0 : "invalid cursor " + cursor;
+ // zero-out the bytes
+ MemorySegmentUtils.setLong(segments, offset + cursor, 0L);
+ MemorySegmentUtils.setLong(segments, offset + cursor + 8, 0L);
+
+ if (value == null) {
+ setNullAt(pos);
+ // keep the offset for future update
+ MemorySegmentUtils.setLong(segments, fieldOffset, ((long) cursor) << 32);
+ } else {
+
+ byte[] bytes = value.toUnscaledBytes();
+ assert (bytes.length <= 16);
+
+ // Write the bytes to the variable length portion.
+ MemorySegmentUtils.copyFromBytes(segments, offset + cursor, bytes, 0, bytes.length);
+ setLong(pos, ((long) cursor << 32) | ((long) bytes.length));
+ }
+ }
+ }
+
+ @Override
+ public void setTimestamp(int pos, Timestamp value, int precision) {
+ assertIndexIsValid(pos);
+
+ if (Timestamp.isCompact(precision)) {
+ setLong(pos, value.getMillisecond());
+ } else {
+ int fieldOffset = getFieldOffset(pos);
+ int cursor = (int) (MemorySegmentUtils.getLong(segments, fieldOffset) >>> 32);
+ assert cursor > 0 : "invalid cursor " + cursor;
+
+ if (value == null) {
+ setNullAt(pos);
+ // zero-out the bytes
+ MemorySegmentUtils.setLong(segments, offset + cursor, 0L);
+ MemorySegmentUtils.setLong(segments, fieldOffset, ((long) cursor) << 32);
+ } else {
+ // write millisecond to variable length portion.
+ MemorySegmentUtils.setLong(segments, offset + cursor, value.getMillisecond());
+ // write nanoOfMillisecond to fixed-length portion.
+ setLong(pos, ((long) cursor << 32) | (long) value.getNanoOfMillisecond());
+ }
+ }
+ }
+
+ @Override
+ public void setBoolean(int pos, boolean value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setBoolean(segments, getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setShort(int pos, short value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setShort(segments, getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setByte(int pos, byte value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setByte(segments, getFieldOffset(pos), value);
+ }
+
+ @Override
+ public void setFloat(int pos, float value) {
+ assertIndexIsValid(pos);
+ setNotNullAt(pos);
+ MemorySegmentUtils.setFloat(segments, getFieldOffset(pos), value);
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.bitGet(segments, offset, pos + 8);
+ }
+
+ @Override
+ public boolean getBoolean(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getBoolean(segments, getFieldOffset(pos));
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getByte(segments, getFieldOffset(pos));
+ }
+
+ @Override
+ public short getShort(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getShort(segments, getFieldOffset(pos));
+ }
+
+ @Override
+ public int getInt(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getInt(segments, getFieldOffset(pos));
+ }
+
+ @Override
+ public long getLong(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getLong(segments, getFieldOffset(pos));
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getFloat(segments, getFieldOffset(pos));
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.getDouble(segments, getFieldOffset(pos));
+ }
+
+ @Override
+ public BinaryString getString(int pos) {
+ assertIndexIsValid(pos);
+ int fieldOffset = getFieldOffset(pos);
+ final long offsetAndLen = MemorySegmentUtils.getLong(segments, fieldOffset);
+ return MemorySegmentUtils.readBinaryString(segments, offset, fieldOffset, offsetAndLen);
+ }
+
+ @Override
+ public Decimal getDecimal(int pos, int precision, int scale) {
+ assertIndexIsValid(pos);
+
+ if (Decimal.isCompact(precision)) {
+ return Decimal.fromUnscaledLong(
+ MemorySegmentUtils.getLong(segments, getFieldOffset(pos)), precision, scale);
+ }
+
+ int fieldOffset = getFieldOffset(pos);
+ final long offsetAndSize = MemorySegmentUtils.getLong(segments, fieldOffset);
+ return MemorySegmentUtils.readDecimal(segments, offset, offsetAndSize, precision, scale);
+ }
+
+ @Override
+ public Timestamp getTimestamp(int pos, int precision) {
+ assertIndexIsValid(pos);
+
+ if (Timestamp.isCompact(precision)) {
+ return Timestamp.fromEpochMillis(
+ MemorySegmentUtils.getLong(segments, getFieldOffset(pos)));
+ }
+
+ int fieldOffset = getFieldOffset(pos);
+ final long offsetAndNanoOfMilli = MemorySegmentUtils.getLong(segments, fieldOffset);
+ return MemorySegmentUtils.readTimestampData(segments, offset, offsetAndNanoOfMilli);
+ }
+
+ @Override
+ public byte[] getBinary(int pos) {
+ assertIndexIsValid(pos);
+ int fieldOffset = getFieldOffset(pos);
+ final long offsetAndLen = MemorySegmentUtils.getLong(segments, fieldOffset);
+ return MemorySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndLen);
+ }
+
+ @Override
+ public InternalRow getRow(int pos, int numFields) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.readRowData(segments, numFields, offset, getLong(pos));
+ }
+
+ @Override
+ public InternalArray getArray(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.readArrayData(segments, offset, getLong(pos));
+ }
+
+ @Override
+ public InternalMap getMap(int pos) {
+ assertIndexIsValid(pos);
+ return MemorySegmentUtils.readMapData(segments, offset, getLong(pos));
+ }
+
+ public NestedRow copy() {
+ return copy(new NestedRow(arity));
+ }
+
+ public NestedRow copy(InternalRow reuse) {
+ return copyInternal((NestedRow) reuse);
+ }
+
+ private NestedRow copyInternal(NestedRow reuse) {
+ byte[] bytes = MemorySegmentUtils.copyToBytes(segments, offset, sizeInBytes);
+ reuse.pointTo(MemorySegment.wrap(bytes), 0, sizeInBytes);
+ return reuse;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ // both BinaryRow and NestedRow have the same memory format
+ if (!(o instanceof NestedRow || o instanceof BinaryRow)) {
+ return false;
+ }
+ final BinarySection that = (BinarySection) o;
+ return sizeInBytes == that.sizeInBytes
+ && MemorySegmentUtils.equals(
+ segments, offset, that.segments, that.offset, sizeInBytes);
+ }
+
+ @Override
+ public int hashCode() {
+ return MemorySegmentUtils.hashByWords(segments, offset, sizeInBytes);
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/PagedTypeSerializer.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/PagedTypeSerializer.java
index 1eaffcc7..6a808822 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/PagedTypeSerializer.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/PagedTypeSerializer.java
@@ -20,7 +20,6 @@ package org.apache.flink.table.store.data;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.table.data.binary.BinaryRowData;
import java.io.IOException;
@@ -32,7 +31,7 @@ public abstract class PagedTypeSerializer<T> extends TypeSerializer<T> {
/**
* Serializes the given record to the given target paged output view. Some implementations may
- * skip some bytes if current page does not have enough space left, .e.g {@link BinaryRowData}.
+ * skip some bytes if current page does not have enough space left, .e.g {@link BinaryRow}.
*
* @param record The record to serialize.
* @param target The output view to write the serialized data to.
@@ -47,7 +46,7 @@ public abstract class PagedTypeSerializer<T> extends TypeSerializer<T> {
/**
* De-serializes a record from the given source paged input view. For consistency with serialize
* format, some implementations may need to skip some bytes of source before de-serializing,
- * .e.g {@link BinaryRowData}. Typically, the content read from source should be copied out when
+ * .e.g {@link BinaryRow}. Typically, the content read from source should be copied out when
* de-serializing, and we are not expecting the underlying data from source is reused. If you
* have such requirement, see {@link #mapFromPages(T, AbstractPagedInputView)}.
*
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RandomAccessInputView.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RandomAccessInputView.java
index e8688e73..9880ba62 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RandomAccessInputView.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RandomAccessInputView.java
@@ -18,8 +18,8 @@
package org.apache.flink.table.store.data;
-import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.core.memory.SeekableDataInputView;
+import org.apache.flink.table.store.memory.MemorySegment;
import org.apache.flink.util.MathUtils;
import java.io.EOFException;
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RandomAccessOutputView.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RandomAccessOutputView.java
new file mode 100644
index 00000000..4e3bd605
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RandomAccessOutputView.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.memory.MemorySegment;
+
+import java.io.EOFException;
+
+/** A {@link AbstractPagedOutputView} with memory segments. */
+public class RandomAccessOutputView extends AbstractPagedOutputView {
+
+ private final MemorySegment[] segments;
+
+ private int currentSegmentIndex;
+
+ public RandomAccessOutputView(MemorySegment[] segments, int segmentSize) {
+ super(segments[0], segmentSize);
+
+ if ((segmentSize & (segmentSize - 1)) != 0) {
+ throw new IllegalArgumentException("Segment size must be a power of 2!");
+ }
+
+ this.segments = segments;
+ }
+
+ @Override
+ protected MemorySegment nextSegment(MemorySegment current, int positionInCurrent)
+ throws EOFException {
+ if (++this.currentSegmentIndex < this.segments.length) {
+ return this.segments[this.currentSegmentIndex];
+ } else {
+ throw new EOFException();
+ }
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RowDataSerializer.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RowDataSerializer.java
index b7911858..fb0d94f7 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RowDataSerializer.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/RowDataSerializer.java
@@ -28,40 +28,36 @@ import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.table.data.GenericRowData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.binary.BinaryRowData;
-import org.apache.flink.table.data.binary.NestedRowData;
-import org.apache.flink.table.types.logical.LogicalType;
-import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.store.types.DataType;
+import org.apache.flink.table.store.types.RowType;
import org.apache.flink.util.InstantiationUtil;
import java.io.IOException;
import java.util.Arrays;
import java.util.stream.IntStream;
-/** Serializer for {@link RowData}. */
+/** Serializer for {@link InternalRow}. */
@Internal
-public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
+public class RowDataSerializer extends AbstractRowDataSerializer<InternalRow> {
private static final long serialVersionUID = 1L;
- private final BinaryRowDataSerializer binarySerializer;
- private final LogicalType[] types;
+ private final BinaryRowSerializer binarySerializer;
+ private final DataType[] types;
private final TypeSerializer[] fieldSerializers;
- private final RowData.FieldGetter[] fieldGetters;
+ private final InternalRow.FieldGetter[] fieldGetters;
- private transient BinaryRowData reuseRow;
+ private transient BinaryRow reuseRow;
private transient BinaryRowWriter reuseWriter;
public RowDataSerializer(RowType rowType) {
this(
- rowType.getChildren().toArray(new LogicalType[0]),
- rowType.getChildren().stream()
+ rowType.getFieldTypes().toArray(new DataType[0]),
+ rowType.getFieldTypes().stream()
.map(InternalSerializers::create)
.toArray(TypeSerializer[]::new));
}
- public RowDataSerializer(LogicalType... types) {
+ public RowDataSerializer(DataType... types) {
this(
types,
Arrays.stream(types)
@@ -69,18 +65,18 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
.toArray(TypeSerializer[]::new));
}
- public RowDataSerializer(LogicalType[] types, TypeSerializer<?>[] fieldSerializers) {
+ public RowDataSerializer(DataType[] types, TypeSerializer<?>[] fieldSerializers) {
this.types = types;
this.fieldSerializers = fieldSerializers;
- this.binarySerializer = new BinaryRowDataSerializer(types.length);
+ this.binarySerializer = new BinaryRowSerializer(types.length);
this.fieldGetters =
IntStream.range(0, types.length)
- .mapToObj(i -> RowData.createFieldGetter(types[i], i))
- .toArray(RowData.FieldGetter[]::new);
+ .mapToObj(i -> InternalRow.createFieldGetter(types[i], i))
+ .toArray(InternalRow.FieldGetter[]::new);
}
@Override
- public TypeSerializer<RowData> duplicate() {
+ public TypeSerializer<InternalRow> duplicate() {
TypeSerializer<?>[] duplicateFieldSerializers = new TypeSerializer[fieldSerializers.length];
for (int i = 0; i < fieldSerializers.length; i++) {
duplicateFieldSerializers[i] = fieldSerializers[i].duplicate();
@@ -89,79 +85,82 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
}
@Override
- public RowData createInstance() {
+ public InternalRow createInstance() {
// default use binary row to deserializer
- return new BinaryRowData(types.length);
+ return new BinaryRow(types.length);
}
@Override
- public void serialize(RowData row, DataOutputView target) throws IOException {
+ public void serialize(InternalRow row, DataOutputView target) throws IOException {
binarySerializer.serialize(toBinaryRow(row), target);
}
@Override
- public RowData deserialize(DataInputView source) throws IOException {
+ public InternalRow deserialize(DataInputView source) throws IOException {
return binarySerializer.deserialize(source);
}
@Override
- public RowData deserialize(RowData reuse, DataInputView source) throws IOException {
- if (reuse instanceof BinaryRowData) {
- return binarySerializer.deserialize((BinaryRowData) reuse, source);
+ public InternalRow deserialize(InternalRow reuse, DataInputView source) throws IOException {
+ if (reuse instanceof BinaryRow) {
+ return binarySerializer.deserialize((BinaryRow) reuse, source);
} else {
return binarySerializer.deserialize(source);
}
}
@Override
- public RowData copy(RowData from) {
- if (from.getArity() != types.length) {
+ public InternalRow copy(InternalRow from) {
+ if (from.getFieldCount() != types.length) {
throw new IllegalArgumentException(
- "Row arity: " + from.getArity() + ", but serializer arity: " + types.length);
+ "Row arity: "
+ + from.getFieldCount()
+ + ", but serializer arity: "
+ + types.length);
}
- if (from instanceof BinaryRowData) {
- return ((BinaryRowData) from).copy();
- } else if (from instanceof NestedRowData) {
- return ((NestedRowData) from).copy();
+ if (from instanceof BinaryRow) {
+ return ((BinaryRow) from).copy();
+ } else if (from instanceof NestedRow) {
+ return ((NestedRow) from).copy();
} else {
- return copyRowData(from, new GenericRowData(from.getArity()));
+ return copyRowData(from, new GenericRow(from.getFieldCount()));
}
}
@Override
- public RowData copy(RowData from, RowData reuse) {
- if (from.getArity() != types.length || reuse.getArity() != types.length) {
+ public InternalRow copy(InternalRow from, InternalRow reuse) {
+ if (from.getFieldCount() != types.length || reuse.getFieldCount() != types.length) {
throw new IllegalArgumentException(
"Row arity: "
- + from.getArity()
+ + from.getFieldCount()
+ ", reuse Row arity: "
- + reuse.getArity()
+ + reuse.getFieldCount()
+ ", but serializer arity: "
+ types.length);
}
- if (from instanceof BinaryRowData) {
- return reuse instanceof BinaryRowData
- ? ((BinaryRowData) from).copy((BinaryRowData) reuse)
- : ((BinaryRowData) from).copy();
- } else if (from instanceof NestedRowData) {
- return reuse instanceof NestedRowData
- ? ((NestedRowData) from).copy(reuse)
- : ((NestedRowData) from).copy();
+ if (from instanceof BinaryRow) {
+ return reuse instanceof BinaryRow
+ ? ((BinaryRow) from).copy((BinaryRow) reuse)
+ : ((BinaryRow) from).copy();
+ } else if (from instanceof NestedRow) {
+ return reuse instanceof NestedRow
+ ? ((NestedRow) from).copy(reuse)
+ : ((NestedRow) from).copy();
} else {
return copyRowData(from, reuse);
}
}
@SuppressWarnings("unchecked")
- private RowData copyRowData(RowData from, RowData reuse) {
- GenericRowData ret;
- if (reuse instanceof GenericRowData) {
- ret = (GenericRowData) reuse;
+ private InternalRow copyRowData(InternalRow from, InternalRow reuse) {
+ GenericRow ret;
+ if (reuse instanceof GenericRow) {
+ ret = (GenericRow) reuse;
} else {
- ret = new GenericRowData(from.getArity());
+ ret = new GenericRow(from.getFieldCount());
}
ret.setRowKind(from.getRowKind());
- for (int i = 0; i < from.getArity(); i++) {
+ for (int i = 0; i < from.getFieldCount(); i++) {
if (!from.isNullAt(i)) {
ret.setField(i, fieldSerializers[i].copy((fieldGetters[i].getFieldOrNull(from))));
} else {
@@ -181,14 +180,14 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
return types.length;
}
- /** Convert {@link RowData} into {@link BinaryRowData}. TODO modify it to code gen. */
+ /** Convert {@link InternalRow} into {@link BinaryRow}. TODO modify it to code gen. */
@Override
- public BinaryRowData toBinaryRow(RowData row) {
- if (row instanceof BinaryRowData) {
- return (BinaryRowData) row;
+ public BinaryRow toBinaryRow(InternalRow row) {
+ if (row instanceof BinaryRow) {
+ return (BinaryRow) row;
}
if (reuseRow == null) {
- reuseRow = new BinaryRowData(types.length);
+ reuseRow = new BinaryRow(types.length);
reuseWriter = new BinaryRowWriter(reuseRow);
}
reuseWriter.reset();
@@ -210,25 +209,27 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
}
@Override
- public int serializeToPages(RowData row, AbstractPagedOutputView target) throws IOException {
+ public int serializeToPages(InternalRow row, AbstractPagedOutputView target)
+ throws IOException {
return binarySerializer.serializeToPages(toBinaryRow(row), target);
}
@Override
- public RowData deserializeFromPages(AbstractPagedInputView source) throws IOException {
+ public InternalRow deserializeFromPages(AbstractPagedInputView source) throws IOException {
throw new UnsupportedOperationException("Not support!");
}
@Override
- public RowData deserializeFromPages(RowData reuse, AbstractPagedInputView source)
+ public InternalRow deserializeFromPages(InternalRow reuse, AbstractPagedInputView source)
throws IOException {
throw new UnsupportedOperationException("Not support!");
}
@Override
- public RowData mapFromPages(RowData reuse, AbstractPagedInputView source) throws IOException {
- if (reuse instanceof BinaryRowData) {
- return binarySerializer.mapFromPages((BinaryRowData) reuse, source);
+ public InternalRow mapFromPages(InternalRow reuse, AbstractPagedInputView source)
+ throws IOException {
+ if (reuse instanceof BinaryRow) {
+ return binarySerializer.mapFromPages((BinaryRow) reuse, source);
} else {
throw new UnsupportedOperationException("Not support!");
}
@@ -265,15 +266,16 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
}
@Override
- public TypeSerializerSnapshot<RowData> snapshotConfiguration() {
+ public TypeSerializerSnapshot<InternalRow> snapshotConfiguration() {
return new RowDataSerializerSnapshot(types, fieldSerializers);
}
- /** {@link TypeSerializerSnapshot} for {@link BinaryRowDataSerializer}. */
- public static final class RowDataSerializerSnapshot implements TypeSerializerSnapshot<RowData> {
+ /** {@link TypeSerializerSnapshot} for {@link BinaryRowSerializer}. */
+ public static final class RowDataSerializerSnapshot
+ implements TypeSerializerSnapshot<InternalRow> {
private static final int CURRENT_VERSION = 3;
- private LogicalType[] previousTypes;
+ private DataType[] previousTypes;
private NestedSerializersSnapshotDelegate nestedSerializersSnapshotDelegate;
@SuppressWarnings("unused")
@@ -281,7 +283,7 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
// this constructor is used when restoring from a checkpoint/savepoint.
}
- RowDataSerializerSnapshot(LogicalType[] types, TypeSerializer[] serializers) {
+ RowDataSerializerSnapshot(DataType[] types, TypeSerializer[] serializers) {
this.previousTypes = types;
this.nestedSerializersSnapshotDelegate =
new NestedSerializersSnapshotDelegate(serializers);
@@ -296,7 +298,7 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
public void writeSnapshot(DataOutputView out) throws IOException {
out.writeInt(previousTypes.length);
DataOutputViewStream stream = new DataOutputViewStream(out);
- for (LogicalType previousType : previousTypes) {
+ for (DataType previousType : previousTypes) {
InstantiationUtil.serializeObject(stream, previousType);
}
nestedSerializersSnapshotDelegate.writeNestedSerializerSnapshots(out);
@@ -307,7 +309,7 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
throws IOException {
int length = in.readInt();
DataInputViewStream stream = new DataInputViewStream(in);
- previousTypes = new LogicalType[length];
+ previousTypes = new DataType[length];
for (int i = 0; i < length; i++) {
try {
previousTypes[i] =
@@ -329,8 +331,8 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
}
@Override
- public TypeSerializerSchemaCompatibility<RowData> resolveSchemaCompatibility(
- TypeSerializer<RowData> newSerializer) {
+ public TypeSerializerSchemaCompatibility<InternalRow> resolveSchemaCompatibility(
+ TypeSerializer<InternalRow> newSerializer) {
if (!(newSerializer instanceof RowDataSerializer)) {
return TypeSerializerSchemaCompatibility.incompatible();
}
@@ -340,7 +342,7 @@ public class RowDataSerializer extends AbstractRowDataSerializer<RowData> {
return TypeSerializerSchemaCompatibility.incompatible();
}
- CompositeTypeSerializerUtil.IntermediateCompatibilityResult<RowData>
+ CompositeTypeSerializerUtil.IntermediateCompatibilityResult<InternalRow>
intermediateResult =
CompositeTypeSerializerUtil.constructIntermediateCompatibilityResult(
newRowSerializer.fieldSerializers,
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/SimpleCollectingOutputView.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/SimpleCollectingOutputView.java
index a1be58f3..876ab121 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/SimpleCollectingOutputView.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/SimpleCollectingOutputView.java
@@ -18,8 +18,8 @@
package org.apache.flink.table.store.data;
-import org.apache.flink.core.memory.MemorySegment;
-import org.apache.flink.core.memory.MemorySegmentSource;
+import org.apache.flink.table.store.memory.MemorySegment;
+import org.apache.flink.table.store.memory.MemorySegmentSource;
import org.apache.flink.util.MathUtils;
import java.io.EOFException;
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/Timestamp.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/Timestamp.java
new file mode 100644
index 00000000..ddb849d2
--- /dev/null
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/Timestamp.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.data;
+
+import org.apache.flink.table.store.types.LocalZonedTimestampType;
+import org.apache.flink.table.store.types.TimestampType;
+import org.apache.flink.util.Preconditions;
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+
+/**
+ * An internal data structure representing data of {@link TimestampType} and {@link
+ * LocalZonedTimestampType}.
+ *
+ * <p>This data structure is immutable and consists of a milliseconds and nanos-of-millisecond since
+ * {@code 1970-01-01 00:00:00}. It might be stored in a compact representation (as a long value) if
+ * values are small enough.
+ */
+public final class Timestamp implements Comparable<Timestamp> {
+
+ // the number of milliseconds in a day
+ private static final long MILLIS_PER_DAY = 86400000; // = 24 * 60 * 60 * 1000
+
+ // this field holds the integral second and the milli-of-second
+ private final long millisecond;
+
+ // this field holds the nano-of-millisecond
+ private final int nanoOfMillisecond;
+
+ private Timestamp(long millisecond, int nanoOfMillisecond) {
+ Preconditions.checkArgument(nanoOfMillisecond >= 0 && nanoOfMillisecond <= 999_999);
+ this.millisecond = millisecond;
+ this.nanoOfMillisecond = nanoOfMillisecond;
+ }
+
+ /** Returns the number of milliseconds since {@code 1970-01-01 00:00:00}. */
+ public long getMillisecond() {
+ return millisecond;
+ }
+
+ /**
+ * Returns the number of nanoseconds (the nanoseconds within the milliseconds).
+ *
+ * <p>The value range is from 0 to 999,999.
+ */
+ public int getNanoOfMillisecond() {
+ return nanoOfMillisecond;
+ }
+
+ /** Converts this {@link Timestamp} object to a {@link java.sql.Timestamp}. */
+ public java.sql.Timestamp toSQLTimestamp() {
+ return java.sql.Timestamp.valueOf(toLocalDateTime());
+ }
+
+ /** Converts this {@link Timestamp} object to a {@link LocalDateTime}. */
+ public LocalDateTime toLocalDateTime() {
+ int date = (int) (millisecond / MILLIS_PER_DAY);
+ int time = (int) (millisecond % MILLIS_PER_DAY);
+ if (time < 0) {
+ --date;
+ time += MILLIS_PER_DAY;
+ }
+ long nanoOfDay = time * 1_000_000L + nanoOfMillisecond;
+ LocalDate localDate = LocalDate.ofEpochDay(date);
+ LocalTime localTime = LocalTime.ofNanoOfDay(nanoOfDay);
+ return LocalDateTime.of(localDate, localTime);
+ }
+
+ /** Converts this {@link Timestamp} object to a {@link Instant}. */
+ public Instant toInstant() {
+ long epochSecond = millisecond / 1000;
+ int milliOfSecond = (int) (millisecond % 1000);
+ if (milliOfSecond < 0) {
+ --epochSecond;
+ milliOfSecond += 1000;
+ }
+ long nanoAdjustment = milliOfSecond * 1_000_000 + nanoOfMillisecond;
+ return Instant.ofEpochSecond(epochSecond, nanoAdjustment);
+ }
+
+ @Override
+ public int compareTo(Timestamp that) {
+ int cmp = Long.compare(this.millisecond, that.millisecond);
+ if (cmp == 0) {
+ cmp = this.nanoOfMillisecond - that.nanoOfMillisecond;
+ }
+ return cmp;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Timestamp)) {
+ return false;
+ }
+ Timestamp that = (Timestamp) obj;
+ return this.millisecond == that.millisecond
+ && this.nanoOfMillisecond == that.nanoOfMillisecond;
+ }
+
+ @Override
+ public String toString() {
+ return toLocalDateTime().toString();
+ }
+
+ @Override
+ public int hashCode() {
+ int ret = (int) millisecond ^ (int) (millisecond >> 32);
+ return 31 * ret + nanoOfMillisecond;
+ }
+
+ // ------------------------------------------------------------------------------------------
+ // Constructor Utilities
+ // ------------------------------------------------------------------------------------------
+
+ /**
+ * Creates an instance of {@link Timestamp} from milliseconds.
+ *
+ * <p>The nanos-of-millisecond field will be set to zero.
+ *
+ * @param milliseconds the number of milliseconds since {@code 1970-01-01 00:00:00}; a negative
+ * number is the number of milliseconds before {@code 1970-01-01 00:00:00}
+ */
+ public static Timestamp fromEpochMillis(long milliseconds) {
+ return new Timestamp(milliseconds, 0);
+ }
+
+ /**
+ * Creates an instance of {@link Timestamp} from milliseconds and a nanos-of-millisecond.
+ *
+ * @param milliseconds the number of milliseconds since {@code 1970-01-01 00:00:00}; a negative
+ * number is the number of milliseconds before {@code 1970-01-01 00:00:00}
+ * @param nanosOfMillisecond the nanoseconds within the millisecond, from 0 to 999,999
+ */
+ public static Timestamp fromEpochMillis(long milliseconds, int nanosOfMillisecond) {
+ return new Timestamp(milliseconds, nanosOfMillisecond);
+ }
+
+ /**
+ * Creates an instance of {@link Timestamp} from an instance of {@link LocalDateTime}.
+ *
+ * @param dateTime an instance of {@link LocalDateTime}
+ */
+ public static Timestamp fromLocalDateTime(LocalDateTime dateTime) {
+ long epochDay = dateTime.toLocalDate().toEpochDay();
+ long nanoOfDay = dateTime.toLocalTime().toNanoOfDay();
+
+ long millisecond = epochDay * MILLIS_PER_DAY + nanoOfDay / 1_000_000;
+ int nanoOfMillisecond = (int) (nanoOfDay % 1_000_000);
+
+ return new Timestamp(millisecond, nanoOfMillisecond);
+ }
+
+ /**
+ * Creates an instance of {@link Timestamp} from an instance of {@link java.sql.Timestamp}.
+ *
+ * @param timestamp an instance of {@link java.sql.Timestamp}
+ */
+ public static Timestamp fromSQLTimestamp(java.sql.Timestamp timestamp) {
+ return fromLocalDateTime(timestamp.toLocalDateTime());
+ }
+
+ /**
+ * Creates an instance of {@link Timestamp} from an instance of {@link Instant}.
+ *
+ * @param instant an instance of {@link Instant}
+ */
+ public static Timestamp fromInstant(Instant instant) {
+ long epochSecond = instant.getEpochSecond();
+ int nanoSecond = instant.getNano();
+
+ long millisecond = epochSecond * 1_000 + nanoSecond / 1_000_000;
+ int nanoOfMillisecond = nanoSecond % 1_000_000;
+
+ return new Timestamp(millisecond, nanoOfMillisecond);
+ }
+
+ /**
+ * Returns whether the timestamp data is small enough to be stored in a long of milliseconds.
+ */
+ public static boolean isCompact(int precision) {
+ return precision <= 3;
+ }
+}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/TimestampDataSerializer.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/TimestampSerializer.java
similarity index 65%
rename from flink-table-store-common/src/main/java/org/apache/flink/table/store/data/TimestampDataSerializer.java
rename to flink-table-store-common/src/main/java/org/apache/flink/table/store/data/TimestampSerializer.java
index 526c7a3d..a9ed099e 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/TimestampDataSerializer.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/TimestampSerializer.java
@@ -24,24 +24,23 @@ import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.table.data.TimestampData;
import java.io.IOException;
/**
- * Serializer for {@link TimestampData}.
+ * Serializer for {@link Timestamp}.
*
- * <p>A {@link TimestampData} instance can be compactly serialized as a long value(= millisecond)
- * when the Timestamp type is compact. Otherwise it's serialized as a long value and a int value.
+ * <p>A {@link Timestamp} instance can be compactly serialized as a long value(= millisecond) when
+ * the Timestamp type is compact. Otherwise it's serialized as a long value and a int value.
*/
@Internal
-public class TimestampDataSerializer extends TypeSerializer<TimestampData> {
+public class TimestampSerializer extends TypeSerializer<Timestamp> {
private static final long serialVersionUID = 1L;
private final int precision;
- public TimestampDataSerializer(int precision) {
+ public TimestampSerializer(int precision) {
this.precision = precision;
}
@@ -51,33 +50,33 @@ public class TimestampDataSerializer extends TypeSerializer<TimestampData> {
}
@Override
- public TypeSerializer<TimestampData> duplicate() {
- return new TimestampDataSerializer(precision);
+ public TypeSerializer<Timestamp> duplicate() {
+ return new TimestampSerializer(precision);
}
@Override
- public TimestampData createInstance() {
- return TimestampData.fromEpochMillis(0);
+ public Timestamp createInstance() {
+ return Timestamp.fromEpochMillis(0);
}
@Override
- public TimestampData copy(TimestampData from) {
+ public Timestamp copy(Timestamp from) {
return from;
}
@Override
- public TimestampData copy(TimestampData from, TimestampData reuse) {
+ public Timestamp copy(Timestamp from, Timestamp reuse) {
return copy(from);
}
@Override
public int getLength() {
- return ((TimestampData.isCompact(precision)) ? 8 : 12);
+ return ((Timestamp.isCompact(precision)) ? 8 : 12);
}
@Override
- public void serialize(TimestampData record, DataOutputView target) throws IOException {
- if (TimestampData.isCompact(precision)) {
+ public void serialize(Timestamp record, DataOutputView target) throws IOException {
+ if (Timestamp.isCompact(precision)) {
assert record.getNanoOfMillisecond() == 0;
target.writeLong(record.getMillisecond());
} else {
@@ -87,25 +86,25 @@ public class TimestampDataSerializer extends TypeSerializer<TimestampData> {
}
@Override
- public TimestampData deserialize(DataInputView source) throws IOException {
- if (TimestampData.isCompact(precision)) {
+ public Timestamp deserialize(DataInputView source) throws IOException {
+ if (Timestamp.isCompact(precision)) {
long val = source.readLong();
- return TimestampData.fromEpochMillis(val);
+ return Timestamp.fromEpochMillis(val);
} else {
long longVal = source.readLong();
int intVal = source.readInt();
- return TimestampData.fromEpochMillis(longVal, intVal);
+ return Timestamp.fromEpochMillis(longVal, intVal);
}
}
@Override
- public TimestampData deserialize(TimestampData reuse, DataInputView source) throws IOException {
+ public Timestamp deserialize(Timestamp reuse, DataInputView source) throws IOException {
return deserialize(source);
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
- if (TimestampData.isCompact(precision)) {
+ if (Timestamp.isCompact(precision)) {
target.writeLong(source.readLong());
} else {
target.writeLong(source.readLong());
@@ -123,7 +122,7 @@ public class TimestampDataSerializer extends TypeSerializer<TimestampData> {
return false;
}
- TimestampDataSerializer that = (TimestampDataSerializer) obj;
+ TimestampSerializer that = (TimestampSerializer) obj;
return precision == that.precision;
}
@@ -133,13 +132,13 @@ public class TimestampDataSerializer extends TypeSerializer<TimestampData> {
}
@Override
- public TypeSerializerSnapshot<TimestampData> snapshotConfiguration() {
+ public TypeSerializerSnapshot<Timestamp> snapshotConfiguration() {
return new TimestampDataSerializerSnapshot(precision);
}
- /** {@link TypeSerializerSnapshot} for {@link TimestampDataSerializer}. */
+ /** {@link TypeSerializerSnapshot} for {@link TimestampSerializer}. */
public static final class TimestampDataSerializerSnapshot
- implements TypeSerializerSnapshot<TimestampData> {
+ implements TypeSerializerSnapshot<Timestamp> {
private static final int CURRENT_VERSION = 1;
@@ -170,20 +169,19 @@ public class TimestampDataSerializer extends TypeSerializer<TimestampData> {
}
@Override
- public TypeSerializer<TimestampData> restoreSerializer() {
- return new TimestampDataSerializer(previousPrecision);
+ public TypeSerializer<Timestamp> restoreSerializer() {
+ return new TimestampSerializer(previousPrecision);
}
@Override
- public TypeSerializerSchemaCompatibility<TimestampData> resolveSchemaCompatibility(
- TypeSerializer<TimestampData> newSerializer) {
- if (!(newSerializer instanceof TimestampDataSerializer)) {
+ public TypeSerializerSchemaCompatibility<Timestamp> resolveSchemaCompatibility(
+ TypeSerializer<Timestamp> newSerializer) {
+ if (!(newSerializer instanceof TimestampSerializer)) {
return TypeSerializerSchemaCompatibility.incompatible();
}
- TimestampDataSerializer timestampDataSerializer =
- (TimestampDataSerializer) newSerializer;
- if (previousPrecision != timestampDataSerializer.precision) {
+ TimestampSerializer timestampSerializer = (TimestampSerializer) newSerializer;
+ if (previousPrecision != timestampSerializer.precision) {
return TypeSerializerSchemaCompatibility.incompatible();
} else {
return TypeSerializerSchemaCompatibility.compatibleAsIs();
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ArrayColumnVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ArrayColumnVector.java
index 696c37a0..a3f7018d 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ArrayColumnVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ArrayColumnVector.java
@@ -18,11 +18,9 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.store.data.InternalArray;
/** Array column vector. */
-@Internal
public interface ArrayColumnVector extends ColumnVector {
- ArrayData getArray(int i);
+ InternalArray getArray(int i);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/BooleanColumnVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/BooleanColumnVector.java
index d1bedb41..5e823645 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/BooleanColumnVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/BooleanColumnVector.java
@@ -18,10 +18,7 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-
/** Boolean column vector. */
-@Internal
public interface BooleanColumnVector extends ColumnVector {
boolean getBoolean(int i);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarArrayData.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarArray.java
similarity index 82%
rename from flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarArrayData.java
rename to flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarArray.java
index 1d6e7261..1480f4f3 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarArrayData.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarArray.java
@@ -18,27 +18,24 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RawValueData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.data.binary.TypedSetters;
+import org.apache.flink.table.store.data.BinaryString;
+import org.apache.flink.table.store.data.DataSetters;
+import org.apache.flink.table.store.data.Decimal;
+import org.apache.flink.table.store.data.InternalArray;
+import org.apache.flink.table.store.data.InternalMap;
+import org.apache.flink.table.store.data.InternalRow;
+import org.apache.flink.table.store.data.Timestamp;
import java.util.Arrays;
/** Columnar array to support access to vector column data. */
-@Internal
-public final class ColumnarArrayData implements ArrayData, TypedSetters {
+public final class ColumnarArray implements InternalArray, DataSetters {
private final ColumnVector data;
private final int offset;
private final int numElements;
- public ColumnarArrayData(ColumnVector data, int offset, int numElements) {
+ public ColumnarArray(ColumnVector data, int offset, int numElements) {
this.data = data;
this.offset = offset;
this.numElements = numElements;
@@ -95,26 +92,21 @@ public final class ColumnarArrayData implements ArrayData, TypedSetters {
}
@Override
- public StringData getString(int pos) {
+ public BinaryString getString(int pos) {
BytesColumnVector.Bytes byteArray = getByteArray(pos);
- return StringData.fromBytes(byteArray.data, byteArray.offset, byteArray.len);
+ return BinaryString.fromBytes(byteArray.data, byteArray.offset, byteArray.len);
}
@Override
- public DecimalData getDecimal(int pos, int precision, int scale) {
+ public Decimal getDecimal(int pos, int precision, int scale) {
return ((DecimalColumnVector) data).getDecimal(offset + pos, precision, scale);
}
@Override
- public TimestampData getTimestamp(int pos, int precision) {
+ public Timestamp getTimestamp(int pos, int precision) {
return ((TimestampColumnVector) data).getTimestamp(offset + pos, precision);
}
- @Override
- public <T> RawValueData<T> getRawValue(int pos) {
- throw new UnsupportedOperationException("RawValueData is not supported.");
- }
-
@Override
public byte[] getBinary(int pos) {
BytesColumnVector.Bytes byteArray = getByteArray(pos);
@@ -126,17 +118,17 @@ public final class ColumnarArrayData implements ArrayData, TypedSetters {
}
@Override
- public ArrayData getArray(int pos) {
+ public InternalArray getArray(int pos) {
return ((ArrayColumnVector) data).getArray(offset + pos);
}
@Override
- public MapData getMap(int pos) {
+ public InternalMap getMap(int pos) {
return ((MapColumnVector) data).getMap(offset + pos);
}
@Override
- public RowData getRow(int pos, int numFields) {
+ public InternalRow getRow(int pos, int numFields) {
return ((RowColumnVector) data).getRow(offset + pos);
}
@@ -176,12 +168,12 @@ public final class ColumnarArrayData implements ArrayData, TypedSetters {
}
@Override
- public void setDecimal(int pos, DecimalData value, int precision) {
+ public void setDecimal(int pos, Decimal value, int precision) {
throw new UnsupportedOperationException("Not support the operation!");
}
@Override
- public void setTimestamp(int pos, TimestampData value, int precision) {
+ public void setTimestamp(int pos, Timestamp value, int precision) {
throw new UnsupportedOperationException("Not support the operation!");
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarMapData.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarMap.java
similarity index 80%
rename from flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarMapData.java
rename to flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarMap.java
index cd372601..68a4592e 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarMapData.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarMap.java
@@ -18,20 +18,18 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.store.data.InternalArray;
+import org.apache.flink.table.store.data.InternalMap;
/** Columnar map to support access to vector column data. */
-@Internal
-public final class ColumnarMapData implements MapData {
+public final class ColumnarMap implements InternalMap {
private final ColumnVector keyColumnVector;
private final ColumnVector valueColumnVector;
private final int offset;
private final int numElements;
- public ColumnarMapData(
+ public ColumnarMap(
ColumnVector keyColumnVector,
ColumnVector valueColumnVector,
int offset,
@@ -48,13 +46,13 @@ public final class ColumnarMapData implements MapData {
}
@Override
- public ArrayData keyArray() {
- return new ColumnarArrayData(keyColumnVector, offset, numElements);
+ public InternalArray keyArray() {
+ return new ColumnarArray(keyColumnVector, offset, numElements);
}
@Override
- public ArrayData valueArray() {
- return new ColumnarArrayData(valueColumnVector, offset, numElements);
+ public InternalArray valueArray() {
+ return new ColumnarArray(valueColumnVector, offset, numElements);
}
@Override
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarRowData.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarRow.java
similarity index 78%
rename from flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarRowData.java
rename to flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarRow.java
index e91b7c56..36d16d58 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarRowData.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarRow.java
@@ -18,36 +18,33 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RawValueData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.StringData;
-import org.apache.flink.table.data.TimestampData;
-import org.apache.flink.table.data.binary.TypedSetters;
+import org.apache.flink.table.store.data.BinaryString;
+import org.apache.flink.table.store.data.DataSetters;
+import org.apache.flink.table.store.data.Decimal;
+import org.apache.flink.table.store.data.InternalArray;
+import org.apache.flink.table.store.data.InternalMap;
+import org.apache.flink.table.store.data.InternalRow;
+import org.apache.flink.table.store.data.Timestamp;
import org.apache.flink.table.store.data.columnar.BytesColumnVector.Bytes;
-import org.apache.flink.types.RowKind;
+import org.apache.flink.table.store.types.RowKind;
/**
* Columnar row to support access to vector column data. It is a row view in {@link
* VectorizedColumnBatch}.
*/
-@Internal
-public final class ColumnarRowData implements RowData, TypedSetters {
+public final class ColumnarRow implements InternalRow, DataSetters {
private RowKind rowKind = RowKind.INSERT;
private VectorizedColumnBatch vectorizedColumnBatch;
private int rowId;
- public ColumnarRowData() {}
+ public ColumnarRow() {}
- public ColumnarRowData(VectorizedColumnBatch vectorizedColumnBatch) {
+ public ColumnarRow(VectorizedColumnBatch vectorizedColumnBatch) {
this(vectorizedColumnBatch, 0);
}
- public ColumnarRowData(VectorizedColumnBatch vectorizedColumnBatch, int rowId) {
+ public ColumnarRow(VectorizedColumnBatch vectorizedColumnBatch, int rowId) {
this.vectorizedColumnBatch = vectorizedColumnBatch;
this.rowId = rowId;
}
@@ -72,7 +69,7 @@ public final class ColumnarRowData implements RowData, TypedSetters {
}
@Override
- public int getArity() {
+ public int getFieldCount() {
return vectorizedColumnBatch.getArity();
}
@@ -117,26 +114,21 @@ public final class ColumnarRowData implements RowData, TypedSetters {
}
@Override
- public StringData getString(int pos) {
+ public BinaryString getString(int pos) {
Bytes byteArray = vectorizedColumnBatch.getByteArray(rowId, pos);
- return StringData.fromBytes(byteArray.data, byteArray.offset, byteArray.len);
+ return BinaryString.fromBytes(byteArray.data, byteArray.offset, byteArray.len);
}
@Override
- public DecimalData getDecimal(int pos, int precision, int scale) {
+ public Decimal getDecimal(int pos, int precision, int scale) {
return vectorizedColumnBatch.getDecimal(rowId, pos, precision, scale);
}
@Override
- public TimestampData getTimestamp(int pos, int precision) {
+ public Timestamp getTimestamp(int pos, int precision) {
return vectorizedColumnBatch.getTimestamp(rowId, pos, precision);
}
- @Override
- public <T> RawValueData<T> getRawValue(int pos) {
- throw new UnsupportedOperationException("RawValueData is not supported.");
- }
-
@Override
public byte[] getBinary(int pos) {
Bytes byteArray = vectorizedColumnBatch.getByteArray(rowId, pos);
@@ -150,17 +142,17 @@ public final class ColumnarRowData implements RowData, TypedSetters {
}
@Override
- public RowData getRow(int pos, int numFields) {
+ public InternalRow getRow(int pos, int numFields) {
return vectorizedColumnBatch.getRow(rowId, pos);
}
@Override
- public ArrayData getArray(int pos) {
+ public InternalArray getArray(int pos) {
return vectorizedColumnBatch.getArray(rowId, pos);
}
@Override
- public MapData getMap(int pos) {
+ public InternalMap getMap(int pos) {
return vectorizedColumnBatch.getMap(rowId, pos);
}
@@ -205,12 +197,12 @@ public final class ColumnarRowData implements RowData, TypedSetters {
}
@Override
- public void setDecimal(int pos, DecimalData value, int precision) {
+ public void setDecimal(int pos, Decimal value, int precision) {
throw new UnsupportedOperationException("Not support the operation!");
}
@Override
- public void setTimestamp(int pos, TimestampData value, int precision) {
+ public void setTimestamp(int pos, Timestamp value, int precision) {
throw new UnsupportedOperationException("Not support the operation!");
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarRowIterator.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarRowIterator.java
index 73fde99b..621539bc 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarRowIterator.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/ColumnarRowIterator.java
@@ -18,26 +18,24 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.store.data.InternalRow;
import org.apache.flink.table.store.file.utils.RecordReader;
import org.apache.flink.table.store.file.utils.RecyclableIterator;
import javax.annotation.Nullable;
/**
- * A {@link RecordReader.RecordIterator} that returns {@link RowData}s. The next row is set by
- * {@link ColumnarRowData#setRowId}.
+ * A {@link RecordReader.RecordIterator} that returns {@link InternalRow}s. The next row is set by
+ * {@link ColumnarRow#setRowId}.
*/
-@Internal
-public class ColumnarRowIterator extends RecyclableIterator<RowData> {
+public class ColumnarRowIterator extends RecyclableIterator<InternalRow> {
- private final ColumnarRowData rowData;
+ private final ColumnarRow rowData;
private int num;
private int pos;
- public ColumnarRowIterator(ColumnarRowData rowData, @Nullable Runnable recycler) {
+ public ColumnarRowIterator(ColumnarRow rowData, @Nullable Runnable recycler) {
super(recycler);
this.rowData = rowData;
}
@@ -49,7 +47,7 @@ public class ColumnarRowIterator extends RecyclableIterator<RowData> {
@Nullable
@Override
- public RowData next() {
+ public InternalRow next() {
if (pos < num) {
rowData.setRowId(pos++);
return rowData;
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/DecimalColumnVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/DecimalColumnVector.java
index f46cc63e..4d8b4e3e 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/DecimalColumnVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/DecimalColumnVector.java
@@ -18,11 +18,9 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.store.data.Decimal;
/** Decimal column vector. */
-@Internal
public interface DecimalColumnVector extends ColumnVector {
- DecimalData getDecimal(int i, int precision, int scale);
+ Decimal getDecimal(int i, int precision, int scale);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/Dictionary.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/Dictionary.java
index 9d2160f8..7f77762f 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/Dictionary.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/Dictionary.java
@@ -17,11 +17,9 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.store.data.Timestamp;
/** The interface for dictionary in AbstractColumnVector to decode dictionary encoded values. */
-@Internal
public interface Dictionary {
int decodeToInt(int id);
@@ -34,5 +32,5 @@ public interface Dictionary {
byte[] decodeToBinary(int id);
- TimestampData decodeToTimestamp(int id);
+ Timestamp decodeToTimestamp(int id);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/MapColumnVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/MapColumnVector.java
index 82c5f2fb..dfd292d6 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/MapColumnVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/MapColumnVector.java
@@ -18,11 +18,9 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.store.data.InternalMap;
/** Map column vector. */
-@Internal
public interface MapColumnVector extends ColumnVector {
- MapData getMap(int i);
+ InternalMap getMap(int i);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/RowColumnVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/RowColumnVector.java
index f167f490..185f1e29 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/RowColumnVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/RowColumnVector.java
@@ -23,5 +23,5 @@ import org.apache.flink.annotation.Internal;
/** Row column vector. */
@Internal
public interface RowColumnVector extends ColumnVector {
- ColumnarRowData getRow(int i);
+ ColumnarRow getRow(int i);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/TimestampColumnVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/TimestampColumnVector.java
index ec281f90..094de975 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/TimestampColumnVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/TimestampColumnVector.java
@@ -18,11 +18,9 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.store.data.Timestamp;
/** Timestamp column vector. */
-@Internal
public interface TimestampColumnVector extends ColumnVector {
- TimestampData getTimestamp(int i, int precision);
+ Timestamp getTimestamp(int i, int precision);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/VectorizedColumnBatch.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/VectorizedColumnBatch.java
index 60bf8d5a..a45408bb 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/VectorizedColumnBatch.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/VectorizedColumnBatch.java
@@ -18,12 +18,11 @@
package org.apache.flink.table.store.data.columnar;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.DecimalData;
-import org.apache.flink.table.data.MapData;
-import org.apache.flink.table.data.RowData;
-import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.store.data.Decimal;
+import org.apache.flink.table.store.data.InternalArray;
+import org.apache.flink.table.store.data.InternalMap;
+import org.apache.flink.table.store.data.InternalRow;
+import org.apache.flink.table.store.data.Timestamp;
import org.apache.flink.table.store.data.columnar.BytesColumnVector.Bytes;
import java.io.Serializable;
@@ -35,7 +34,6 @@ import java.nio.charset.StandardCharsets;
*
* <p>{@code VectorizedColumnBatch}s are influenced by Apache Hive VectorizedRowBatch.
*/
-@Internal
public class VectorizedColumnBatch implements Serializable {
private static final long serialVersionUID = 8180323238728166155L;
@@ -114,23 +112,23 @@ public class VectorizedColumnBatch implements Serializable {
return new String(byteArray.data, byteArray.offset, byteArray.len, StandardCharsets.UTF_8);
}
- public DecimalData getDecimal(int rowId, int colId, int precision, int scale) {
+ public Decimal getDecimal(int rowId, int colId, int precision, int scale) {
return ((DecimalColumnVector) (columns[colId])).getDecimal(rowId, precision, scale);
}
- public TimestampData getTimestamp(int rowId, int colId, int precision) {
+ public Timestamp getTimestamp(int rowId, int colId, int precision) {
return ((TimestampColumnVector) (columns[colId])).getTimestamp(rowId, precision);
}
- public ArrayData getArray(int rowId, int colId) {
+ public InternalArray getArray(int rowId, int colId) {
return ((ArrayColumnVector) columns[colId]).getArray(rowId);
}
- public RowData getRow(int rowId, int colId) {
+ public InternalRow getRow(int rowId, int colId) {
return ((RowColumnVector) columns[colId]).getRow(rowId);
}
- public MapData getMap(int rowId, int colId) {
+ public InternalMap getMap(int rowId, int colId) {
return ((MapColumnVector) columns[colId]).getMap(rowId);
}
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapArrayVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapArrayVector.java
index 736e645e..bb3cfeff 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapArrayVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapArrayVector.java
@@ -18,15 +18,13 @@
package org.apache.flink.table.store.data.columnar.heap;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
+import org.apache.flink.table.store.data.InternalArray;
import org.apache.flink.table.store.data.columnar.ArrayColumnVector;
import org.apache.flink.table.store.data.columnar.ColumnVector;
-import org.apache.flink.table.store.data.columnar.ColumnarArrayData;
+import org.apache.flink.table.store.data.columnar.ColumnarArray;
import org.apache.flink.table.store.data.columnar.writable.WritableColumnVector;
/** This class represents a nullable heap array column vector. */
-@Internal
public class HeapArrayVector extends AbstractHeapVector
implements WritableColumnVector, ArrayColumnVector {
@@ -81,9 +79,9 @@ public class HeapArrayVector extends AbstractHeapVector
}
@Override
- public ArrayData getArray(int i) {
+ public InternalArray getArray(int i) {
long offset = offsets[i];
long length = lengths[i];
- return new ColumnarArrayData(child, (int) offset, (int) length);
+ return new ColumnarArray(child, (int) offset, (int) length);
}
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapMapVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapMapVector.java
index 2dc50e4c..7d84a741 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapMapVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapMapVector.java
@@ -18,15 +18,13 @@
package org.apache.flink.table.store.data.columnar.heap;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.MapData;
+import org.apache.flink.table.store.data.InternalMap;
import org.apache.flink.table.store.data.columnar.ColumnVector;
-import org.apache.flink.table.store.data.columnar.ColumnarMapData;
+import org.apache.flink.table.store.data.columnar.ColumnarMap;
import org.apache.flink.table.store.data.columnar.MapColumnVector;
import org.apache.flink.table.store.data.columnar.writable.WritableColumnVector;
/** This class represents a nullable heap map column vector. */
-@Internal
public class HeapMapVector extends AbstractHeapVector
implements WritableColumnVector, MapColumnVector {
@@ -69,9 +67,9 @@ public class HeapMapVector extends AbstractHeapVector
}
@Override
- public MapData getMap(int i) {
+ public InternalMap getMap(int i) {
long offset = offsets[i];
long length = lengths[i];
- return new ColumnarMapData(keys, values, (int) offset, (int) length);
+ return new ColumnarMap(keys, values, (int) offset, (int) length);
}
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapRowVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapRowVector.java
index dd66910e..27f22475 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapRowVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapRowVector.java
@@ -19,7 +19,7 @@
package org.apache.flink.table.store.data.columnar.heap;
import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.store.data.columnar.ColumnarRowData;
+import org.apache.flink.table.store.data.columnar.ColumnarRow;
import org.apache.flink.table.store.data.columnar.RowColumnVector;
import org.apache.flink.table.store.data.columnar.VectorizedColumnBatch;
import org.apache.flink.table.store.data.columnar.writable.WritableColumnVector;
@@ -41,10 +41,10 @@ public class HeapRowVector extends AbstractHeapVector
}
@Override
- public ColumnarRowData getRow(int i) {
- ColumnarRowData columnarRowData = new ColumnarRowData(new VectorizedColumnBatch(fields));
- columnarRowData.setRowId(i);
- return columnarRowData;
+ public ColumnarRow getRow(int i) {
+ ColumnarRow columnarRow = new ColumnarRow(new VectorizedColumnBatch(fields));
+ columnarRow.setRowId(i);
+ return columnarRow;
}
@Override
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapTimestampVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapTimestampVector.java
index 1ef307fe..a0d25e17 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapTimestampVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/heap/HeapTimestampVector.java
@@ -18,14 +18,12 @@
package org.apache.flink.table.store.data.columnar.heap;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.store.data.Timestamp;
import org.apache.flink.table.store.data.columnar.writable.WritableTimestampVector;
import java.util.Arrays;
/** This class represents a nullable byte column vector. */
-@Internal
public class HeapTimestampVector extends AbstractHeapVector implements WritableTimestampVector {
private static final long serialVersionUID = 1L;
@@ -40,22 +38,22 @@ public class HeapTimestampVector extends AbstractHeapVector implements WritableT
}
@Override
- public TimestampData getTimestamp(int i, int precision) {
+ public Timestamp getTimestamp(int i, int precision) {
if (dictionary == null) {
- return TimestampData.fromEpochMillis(milliseconds[i], nanoOfMilliseconds[i]);
+ return Timestamp.fromEpochMillis(milliseconds[i], nanoOfMilliseconds[i]);
} else {
return dictionary.decodeToTimestamp(dictionaryIds.vector[i]);
}
}
@Override
- public void setTimestamp(int i, TimestampData timestamp) {
+ public void setTimestamp(int i, Timestamp timestamp) {
milliseconds[i] = timestamp.getMillisecond();
nanoOfMilliseconds[i] = timestamp.getNanoOfMillisecond();
}
@Override
- public void fill(TimestampData value) {
+ public void fill(Timestamp value) {
Arrays.fill(milliseconds, value.getMillisecond());
Arrays.fill(nanoOfMilliseconds, value.getNanoOfMillisecond());
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/writable/WritableTimestampVector.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/writable/WritableTimestampVector.java
index 6d271159..78a93763 100644
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/writable/WritableTimestampVector.java
+++ b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/columnar/writable/WritableTimestampVector.java
@@ -18,17 +18,15 @@
package org.apache.flink.table.store.data.columnar.writable;
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.store.data.Timestamp;
import org.apache.flink.table.store.data.columnar.TimestampColumnVector;
/** Writable {@link TimestampColumnVector}. */
-@Internal
public interface WritableTimestampVector extends WritableColumnVector, TimestampColumnVector {
- /** Set {@link TimestampData} at rowId with the provided value. */
- void setTimestamp(int rowId, TimestampData timestamp);
+ /** Set {@link Timestamp} at rowId with the provided value. */
+ void setTimestamp(int rowId, Timestamp timestamp);
/** Fill the column vector with the provided value. */
- void fill(TimestampData value);
+ void fill(Timestamp value);
}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayBooleanArrayConverter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayBooleanArrayConverter.java
deleted file mode 100644
index 71b0b5f0..00000000
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayBooleanArrayConverter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.store.data.converter;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.types.logical.ArrayType;
-
-/** Converter for {@link ArrayType} of {@code boolean[]} external type. */
-@Internal
-public class ArrayBooleanArrayConverter implements DataStructureConverter<ArrayData, boolean[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public ArrayData toInternal(boolean[] external) {
- return new GenericArrayData(external);
- }
-
- @Override
- public boolean[] toExternal(ArrayData internal) {
- return internal.toBooleanArray();
- }
-}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayByteArrayConverter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayByteArrayConverter.java
deleted file mode 100644
index df5a25fa..00000000
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayByteArrayConverter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.store.data.converter;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.types.logical.ArrayType;
-
-/** Converter for {@link ArrayType} of {@code byte[]} external type. */
-@Internal
-public class ArrayByteArrayConverter implements DataStructureConverter<ArrayData, byte[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public ArrayData toInternal(byte[] external) {
- return new GenericArrayData(external);
- }
-
- @Override
- public byte[] toExternal(ArrayData internal) {
- return internal.toByteArray();
- }
-}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayDoubleArrayConverter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayDoubleArrayConverter.java
deleted file mode 100644
index 5e7aa31d..00000000
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayDoubleArrayConverter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.store.data.converter;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.types.logical.ArrayType;
-
-/** Converter for {@link ArrayType} of {@code double[]} external type. */
-@Internal
-public class ArrayDoubleArrayConverter implements DataStructureConverter<ArrayData, double[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public ArrayData toInternal(double[] external) {
- return new GenericArrayData(external);
- }
-
- @Override
- public double[] toExternal(ArrayData internal) {
- return internal.toDoubleArray();
- }
-}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayFloatArrayConverter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayFloatArrayConverter.java
deleted file mode 100644
index 804d740a..00000000
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayFloatArrayConverter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.store.data.converter;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.types.logical.ArrayType;
-
-/** Converter for {@link ArrayType} of {@code float[]} external type. */
-@Internal
-public class ArrayFloatArrayConverter implements DataStructureConverter<ArrayData, float[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public ArrayData toInternal(float[] external) {
- return new GenericArrayData(external);
- }
-
- @Override
- public float[] toExternal(ArrayData internal) {
- return internal.toFloatArray();
- }
-}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayIntArrayConverter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayIntArrayConverter.java
deleted file mode 100644
index c83a257e..00000000
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayIntArrayConverter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.store.data.converter;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.types.logical.ArrayType;
-
-/** Converter for {@link ArrayType} of {@code int[]} external type. */
-@Internal
-public class ArrayIntArrayConverter implements DataStructureConverter<ArrayData, int[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public ArrayData toInternal(int[] external) {
- return new GenericArrayData(external);
- }
-
- @Override
- public int[] toExternal(ArrayData internal) {
- return internal.toIntArray();
- }
-}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayListConverter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayListConverter.java
deleted file mode 100644
index c7404d96..00000000
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayListConverter.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.store.data.converter;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.types.DataType;
-import org.apache.flink.table.types.logical.ArrayType;
-
-import java.lang.reflect.Array;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.apache.commons.lang3.ClassUtils.primitiveToWrapper;
-
-/** Converter for {@link ArrayType} of {@link List} external type. */
-@Internal
-public class ArrayListConverter<E> implements DataStructureConverter<ArrayData, List<E>> {
-
- private static final long serialVersionUID = 1L;
-
- private final E[] arrayKind;
-
- private final ArrayObjectArrayConverter<E> elementsConverter;
-
- private ArrayListConverter(E[] arrayKind, ArrayObjectArrayConverter<E> elementsConverter) {
- this.arrayKind = arrayKind;
- this.elementsConverter = elementsConverter;
- }
-
- @Override
- public void open(ClassLoader classLoader) {
- elementsConverter.open(classLoader);
- }
-
- @Override
- public ArrayData toInternal(List<E> external) {
- return elementsConverter.toInternal(external.toArray(arrayKind));
- }
-
- @Override
- public List<E> toExternal(ArrayData internal) {
- return new ArrayList<>(Arrays.asList(elementsConverter.toExternal(internal)));
- }
-
- // --------------------------------------------------------------------------------------------
- // Factory method
- // --------------------------------------------------------------------------------------------
-
- public static ArrayListConverter<?> create(DataType dataType) {
- final DataType elementDataType = dataType.getChildren().get(0);
- return new ArrayListConverter<>(
- createObjectArrayKind(elementDataType.getConversionClass()),
- ArrayObjectArrayConverter.createForElement(elementDataType));
- }
-
- /** Creates the kind of array for {@link List#toArray(Object[])}. */
- private static Object[] createObjectArrayKind(Class<?> elementClazz) {
- // e.g. int[] is not a Object[]
- if (elementClazz.isPrimitive()) {
- return (Object[]) Array.newInstance(primitiveToWrapper(elementClazz), 0);
- }
- // e.g. int[][] and Integer[] are Object[]
- return (Object[]) Array.newInstance(elementClazz, 0);
- }
-}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayLongArrayConverter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayLongArrayConverter.java
deleted file mode 100644
index b66fc0f3..00000000
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayLongArrayConverter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.store.data.converter;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.types.logical.ArrayType;
-
-/** Converter for {@link ArrayType} of {@code long[]} external type. */
-@Internal
-public class ArrayLongArrayConverter implements DataStructureConverter<ArrayData, long[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public ArrayData toInternal(long[] external) {
- return new GenericArrayData(external);
- }
-
- @Override
- public long[] toExternal(ArrayData internal) {
- return internal.toLongArray();
- }
-}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayObjectArrayConverter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayObjectArrayConverter.java
deleted file mode 100644
index d874b377..00000000
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayObjectArrayConverter.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.store.data.converter;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.data.binary.BinaryArrayData;
-import org.apache.flink.table.store.data.BinaryArrayWriter;
-import org.apache.flink.table.store.data.BinaryWriter;
-import org.apache.flink.table.types.DataType;
-import org.apache.flink.table.types.logical.ArrayType;
-import org.apache.flink.table.types.logical.DistinctType;
-import org.apache.flink.table.types.logical.LogicalType;
-
-import org.apache.commons.lang3.ArrayUtils;
-
-import java.io.Serializable;
-import java.lang.reflect.Array;
-
-import static org.apache.commons.lang3.ClassUtils.primitiveToWrapper;
-
-/** Converter for {@link ArrayType} of nested primitive or object arrays external types. */
-@Internal
-@SuppressWarnings("unchecked")
-public class ArrayObjectArrayConverter<E> implements DataStructureConverter<ArrayData, E[]> {
-
- private static final long serialVersionUID = 1L;
-
- private final Class<E> elementClass;
-
- private final int elementSize;
-
- private final BinaryArrayWriter.NullSetter writerNullSetter;
-
- private final BinaryWriter.ValueSetter writerValueSetter;
-
- private final GenericToJavaArrayConverter<E> genericToJavaArrayConverter;
-
- private transient BinaryArrayData reuseArray;
-
- private transient BinaryArrayWriter reuseWriter;
-
- final boolean hasInternalElements;
-
- final ArrayData.ElementGetter elementGetter;
-
- final DataStructureConverter<Object, E> elementConverter;
-
- private ArrayObjectArrayConverter(
- Class<E> elementClass,
- int elementSize,
- BinaryArrayWriter.NullSetter writerNullSetter,
- BinaryWriter.ValueSetter writerValueSetter,
- GenericToJavaArrayConverter<E> genericToJavaArrayConverter,
- ArrayData.ElementGetter elementGetter,
- DataStructureConverter<Object, E> elementConverter) {
- this.elementClass = elementClass;
- this.elementSize = elementSize;
- this.writerNullSetter = writerNullSetter;
- this.writerValueSetter = writerValueSetter;
- this.genericToJavaArrayConverter = genericToJavaArrayConverter;
- this.hasInternalElements = elementConverter.isIdentityConversion();
- this.elementGetter = elementGetter;
- this.elementConverter = elementConverter;
- }
-
- @Override
- public void open(ClassLoader classLoader) {
- reuseArray = new BinaryArrayData();
- reuseWriter = new BinaryArrayWriter(reuseArray, 0, elementSize);
- elementConverter.open(classLoader);
- }
-
- @Override
- public ArrayData toInternal(E[] external) {
- return hasInternalElements ? new GenericArrayData(external) : toBinaryArrayData(external);
- }
-
- @Override
- public E[] toExternal(ArrayData internal) {
- if (hasInternalElements && internal instanceof GenericArrayData) {
- final GenericArrayData genericArray = (GenericArrayData) internal;
- if (genericArray.isPrimitiveArray()) {
- return genericToJavaArrayConverter.convert((GenericArrayData) internal);
- }
- return (E[]) genericArray.toObjectArray();
- }
- return toJavaArray(internal);
- }
-
- // --------------------------------------------------------------------------------------------
- // Runtime helper methods
- // --------------------------------------------------------------------------------------------
-
- private ArrayData toBinaryArrayData(E[] external) {
- final int length = external.length;
- allocateWriter(length);
- for (int pos = 0; pos < length; pos++) {
- writeElement(pos, external[pos]);
- }
- return completeWriter().copy();
- }
-
- private E[] toJavaArray(ArrayData internal) {
- final int size = internal.size();
- final E[] values = (E[]) Array.newInstance(elementClass, size);
- for (int pos = 0; pos < size; pos++) {
- final Object value = elementGetter.getElementOrNull(internal, pos);
- values[pos] = elementConverter.toExternalOrNull(value);
- }
- return values;
- }
-
- interface GenericToJavaArrayConverter<E> extends Serializable {
- E[] convert(GenericArrayData internal);
- }
-
- // --------------------------------------------------------------------------------------------
- // Shared code
- // --------------------------------------------------------------------------------------------
-
- void allocateWriter(int length) {
- if (reuseWriter.getNumElements() != length) {
- reuseWriter = new BinaryArrayWriter(reuseArray, length, elementSize);
- } else {
- reuseWriter.reset();
- }
- }
-
- void writeElement(int pos, E element) {
- if (element == null) {
- writerNullSetter.setNull(reuseWriter, pos);
- } else {
- writerValueSetter.setValue(reuseWriter, pos, elementConverter.toInternal(element));
- }
- }
-
- BinaryArrayData completeWriter() {
- reuseWriter.complete();
- return reuseArray;
- }
-
- // --------------------------------------------------------------------------------------------
- // Factory method
- // --------------------------------------------------------------------------------------------
-
- public static ArrayObjectArrayConverter<?> create(DataType dataType) {
- return createForElement(dataType.getChildren().get(0));
- }
-
- public static <E> ArrayObjectArrayConverter<E> createForElement(DataType elementDataType) {
- final LogicalType elementType = elementDataType.getLogicalType();
- return new ArrayObjectArrayConverter<>(
- (Class<E>) primitiveToWrapper(elementDataType.getConversionClass()),
- BinaryArrayData.calculateFixLengthPartSize(elementType),
- BinaryArrayWriter.createNullSetter(elementType),
- BinaryWriter.createValueSetter(elementType),
- createGenericToJavaArrayConverter(elementType),
- ArrayData.createElementGetter(elementType),
- (DataStructureConverter<Object, E>)
- DataStructureConverters.getConverter(elementDataType));
- }
-
- @SuppressWarnings("unchecked")
- private static <E> GenericToJavaArrayConverter<E> createGenericToJavaArrayConverter(
- LogicalType elementType) {
- switch (elementType.getTypeRoot()) {
- case BOOLEAN:
- return internal -> (E[]) ArrayUtils.toObject(internal.toBooleanArray());
- case TINYINT:
- return internal -> (E[]) ArrayUtils.toObject(internal.toByteArray());
- case SMALLINT:
- return internal -> (E[]) ArrayUtils.toObject(internal.toShortArray());
- case INTEGER:
- return internal -> (E[]) ArrayUtils.toObject(internal.toIntArray());
- case BIGINT:
- return internal -> (E[]) ArrayUtils.toObject(internal.toLongArray());
- case FLOAT:
- return internal -> (E[]) ArrayUtils.toObject(internal.toFloatArray());
- case DOUBLE:
- return internal -> (E[]) ArrayUtils.toObject(internal.toDoubleArray());
- case DISTINCT_TYPE:
- return createGenericToJavaArrayConverter(
- ((DistinctType) elementType).getSourceType());
- default:
- return internal -> {
- throw new IllegalStateException();
- };
- }
- }
-}
diff --git a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayShortArrayConverter.java b/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayShortArrayConverter.java
deleted file mode 100644
index d8d73b40..00000000
--- a/flink-table-store-common/src/main/java/org/apache/flink/table/store/data/converter/ArrayShortArrayConverter.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.store.data.converter;
-
-import org.apache.flink.annotation.Internal;
-import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.GenericArrayData;
-import org.apache.flink.table.types.logical.ArrayType;
-
-/** Converter for {@link ArrayType} of {@code short[]} external type. */
-@Internal
-public class ArrayShortArrayConverter implements DataStructureConverter<ArrayData, short[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public ArrayData toInternal(short[] external) {
... 36510 lines suppressed ...