You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by hu...@apache.org on 2019/03/24 07:20:33 UTC
[hawq] branch huor_dbcommon created (now 39d8eb5)
This is an automated email from the ASF dual-hosted git repository.
huor pushed a change to branch huor_dbcommon
in repository https://gitbox.apache.org/repos/asf/hawq.git.
at 39d8eb5 HAWQ-1690. Add common utility used by ORC foramt
This branch includes the following new commits:
new 39d8eb5 HAWQ-1690. Add common utility used by ORC foramt
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[hawq] 01/01: HAWQ-1690. Add common utility used by ORC foramt
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
huor pushed a commit to branch huor_dbcommon
in repository https://gitbox.apache.org/repos/asf/hawq.git
commit 39d8eb55585c9dba390c6f434cc4369dd9a6e030
Author: Ruilong Huo <hu...@apache.org>
AuthorDate: Fri Mar 22 21:34:52 2019 +0800
HAWQ-1690. Add common utility used by ORC foramt
---
depends/Makefile | 34 +
depends/README | 0
depends/bootstrap | 27 +
depends/build-all.sh | 126 ++
depends/dbcommon/.gitignore | 8 +
.../dbcommon/CMake/CMakeTestCompileInt64tType.cc | 30 +
depends/dbcommon/CMake/FindCogapp.cmake | 54 +
depends/dbcommon/CMake/FindGFlags.cmake | 48 +
depends/dbcommon/CMake/FindGlog.cmake | 49 +
depends/dbcommon/CMake/FindJSON.cmake | 38 +
depends/dbcommon/CMake/FindSnappy.cmake | 30 +
depends/dbcommon/CMake/Functions.cmake | 46 +
depends/dbcommon/CMake/Options.cmake | 71 +
depends/dbcommon/CMake/Platform.cmake | 47 +
depends/dbcommon/CMakeLists.txt | 28 +
depends/dbcommon/README | 14 +
depends/dbcommon/bootstrap | 92 +
depends/dbcommon/src/CMakeLists.txt | 154 ++
.../src/dbcommon/checksum/checksum-util.cc | 42 +
.../dbcommon/src/dbcommon/checksum/checksum-util.h | 55 +
depends/dbcommon/src/dbcommon/checksum/checksum.h | 61 +
.../dbcommon/src/dbcommon/checksum/hw-crc32c.cc | 160 ++
depends/dbcommon/src/dbcommon/checksum/hw-crc32c.h | 69 +
.../dbcommon/src/dbcommon/checksum/sw-crc32c.cc | 94 +
depends/dbcommon/src/dbcommon/checksum/sw-crc32c.h | 45 +
.../src/dbcommon/common/node-deserializer.cc | 33 +
.../src/dbcommon/common/node-deserializer.h | 75 +
.../dbcommon/src/dbcommon/common/node-serializer.h | 51 +
.../src/dbcommon/common/tuple-batch-store.cc | 108 +
.../src/dbcommon/common/tuple-batch-store.h | 82 +
.../dbcommon/src/dbcommon/common/tuple-batch.cc | 860 +++++++
depends/dbcommon/src/dbcommon/common/tuple-batch.h | 329 +++
depends/dbcommon/src/dbcommon/common/tuple-desc.cc | 121 +
depends/dbcommon/src/dbcommon/common/tuple-desc.h | 70 +
.../src/dbcommon/common/vector-transformer.h | 311 +++
depends/dbcommon/src/dbcommon/common/vector.cc | 260 +++
depends/dbcommon/src/dbcommon/common/vector.h | 763 +++++++
.../src/dbcommon/common/vector/decimal-vector.cc | 620 +++++
.../src/dbcommon/common/vector/decimal-vector.h | 215 ++
.../dbcommon/common/vector/fixed-length-vector.h | 702 ++++++
.../src/dbcommon/common/vector/interval-vector.cc | 553 +++++
.../src/dbcommon/common/vector/interval-vector.h | 166 ++
.../src/dbcommon/common/vector/list-vector.h | 621 +++++
.../src/dbcommon/common/vector/struct-vector.h | 228 ++
.../src/dbcommon/common/vector/timestamp-vector.h | 698 ++++++
.../common/vector/variable-length-vector.cc | 83 +
.../common/vector/variable-length-vector.h | 1173 ++++++++++
.../src/dbcommon/filesystem/file-system-manager.cc | 112 +
.../src/dbcommon/filesystem/file-system-manager.h | 72 +
.../dbcommon/src/dbcommon/filesystem/file-system.h | 226 ++
.../dbcommon/filesystem/hdfs/hdfs-file-system.cc | 279 +++
.../dbcommon/filesystem/hdfs/hdfs-file-system.h | 160 ++
.../src/dbcommon/filesystem/hive/fb303.thrift | 113 +
.../dbcommon/filesystem/hive/hive-file-system.cc | 58 +
.../dbcommon/filesystem/hive/hive-file-system.h | 36 +
.../dbcommon/filesystem/hive/hive_metastore.thrift | 1538 +++++++++++++
.../dbcommon/filesystem/local/local-file-system.cc | 499 +++++
.../dbcommon/filesystem/local/local-file-system.h | 191 ++
depends/dbcommon/src/dbcommon/function/CPPLINT.cfg | 1 +
depends/dbcommon/src/dbcommon/function/README | 2 +
depends/dbcommon/src/dbcommon/function/agg-func.cc | 2366 ++++++++++++++++++++
depends/dbcommon/src/dbcommon/function/agg-func.h | 323 +++
.../src/dbcommon/function/arith-cmp-func.h | 133 ++
.../dbcommon/src/dbcommon/function/arith-func.cc | 86 +
.../src/dbcommon/function/arithmetic-function.h | 351 +++
.../src/dbcommon/function/array-function.cc | 1021 +++++++++
.../src/dbcommon/function/array-function.h | 48 +
.../src/dbcommon/function/binary-function.cc | 299 +++
depends/dbcommon/src/dbcommon/function/cmp-func.cc | 537 +++++
.../src/dbcommon/function/comparison-function.h | 170 ++
.../src/dbcommon/function/date-function.cc | 1292 +++++++++++
.../dbcommon/src/dbcommon/function/date-function.h | 104 +
.../src/dbcommon/function/decimal-function.cc | 952 ++++++++
.../src/dbcommon/function/decimal-function.h | 110 +
depends/dbcommon/src/dbcommon/function/func-kind.h | 355 +++
depends/dbcommon/src/dbcommon/function/func.cc | 584 +++++
depends/dbcommon/src/dbcommon/function/func.h | 91 +
depends/dbcommon/src/dbcommon/function/function.h | 719 ++++++
depends/dbcommon/src/dbcommon/function/invoker.cc | 48 +
depends/dbcommon/src/dbcommon/function/invoker.h | 65 +
.../src/dbcommon/function/mathematical-function.cc | 306 +++
.../src/dbcommon/function/mathematical-function.h | 543 +++++
.../src/dbcommon/function/string-binary-function.h | 67 +
.../src/dbcommon/function/string-function.cc | 969 ++++++++
.../src/dbcommon/function/typecast-func.cc | 662 ++++++
.../dbcommon/src/dbcommon/function/typecast-func.h | 87 +
.../src/dbcommon/function/typecast-function.cc | 229 ++
.../src/dbcommon/function/typecast-function.h | 40 +
.../dbcommon/function/typecast-texttonum-func.cc | 240 ++
.../dbcommon/function/typecast-texttonum-func.h | 37 +
.../src/dbcommon/function/volatile-func.cc | 56 +
.../dbcommon/src/dbcommon/function/volatile-func.h | 30 +
depends/dbcommon/src/dbcommon/hash/cdb-hash.h | 95 +
depends/dbcommon/src/dbcommon/hash/fast-hash.h | 267 +++
depends/dbcommon/src/dbcommon/hash/hash-keys.cc | 62 +
depends/dbcommon/src/dbcommon/hash/hash-keys.h | 357 +++
.../src/dbcommon/hash/native-hash-table.cc | 97 +
.../dbcommon/src/dbcommon/hash/native-hash-table.h | 472 ++++
.../src/dbcommon/hash/tuple-batch-hasher.h | 168 ++
depends/dbcommon/src/dbcommon/log/debug-logger.cc | 83 +
depends/dbcommon/src/dbcommon/log/debug-logger.h | 120 +
depends/dbcommon/src/dbcommon/log/error-code.h | 444 ++++
depends/dbcommon/src/dbcommon/log/exception.h | 54 +
depends/dbcommon/src/dbcommon/log/logger.cc | 47 +
depends/dbcommon/src/dbcommon/log/logger.h | 154 ++
depends/dbcommon/src/dbcommon/log/stack-printer.cc | 663 ++++++
depends/dbcommon/src/dbcommon/log/stack-printer.h | 35 +
.../dbcommon/network/socket-tcp-message-client.cc | 124 +
.../dbcommon/network/socket-tcp-message-client.h | 45 +
.../dbcommon/network/socket-tcp-message-common.cc | 307 +++
.../dbcommon/network/socket-tcp-message-common.h | 197 ++
.../dbcommon/network/socket-tcp-message-server.cc | 193 ++
.../dbcommon/network/socket-tcp-message-server.h | 90 +
.../network/socket-tcp-message-serverhandler.h | 58 +
.../dbcommon/src/dbcommon/network/socket-tcp.cc | 103 +
depends/dbcommon/src/dbcommon/network/socket-tcp.h | 35 +
depends/dbcommon/src/dbcommon/nodes/datum.cc | 65 +
depends/dbcommon/src/dbcommon/nodes/datum.h | 483 ++++
depends/dbcommon/src/dbcommon/nodes/scalar.cc | 46 +
depends/dbcommon/src/dbcommon/nodes/scalar.h | 88 +
depends/dbcommon/src/dbcommon/nodes/select-list.cc | 175 ++
depends/dbcommon/src/dbcommon/nodes/select-list.h | 170 ++
.../dbcommon/src/dbcommon/python/code_generator.py | 225 ++
.../src/dbcommon/testutil/agg-func-utils.h | 271 +++
.../src/dbcommon/testutil/function-utils.cc | 480 ++++
.../src/dbcommon/testutil/function-utils.h | 225 ++
.../dbcommon/src/dbcommon/testutil/scalar-utils.h | 62 +
.../src/dbcommon/testutil/tuple-batch-utils.h | 301 +++
.../dbcommon/src/dbcommon/testutil/vector-utils.h | 335 +++
.../src/dbcommon/thread/err-detect-callback.h | 133 ++
.../dbcommon/src/dbcommon/thread/thread-base.cc | 36 +
depends/dbcommon/src/dbcommon/thread/thread-base.h | 172 ++
depends/dbcommon/src/dbcommon/type/CPPLINT.cfg | 1 +
depends/dbcommon/src/dbcommon/type/README | 2 +
depends/dbcommon/src/dbcommon/type/array.cc | 198 ++
depends/dbcommon/src/dbcommon/type/array.h | 132 ++
depends/dbcommon/src/dbcommon/type/bool.cc | 62 +
depends/dbcommon/src/dbcommon/type/bool.h | 63 +
depends/dbcommon/src/dbcommon/type/date.cc | 420 ++++
depends/dbcommon/src/dbcommon/type/date.h | 909 ++++++++
depends/dbcommon/src/dbcommon/type/decimal.cc | 665 ++++++
depends/dbcommon/src/dbcommon/type/decimal.h | 638 ++++++
depends/dbcommon/src/dbcommon/type/float.h | 137 ++
depends/dbcommon/src/dbcommon/type/integer.h | 143 ++
depends/dbcommon/src/dbcommon/type/interval.h | 261 +++
depends/dbcommon/src/dbcommon/type/magma-tid.h | 105 +
depends/dbcommon/src/dbcommon/type/type-kind.h | 118 +
depends/dbcommon/src/dbcommon/type/type-modifier.h | 66 +
depends/dbcommon/src/dbcommon/type/type-util.cc | 225 ++
depends/dbcommon/src/dbcommon/type/type-util.h | 137 ++
depends/dbcommon/src/dbcommon/type/typebase.cc | 178 ++
depends/dbcommon/src/dbcommon/type/typebase.h | 280 +++
depends/dbcommon/src/dbcommon/type/varlen.cc | 182 ++
depends/dbcommon/src/dbcommon/type/varlen.h | 150 ++
depends/dbcommon/src/dbcommon/utils/async-queue.h | 340 +++
.../src/dbcommon/utils/block-memory-buffer.h | 176 ++
depends/dbcommon/src/dbcommon/utils/bool-buffer.cc | 351 +++
depends/dbcommon/src/dbcommon/utils/bool-buffer.h | 225 ++
depends/dbcommon/src/dbcommon/utils/byte-buffer.h | 265 +++
.../dbcommon/src/dbcommon/utils/comp/compressor.h | 64 +
.../src/dbcommon/utils/comp/lz4-compressor.cc | 94 +
.../src/dbcommon/utils/comp/lz4-compressor.h | 51 +
.../src/dbcommon/utils/comp/snappy-compressor.cc | 28 +
.../src/dbcommon/utils/comp/snappy-compressor.h | 90 +
.../src/dbcommon/utils/comp/zlib-compressor.cc | 81 +
.../src/dbcommon/utils/comp/zlib-compressor.h | 57 +
depends/dbcommon/src/dbcommon/utils/cutils.cc | 71 +
depends/dbcommon/src/dbcommon/utils/cutils.h | 63 +
depends/dbcommon/src/dbcommon/utils/file-info.h | 41 +
.../src/dbcommon/utils/flat-memory-buffer.h | 207 ++
depends/dbcommon/src/dbcommon/utils/global.cc | 23 +
depends/dbcommon/src/dbcommon/utils/global.h | 29 +
depends/dbcommon/src/dbcommon/utils/instrument.cc | 80 +
depends/dbcommon/src/dbcommon/utils/instrument.h | 66 +
depends/dbcommon/src/dbcommon/utils/int-util.h | 357 +++
.../src/dbcommon/utils/join-tuple-buffer.cc | 488 ++++
.../src/dbcommon/utils/join-tuple-buffer.h | 130 ++
depends/dbcommon/src/dbcommon/utils/lock.cc | 220 ++
depends/dbcommon/src/dbcommon/utils/lock.h | 116 +
depends/dbcommon/src/dbcommon/utils/macro.h | 44 +
.../dbcommon/src/dbcommon/utils/mb/mb-converter.cc | 246 ++
.../dbcommon/src/dbcommon/utils/mb/mb-converter.h | 116 +
depends/dbcommon/src/dbcommon/utils/memory-pool.cc | 28 +
depends/dbcommon/src/dbcommon/utils/memory-pool.h | 208 ++
depends/dbcommon/src/dbcommon/utils/net-client.h | 46 +
.../dbcommon/src/dbcommon/utils/object-counter.cc | 35 +
.../dbcommon/src/dbcommon/utils/object-counter.h | 54 +
depends/dbcommon/src/dbcommon/utils/parameters.h | 130 ++
depends/dbcommon/src/dbcommon/utils/string-util.cc | 231 ++
depends/dbcommon/src/dbcommon/utils/string-util.h | 277 +++
depends/dbcommon/src/dbcommon/utils/sys-info.h | 51 +
depends/dbcommon/src/dbcommon/utils/time-util.h | 62 +
.../dbcommon/src/dbcommon/utils/timezone-util.cc | 458 ++++
.../dbcommon/src/dbcommon/utils/timezone-util.h | 527 +++++
depends/dbcommon/src/dbcommon/utils/url.cc | 125 ++
depends/dbcommon/src/dbcommon/utils/url.h | 89 +
depends/dbcommon/test/CMakeLists.txt | 34 +
depends/dbcommon/test/data/checksum1.in | 512 +++++
depends/dbcommon/test/data/checksum2.in | 512 +++++
.../dbcommon/test/parallel/parallel-launcher.py | 153 ++
depends/dbcommon/test/unit/CMakeLists.txt | 30 +
.../dbcommon/test/unit/checksum/test-checksum.cc | 134 ++
.../dbcommon/test/unit/common/test-async-queue.cc | 82 +
depends/dbcommon/test/unit/common/test-function.cc | 54 +
.../unit/common/test-tuple-batch-copy-control.cc | 69 +
.../test/unit/common/test-tuple-batch-store.cc | 55 +
.../dbcommon/test/unit/common/test-tuple-batch.cc | 684 ++++++
.../test/unit/common/test-vector-copy-control.cc | 624 ++++++
depends/dbcommon/test/unit/common/test-vector.cc | 731 ++++++
.../test/unit/filesystem/test-file-system.cc | 93 +
depends/dbcommon/test/unit/function/CPPLINT.cfg | 1 +
.../unit/function/test-agg-func-has-no-group-by.cc | 876 ++++++++
.../unit/function/test-agg-func-small-scale.cc | 704 ++++++
.../dbcommon/test/unit/function/test-agg-func.cc | 749 +++++++
.../dbcommon/test/unit/function/test-array-func.cc | 377 ++++
.../test/unit/function/test-binary-cmp-function.cc | 429 ++++
.../test/unit/function/test-binary-function.cc | 41 +
.../unit/function/test-codegen-arith-functions.cc | 617 +++++
.../unit/function/test-codegen-cmp-functions.cc | 413 ++++
.../function/test-codegen-typecast-functions.cc | 261 +++
.../test/unit/function/test-date-function.cc | 449 ++++
.../test/unit/function/test-decimal-function.cc | 420 ++++
.../unit/function/test-mathematical-function.cc | 1039 +++++++++
.../test/unit/function/test-string-cmp-function.cc | 427 ++++
.../test/unit/function/test-string-function.cc | 1579 +++++++++++++
.../test/unit/function/test-timestamp-function.cc | 543 +++++
.../test/unit/function/test-typecast-function.cc | 295 +++
.../unit/function/test-typecast-texttonum-func.cc | 107 +
.../dbcommon/test/unit/log/test-debug-logger.cc | 41 +
.../unit/network/test-socket-tcp-message-comm.cc | 105 +
depends/dbcommon/test/unit/nodes/test-datum.cc | 195 ++
.../dbcommon/test/unit/nodes/test-select-list.cc | 168 ++
depends/dbcommon/test/unit/test-hash-table.cc | 231 ++
.../test/unit/thread/test-err-detect-callback.cc | 120 +
depends/dbcommon/test/unit/type/test-type.cc | 454 ++++
depends/dbcommon/test/unit/unit-test-main.cc | 34 +
.../dbcommon/test/unit/utils/test-bool-buffer.cc | 272 +++
.../dbcommon/test/unit/utils/test-byte-buffer.cc | 89 +
depends/dbcommon/test/unit/utils/test-cutils.cc | 50 +
depends/dbcommon/test/unit/utils/test-int-util.cc | 88 +
.../test/unit/utils/test-join-tuple-buffer.cc | 236 ++
depends/dbcommon/test/unit/utils/test-lock.cc | 180 ++
.../dbcommon/test/unit/utils/test-lz4-compress.cc | 57 +
.../dbcommon/test/unit/utils/test-mb-converter.cc | 49 +
.../dbcommon/test/unit/utils/test-string-util.cc | 41 +
depends/dbcommon/test/unit/utils/test-url.cc | 112 +
246 files changed, 60416 insertions(+)
diff --git a/depends/Makefile b/depends/Makefile
new file mode 100644
index 0000000..739e200
--- /dev/null
+++ b/depends/Makefile
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+.PHONY: release debug
+
+incremental:
+ ./build-all.sh incremental
+
+release:
+ ./build-all.sh release
+
+debug:
+ ./build-all.sh debug
+
+coverage:
+ ./build-all.sh coverage
+
+noavx:
+ ./build-all.sh noavx
diff --git a/depends/README b/depends/README
new file mode 100644
index 0000000..e69de29
diff --git a/depends/bootstrap b/depends/bootstrap
new file mode 100755
index 0000000..1851fd2
--- /dev/null
+++ b/depends/bootstrap
@@ -0,0 +1,27 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source_dir=`cd "\`dirname \"$0\"\`";pwd`
+if [ $source_dir = `pwd` ]; then
+ echo 'Error: Don''t bootstrap in the source folder'
+ exit
+fi
+
+cp $source_dir/Makefile .
+cat $source_dir/build-all.sh| sed "s|../bootstrap|$source_dir/\$MODULE/bootstrap|g" > ./build-all.sh
+chmod +x ./build-all.sh
diff --git a/depends/build-all.sh b/depends/build-all.sh
new file mode 100755
index 0000000..17d96b8
--- /dev/null
+++ b/depends/build-all.sh
@@ -0,0 +1,126 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PREFIX="/opt/dependency/package"
+export LD_LIBRARY_PATH=$PREFIX/lib:$LD_LIBRARY_PATH
+export DYLD_FALLBACK_LIBRARY_PATH=$PREFIX/lib:$DYLD_FALLBACK_LIBRARY_PATH
+
+if [[ -z $RUN_UNITTEST ]]; then
+ RUN_UNITTEST=YES
+fi
+OPTION=""
+case $1 in
+ release)
+ echo "Build Release Version ..."
+ OPTION=""
+ ;;
+ debug)
+ echo "Build Debug Version ..."
+ OPTION="--enable-debug"
+ ;;
+ coverage)
+ echo "Build Coverage Version ..."
+ OPTION="--enable-coverage"
+ ;;
+ noavx)
+ echo "Build AVX Version ..."
+ OPTION="--enable-avx=OFF"
+ ;;
+ incremental)
+ echo "Build incremental ..."
+ OPTION="incremental"
+ ;;
+ *)
+ echo "Build Release Version ..."
+ OPTION="";
+ ;;
+esac
+
+function build() {
+ echo "Build $1 ..."
+
+ MODULE=$1
+ MAGMA_OPTION=""
+ ###kv_client && kv_server#####
+ if [ $1 = "magma_client" ] ;then
+ MAGMA_OPTION="--enable-client"
+ MODULE=magma
+ fi
+ if [ $1 = "magma_server" ] ;then
+ MAGMA_OPTION="--enable-server"
+ MODULE=magma
+ fi
+
+ if [ "$OPTION" != "incremental" ]; then
+ echo "rm $DIR/$MODULE/build"
+ rm -rf $DIR/$MODULE/build
+
+ mkdir -p $DIR/$MODULE/build
+ cd $DIR/$MODULE/build
+ echo "../bootstrap $OPTION $MAGMA_OPTION"
+ ../bootstrap $OPTION $MAGMA_OPTION
+ CMAKE_OPTION=$OPTION
+ else
+ if [ ! -d $DIR/$MODULE/build ]; then mkdir -p $DIR/$MODULE/build; fi
+ cd $DIR/$MODULE/build
+ CMAKE_OPTION=$(grep 'CMAKE_BUILD_TYPE:STRING=' CMakeCache.txt | cut -d'=' -f2)
+ case $CMAKE_OPTION in
+ Release)
+ CMAKE_OPTION=""
+ ;;
+ Debug)
+ CMAKE_OPTION="--enable-debug"
+ codecov=$(grep 'ENABLE_COVERAGE:BOOL=' CMakeCache.txt | cut -d'=' -f2)
+ if [ $codecov = 'ON' ]; then
+ CMAKE_OPTION="--enable-coverage"
+ fi
+ ;;
+ *)
+ CMAKE_OPTION="";
+ ;;
+ esac
+ echo "../bootstrap $CMAKE_OPTION $MAGMA_OPTION"
+ ../bootstrap $CMAKE_OPTION $MAGMA_OPTION
+ fi
+
+ if [[ $OPTION == "--enable-coverage" || $CMAKE_OPTION == "--enable-coverage" ]]
+ then
+ make coverage
+ else
+ make -j8 ;
+ if [[ $RUN_UNITTEST == "YES" ]]; then
+ echo "Run unittest ...."
+ make -j8 punittest
+ else
+ echo "Do not run unittest...."
+ fi
+ fi
+ make -j8 install
+
+ echo "Done."
+}
+echo "Delete headers in ${PREFIX}/include ..."
+rm -rf $PREFIX/include/dbcommon
+echo "Done."
+
+echo "Delete libs in ${PREFIX}/lib ..."
+rm -rf $PREFIX/lib/libdbcommon*
+echo "Done."
+
+build dbcommon
diff --git a/depends/dbcommon/.gitignore b/depends/dbcommon/.gitignore
new file mode 100644
index 0000000..a9b913a
--- /dev/null
+++ b/depends/dbcommon/.gitignore
@@ -0,0 +1,8 @@
+.DS_Store
+.cproject
+.project
+.settings
+.pydevproject
+*.pyc
+build/
+CodeCoverageReport/
diff --git a/depends/dbcommon/CMake/CMakeTestCompileInt64tType.cc b/depends/dbcommon/CMake/CMakeTestCompileInt64tType.cc
new file mode 100644
index 0000000..ad2fc8e
--- /dev/null
+++ b/depends/dbcommon/CMake/CMakeTestCompileInt64tType.cc
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <cstdint>
+
+// test if int64_t is typedef to be long long
+
+void test(long long *i) {} // NOLINT
+
+int main() {
+ int64_t i = 0;
+ test(&i);
+ return 0;
+}
diff --git a/depends/dbcommon/CMake/FindCogapp.cmake b/depends/dbcommon/CMake/FindCogapp.cmake
new file mode 100644
index 0000000..c58e1dd
--- /dev/null
+++ b/depends/dbcommon/CMake/FindCogapp.cmake
@@ -0,0 +1,54 @@
+# locate cogapp and generate source code from template
+#
+# find_package(Cogapp REQUIRED)
+#
+# COGAPP_GENERATE (public function)
+# GENERATED_CODE = Variable to define with generated source files.
+# TEMPLATE = Template used to generate source files.
+#
+# NOTE: The COGAPP_GENERATE macro & add_executable() or add_library()
+# calls only work properly within the same directory.
+#
+
+find_package(PythonInterp REQUIRED)
+
+function(COGAPP_GENERATE GENERATED_CODE)
+ if(NOT ARGN)
+ message(SEND_ERROR "Error: COGAPP_GENERATE() called without any template files")
+ return()
+ endif()
+
+ message("in COGAPP_GENERATE")
+
+ set(${GENERATED_CODE})
+ foreach(FIL ${ARGN})
+ get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+ file(RELATIVE_PATH FIL_REL ${CMAKE_SOURCE_DIR} ${ABS_FIL})
+
+ get_filename_component(FIL_DIR ${CMAKE_BINARY_DIR}/codegen/${FIL_REL} DIRECTORY)
+ file(MAKE_DIRECTORY ${FIL_DIR})
+
+ get_filename_component(FIL_WE "${CMAKE_BINARY_DIR}/codegen/${FIL_REL}" NAME_WE)
+ get_filename_component(FIL_EXT "${CMAKE_BINARY_DIR}/codegen/${FIL_REL}" EXT)
+
+ set(FIL_OUT "${FIL_DIR}/${FIL_WE}.cg${FIL_EXT}")
+ list(APPEND ${GENERATED_CODE} ${FIL_OUT})
+
+ if(NOT EXISTS ${ABS_FIL})
+ MESSAGE(FATAL_ERROR "file ${ABS_FIL} does not exist")
+ endif()
+
+ add_custom_command(
+ OUTPUT ${FIL_OUT}
+ COMMAND ${PYTHON_EXECUTABLE}
+ ARGS -m cogapp -d -o ${FIL_OUT} ${ABS_FIL}
+ DEPENDS ${ABS_FIL}
+ COMMENT "Running cog on ${FIL}"
+ VERBATIM )
+
+ message("in COGAPP_GENERATE loop ${FIL_OUT} ${PYTHON_EXECUTABLE} ${ABS_FIL} ${FIL}")
+ endforeach()
+
+ set_source_files_properties(${${GENERATED_CODE}} PROPERTIES GENERATED TRUE)
+ set(${GENERATED_CODE} ${${GENERATED_CODE}} PARENT_SCOPE)
+endfunction()
\ No newline at end of file
diff --git a/depends/dbcommon/CMake/FindGFlags.cmake b/depends/dbcommon/CMake/FindGFlags.cmake
new file mode 100644
index 0000000..f93c571
--- /dev/null
+++ b/depends/dbcommon/CMake/FindGFlags.cmake
@@ -0,0 +1,48 @@
+# - Try to find GFLAGS
+#
+# The following variables are optionally searched for defaults
+# GFLAGS_ROOT_DIR: Base directory where all GFLAGS components are found
+#
+# The following are set after configuration is done:
+# GFLAGS_FOUND
+# GFLAGS_INCLUDE_DIRS
+# GFLAGS_LIBRARIES
+# GFLAGS_LIBRARYRARY_DIRS
+
+include(FindPackageHandleStandardArgs)
+
+set(GFLAGS_ROOT_DIR "" CACHE PATH "Folder contains Gflags")
+
+# We are testing only a couple of files in the include directories
+if(WIN32)
+ find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h
+ PATHS ${GFLAGS_ROOT_DIR}/src/windows)
+else()
+ find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h
+ PATHS ${GFLAGS_ROOT_DIR})
+endif()
+
+if(MSVC)
+ find_library(GFLAGS_LIBRARY_RELEASE
+ NAMES libgflags
+ PATHS ${GFLAGS_ROOT_DIR}
+ PATH_SUFFIXES Release)
+
+ find_library(GFLAGS_LIBRARY_DEBUG
+ NAMES libgflags-debug
+ PATHS ${GFLAGS_ROOT_DIR}
+ PATH_SUFFIXES Debug)
+
+ set(GFLAGS_LIBRARY optimized ${GFLAGS_LIBRARY_RELEASE} debug ${GFLAGS_LIBRARY_DEBUG})
+else()
+ find_library(GFLAGS_LIBRARY gflags)
+endif()
+
+find_package_handle_standard_args(GFLAGS DEFAULT_MSG
+ GFLAGS_INCLUDE_DIR GFLAGS_LIBRARY)
+
+
+if(GFLAGS_FOUND)
+ set(GFLAGS_INCLUDE_DIRS ${GFLAGS_INCLUDE_DIR})
+ set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY})
+endif()
diff --git a/depends/dbcommon/CMake/FindGlog.cmake b/depends/dbcommon/CMake/FindGlog.cmake
new file mode 100644
index 0000000..d9f0ee0
--- /dev/null
+++ b/depends/dbcommon/CMake/FindGlog.cmake
@@ -0,0 +1,49 @@
+
+# - Try to find Glog
+#
+# The following variables are optionally searched for defaults
+# GLOG_ROOT_DIR: Base directory where all GLOG components are found
+#
+# The following are set after configuration is done:
+# GLOG_FOUND
+# GLOG_INCLUDE_DIRS
+# GLOG_LIBRARIES
+# GLOG_LIBRARYRARY_DIRS
+
+include(FindPackageHandleStandardArgs)
+
+set(GLOG_ROOT_DIR "" CACHE PATH "Folder contains Google glog")
+
+if(WIN32)
+ find_path(GLOG_INCLUDE_DIR glog/logging.h
+ PATHS ${GLOG_ROOT_DIR}/src/windows)
+else()
+ find_path(GLOG_INCLUDE_DIR glog/logging.h
+ PATHS ${GLOG_ROOT_DIR})
+endif()
+
+if(MSVC)
+ find_library(GLOG_LIBRARY_RELEASE libglog_static
+ PATHS ${GLOG_ROOT_DIR}
+ PATH_SUFFIXES Release)
+
+ find_library(GLOG_LIBRARY_DEBUG libglog_static
+ PATHS ${GLOG_ROOT_DIR}
+ PATH_SUFFIXES Debug)
+
+ set(GLOG_LIBRARY optimized ${GLOG_LIBRARY_RELEASE} debug ${GLOG_LIBRARY_DEBUG})
+else()
+ find_library(GLOG_LIBRARY glog
+ PATHS ${GLOG_ROOT_DIR}
+ PATH_SUFFIXES
+ lib
+ lib64)
+endif()
+
+find_package_handle_standard_args(GLOG DEFAULT_MSG
+ GLOG_INCLUDE_DIR GLOG_LIBRARY)
+
+if(GLOG_FOUND)
+ set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR})
+ set(GLOG_LIBRARIES ${GLOG_LIBRARY})
+endif()
\ No newline at end of file
diff --git a/depends/dbcommon/CMake/FindJSON.cmake b/depends/dbcommon/CMake/FindJSON.cmake
new file mode 100644
index 0000000..a334948
--- /dev/null
+++ b/depends/dbcommon/CMake/FindJSON.cmake
@@ -0,0 +1,38 @@
+# - Find json
+# Find the native JSON headers and libraries.
+#
+# JSON_INCLUDE_DIRS - where to find json/json.h, etc.
+# JSON_LIBRARIES - List of libraries when using json.
+# JSON_FOUND - True if json found.
+
+#=============================================================================
+# Copyright 2006-2009 Kitware, Inc.
+# Copyright 2012 Rolf Eike Beer <ei...@sf-mail.de>
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of CMake, substitute the full
+# License text for the above reference.)
+
+# Look for the header file.
+find_path(JSON_INCLUDE_DIR NAMES json/json.h)
+mark_as_advanced(JSON_INCLUDE_DIR)
+
+# Look for the library (sorted from most current/relevant entry to least).
+find_library(JSON_LIBRARY NAMES jsoncpp
+)
+mark_as_advanced(JSON_LIBRARY)
+
+# handle the QUIETLY and REQUIRED arguments and set JSON_FOUND to TRUE if
+# all listed variables are TRUE
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(JSON DEFAULT_MSG JSON_INCLUDE_DIR JSON_LIBRARY)
+
+if(JSON_FOUND)
+ set(JSON_LIBRARIES ${JSON_LIBRARY})
+ set(JSON_INCLUDE_DIRS ${JSON_INCLUDE_DIR})
+endif()
diff --git a/depends/dbcommon/CMake/FindSnappy.cmake b/depends/dbcommon/CMake/FindSnappy.cmake
new file mode 100644
index 0000000..623d2d7
--- /dev/null
+++ b/depends/dbcommon/CMake/FindSnappy.cmake
@@ -0,0 +1,30 @@
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+# SNAPPY_ROOT_DIR Set this variable to the root installation of
+# Snappy if the module has problems finding
+# the proper installation path.
+#
+# Variables defined by this module:
+#
+# SNAPPY_FOUND System has Snappy libs/headers
+# SNAPPY_LIBRARIES The Snappy libraries
+# SNAPPY_INCLUDE_DIR The location of Snappy headers
+
+find_path(SNAPPY_INCLUDE_DIR
+ NAMES snappy.h
+ HINTS ${SNAPPY_ROOT_DIR}/include)
+
+find_library(SNAPPY_LIBRARIES
+ NAMES snappy
+ HINTS ${SNAPPY_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Snappy DEFAULT_MSG
+ SNAPPY_LIBRARIES
+ SNAPPY_INCLUDE_DIR)
+
+mark_as_advanced(
+ SNAPPY_ROOT_DIR
+ SNAPPY_LIBRARIES
+ SNAPPY_INCLUDE_DIR)
diff --git a/depends/dbcommon/CMake/Functions.cmake b/depends/dbcommon/CMake/Functions.cmake
new file mode 100644
index 0000000..a771b60
--- /dev/null
+++ b/depends/dbcommon/CMake/Functions.cmake
@@ -0,0 +1,46 @@
+FUNCTION(AUTO_SOURCES RETURN_VALUE PATTERN SOURCE_SUBDIRS)
+
+ IF ("${SOURCE_SUBDIRS}" STREQUAL "RECURSE")
+ SET(PATH ".")
+ IF (${ARGC} EQUAL 4)
+ LIST(GET ARGV 3 PATH)
+ ENDIF ()
+ ENDIF()
+
+ IF ("${SOURCE_SUBDIRS}" STREQUAL "RECURSE")
+ UNSET(${RETURN_VALUE})
+ FILE(GLOB SUBDIR_FILES "${PATH}/${PATTERN}")
+ LIST(APPEND ${RETURN_VALUE} ${SUBDIR_FILES})
+
+ FILE(GLOB SUBDIRS RELATIVE ${PATH} ${PATH}/*)
+
+ FOREACH(DIR ${SUBDIRS})
+ IF (IS_DIRECTORY ${PATH}/${DIR})
+ IF (NOT "${DIR}" STREQUAL "CMAKEFILES")
+ FILE(GLOB_RECURSE SUBDIR_FILES "${PATH}/${DIR}/${PATTERN}")
+ LIST(APPEND ${RETURN_VALUE} ${SUBDIR_FILES})
+ ENDIF()
+ ENDIF()
+ ENDFOREACH()
+ ELSE ()
+ FILE(GLOB ${RETURN_VALUE} "${PATTERN}")
+
+ FOREACH (PATH ${SOURCE_SUBDIRS})
+ FILE(GLOB SUBDIR_FILES "${PATH}/${PATTERN}")
+ LIST(APPEND ${RETURN_VALUE} ${SUBDIR_FILES})
+ ENDFOREACH(PATH ${SOURCE_SUBDIRS})
+ ENDIF ()
+
+ IF (${FILTER_OUT})
+ LIST(REMOVE_ITEM ${RETURN_VALUE} ${FILTER_OUT})
+ ENDIF()
+
+ SET(${RETURN_VALUE} ${${RETURN_VALUE}} PARENT_SCOPE)
+ENDFUNCTION(AUTO_SOURCES)
+
+FUNCTION(CONTAINS_STRING FILE SEARCH RETURN_VALUE)
+ FILE(STRINGS ${FILE} FILE_CONTENTS REGEX ".*${SEARCH}.*")
+ IF (FILE_CONTENTS)
+ SET(${RETURN_VALUE} TRUE PARENT_SCOPE)
+ ENDIF()
+ENDFUNCTION(CONTAINS_STRING)
diff --git a/depends/dbcommon/CMake/Options.cmake b/depends/dbcommon/CMake/Options.cmake
new file mode 100644
index 0000000..36841ea
--- /dev/null
+++ b/depends/dbcommon/CMake/Options.cmake
@@ -0,0 +1,71 @@
+##############################################################################
+# In this file we handle all env and customer's settings
+##############################################################################
+
+##############################################################################
+# Setup build and dependencies information
+##############################################################################
+SET(DEPENDENCY_INSTALL_PREFIX "/opt/dependency")
+IF($ENV{DEPENDENCY_INSTALL_PREFIX})
+ SET(DEPENDENCY_INSTALL_PREFIX $ENV{DEPENDENCY_INSTALL_PREFIX})
+ENDIF()
+
+SET(DEPENDENCY_DIST_PACKAGE_NAME "dependency-dist-package.tar.gz")
+IF($ENV{DEPENDENCY_DIST_PACKAGE_NAME})
+ SET(DEPENDENCY_DIST_PACKAGE_NAME $ENV{DEPENDENCY_DIST_PACKAGE_NAME})
+ENDIF()
+
+SET(CMAKE_PREFIX_PATH "${DEPENDENCY_INSTALL_PREFIX}" ${CMAKE_PREFIX_PATH})
+SET(CMAKE_PREFIX_PATH "${DEPENDENCY_INSTALL_PREFIX}/package" ${CMAKE_PREFIX_PATH})
+SET(CMAKE_PREFIX_PATH "${DEPENDENCY_INSTALL_PREFIX}/tools" ${CMAKE_PREFIX_PATH})
+
+SET(DEPENDENCY_LIBRARY_PATH "${DEPENDENCY_INSTALL_PREFIX}/package/lib:${DEPENDENCY_LIBRARY_PATH}")
+SET(DEPENDENCY_LIBRARY_PATH "${DEPENDENCY_INSTALL_PREFIX}/package/lib64:${DEPENDENCY_LIBRARY_PATH}")
+
+##############################################################################
+# Setup build flags
+##############################################################################
+OPTION(ENABLE_COVERAGE "enable code coverage." OFF)
+
+IF(NOT CMAKE_BUILD_TYPE)
+ SET(CMAKE_BUILD_TYPE Debug CACHE STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE)
+ENDIF(NOT CMAKE_BUILD_TYPE)
+
+IF(ENABLE_COVERAGE STREQUAL ON)
+ INCLUDE(CodeCoverage)
+ENDIF(ENABLE_COVERAGE STREQUAL ON)
+
+IF(CMAKE_BUILD_TYPE MATCHES Debug)
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0")
+ENDIF()
+
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fno-omit-frame-pointer -fno-strict-aliasing")
+
+IF(ENABLE_AVX STREQUAL ON)
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx -mno-avx2")
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DAVX_OPT")
+ELSE()
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mno-avx -mno-avx2")
+ENDIF()
+
+#c++11 is needed to provide thread saft singleton implementation.
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wno-deprecated-register")
+#-Rpass-missed=loop-vectorize -Wall -Wconversion
+
+IF(CMAKE_COMPILER_IS_CLANG)
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-limit-debug-info -stdlib=libc++ -DUSE_CLANG")
+ IF(OS_LINUX)
+ SET(CLANG_LDFLAGS "-lc++abi -lc++" ${CLANG_LDFLAGS})
+ ENDIF(OS_LINUX)
+ENDIF(CMAKE_COMPILER_IS_CLANG)
+
+TRY_COMPILE(INT64T_EQUAL_LONGLONG
+ ${CMAKE_BINARY_DIR}
+ ${CMAKE_CURRENT_SOURCE_DIR}/CMake/CMakeTestCompileInt64tType.cc
+ OUTPUT_VARIABLE OUTPUT)
+
+IF(INT64T_EQUAL_LONGLONG)
+ MESSAGE(STATUS "Checking whether int64_t is typedef to long long -- yes")
+ELSE(INT64T_EQUAL_LONGLONG)
+ MESSAGE(STATUS "Checking whether int64_t is typedef to long long -- no")
+ENDIF(INT64T_EQUAL_LONGLONG)
diff --git a/depends/dbcommon/CMake/Platform.cmake b/depends/dbcommon/CMake/Platform.cmake
new file mode 100644
index 0000000..1ee0238
--- /dev/null
+++ b/depends/dbcommon/CMake/Platform.cmake
@@ -0,0 +1,47 @@
+IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ SET(OS_LINUX true CACHE INTERNAL "Linux operating system")
+ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+ SET(OS_MACOSX true CACHE INTERNAL "Mac Darwin operating system")
+ELSE(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ MESSAGE(FATAL_ERROR "Unsupported OS: \"${CMAKE_SYSTEM_NAME}\"")
+ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+
+IF(CMAKE_COMPILER_IS_GNUCXX)
+ EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE COMPILER_OUTPUT)
+
+ STRING(REGEX MATCH "[0-9]\\.[0-9]\\.[0-9]" GCC_COMPILER_VERSION ${COMPILER_OUTPUT})
+ STRING(REGEX MATCHALL "[0-9]" GCC_COMPILER_VERSION ${GCC_COMPILER_VERSION})
+
+ LIST(LENGTH GCC_COMPILER_VERSION GCC_COMPILER_VERSION_LEN)
+ IF (NOT 3 EQUAL ${GCC_COMPILER_VERSION_LEN})
+ MESSAGE(FATAL_ERROR "Cannot get gcc version from \"${COMPILER_OUTPUT}\"")
+ ENDIF(NOT 3 EQUAL ${GCC_COMPILER_VERSION_LEN})
+
+ LIST(GET GCC_COMPILER_VERSION 0 GCC_COMPILER_VERSION_MAJOR)
+ LIST(GET GCC_COMPILER_VERSION 1 GCC_COMPILER_VERSION_MINOR)
+ LIST(GET GCC_COMPILER_VERSION 2 GCC_COMPILER_VERSION_PATCH)
+
+ SET(GCC_COMPILER_VERSION_MAJOR ${GCC_COMPILER_VERSION_MAJOR} CACHE INTERNAL "gcc major version")
+ SET(GCC_COMPILER_VERSION_MINOR ${GCC_COMPILER_VERSION_MINOR} CACHE INTERNAL "gcc minor version")
+ SET(GCC_COMPILER_VERSION_PATCH ${GCC_COMPILER_VERSION_PATCH} CACHE INTERNAL "gcc patch version")
+
+ MESSAGE(STATUS "checking compiler: GCC (${GCC_COMPILER_VERSION_MAJOR}.${GCC_COMPILER_VERSION_MINOR}.${GCC_COMPILER_VERSION_PATCH})")
+ELSE(CMAKE_COMPILER_IS_GNUCXX)
+ EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE COMPILER_OUTPUT)
+ IF(COMPILER_OUTPUT MATCHES "clang")
+ SET(CMAKE_COMPILER_IS_CLANG true CACHE INTERNAL "using clang as compiler")
+ MESSAGE(STATUS "checking compiler: CLANG")
+ ELSE(COMPILER_OUTPUT MATCHES "clang")
+ MESSAGE(FATAL_ERROR "Unsupported compiler: \"${CMAKE_CXX_COMPILER}\"")
+ ENDIF(COMPILER_OUTPUT MATCHES "clang")
+ENDIF(CMAKE_COMPILER_IS_GNUCXX)
+
+INCLUDE (TestBigEndian)
+TEST_BIG_ENDIAN(IS_BIG_ENDIAN)
+if(IS_BIG_ENDIAN)
+ message(STATUS "BIG_ENDIAN")
+ ADD_DEFINITIONS(-DIS_BIG_ENDIAN)
+else()
+ message(STATUS "LITTLE_ENDIAN")
+ ADD_DEFINITIONS(-DIS_LITTLE_ENDIAN)
+endif()
diff --git a/depends/dbcommon/CMakeLists.txt b/depends/dbcommon/CMakeLists.txt
new file mode 100644
index 0000000..dd15bfc
--- /dev/null
+++ b/depends/dbcommon/CMakeLists.txt
@@ -0,0 +1,28 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(dbcommon)
+
+##############################################################################
+# General CMake initialization
+##############################################################################
+SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
+SET(CMAKE_VERBOSE_MAKEFILE OFF CACHE STRING "Verbose build." FORCE)
+
+IF(${CMAKE_CURRENT_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR})
+ MESSAGE(FATAL_ERROR "cannot build the project in the source directory! Out-of-source build is enforced!")
+ENDIF()
+
+##############################################################################
+# Import env, customer settings and utilities
+##############################################################################
+INCLUDE(Functions)
+INCLUDE(Platform)
+INCLUDE(Options)
+
+ADD_SUBDIRECTORY(src)
+ADD_SUBDIRECTORY(test)
+
+ADD_CUSTOM_TARGET(coverage
+ COMMAND make resetcoverage
+ COMMAND make -j8 unittest
+ COMMAND make ucoverage
+ COMMENT "Run all unit tests and get coverage...")
diff --git a/depends/dbcommon/README b/depends/dbcommon/README
new file mode 100644
index 0000000..207ecb0
--- /dev/null
+++ b/depends/dbcommon/README
@@ -0,0 +1,14 @@
+## How to build
+
+ cd dbcommon
+ mkdir build
+ cd build
+ ../bootstrap --prefix=/opt/dependency/package (default)
+
+## Build
+
+ make
+
+## Install
+
+ make install
diff --git a/depends/dbcommon/bootstrap b/depends/dbcommon/bootstrap
new file mode 100755
index 0000000..1554c7b
--- /dev/null
+++ b/depends/dbcommon/bootstrap
@@ -0,0 +1,92 @@
+#!/bin/sh
+
+die() {
+ echo "$@" 1>&2 ; exit 1
+}
+
+arg ()
+{
+ echo "$1" | sed "s/^${2-[^=]*=}//"
+}
+
+# Detect directory information.
+source_dir=`cd "\`dirname \"$0\"\`";pwd`
+binary_dir=`pwd`
+
+# Choose the default install prefix.
+default_prefix="/opt/dependency/package"
+
+# Display bootstrap usage
+usage() {
+echo '
+Usage: '"$0"' [<options>]
+Options: [defaults in brackets after descriptions]
+Configuration:
+ --help print this message
+ --prefix=PREFIX install files in tree rooted at PREFIX
+ ['"${default_prefix}"']
+ --enable-coverage enable code coverage, must be used together with --enable-debug
+ --enable-debug enable debug build
+ --enable-avx enable avx for vector instruction optimization
+'
+ exit 10
+}
+
+# Parse arguments
+prefix_dir="${default_prefix}"
+build_type="Release"
+enable_coverage="OFF"
+enable_avx="ON"
+while test $# != 0; do
+ case "$1" in
+ --prefix=*) dir=`arg "$1"`
+ prefix_dir="$dir";;
+ --enable-coverage) enable_coverage="ON"
+ build_type="Debug";;
+ --enable-debug) build_type="Debug";;
+ --enable-avx=*) avx=`arg "$1"`
+ enable_avx="$avx";;
+ --help) usage ;;
+ *) die "Unknown option: $1" ;;
+ esac
+ shift
+done
+
+if [ ${source_dir} = ${binary_dir} ]; then
+ die "cannot build the project in the source directory! Out-of-source build is enforced!"
+fi
+
+enable_avx_upper=`echo "${enable_avx}" | tr [a-z] [A-Z]`
+if [ ${enable_avx_upper} != "ON" ] && [ ${enable_avx_upper} != "OFF" ]; then
+ die "unknown value for option enable-avx: ${enable_avx}, valid options are: on and off"
+fi
+
+# Check clang compiler
+if [[ x"${CC}" = x"" ]]; then
+ CC=gcc
+fi
+
+if [[ x"${CXX}" = x"" ]]; then
+ CXX=g++
+fi
+
+c_compiler=`which ${CC}`
+cxx_compiler=`which ${CXX}`
+cmake=`which cmake`
+
+if [ ! -x ${c_compiler} ]; then
+ die "cannot found c compiler"
+fi
+
+if [ ! -x ${cxx_compiler} ]; then
+ die "cannot found c++ compiler"
+fi
+
+if [ ! -x ${cmake} ]; then
+ die "cannot found cmake"
+fi
+
+# Configure
+${cmake} -DCMAKE_BUILD_TYPE=${build_type} -DCMAKE_INSTALL_PREFIX=${prefix_dir} -DCMAKE_C_COMPILER=${c_compiler} -DCMAKE_CXX_COMPILER=${cxx_compiler} -DENABLE_COVERAGE=${enable_coverage} -DENABLE_AVX=${enable_avx_upper} ${source_dir} || die "failed to configure the project"
+
+echo 'bootstrap success. Run "make" to build.'
diff --git a/depends/dbcommon/src/CMakeLists.txt b/depends/dbcommon/src/CMakeLists.txt
new file mode 100644
index 0000000..d322580
--- /dev/null
+++ b/depends/dbcommon/src/CMakeLists.txt
@@ -0,0 +1,154 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(dbcommon)
+
+FIND_PACKAGE(Glog REQUIRED)
+
+SET(dbcommon_VERSION_MAJOR 0)
+SET(dbcommon_VERSION_MINOR 1)
+SET(dbcommon_VERSION_PATCH 0)
+SET(dbcommon_VERSION_API 1)
+set(CMAKE_MACOSX_RPATH 1)
+
+SET(dbcommon_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+SET(dbcommon_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dbcommon)
+# cogapp generation
+find_package(Cogapp)
+SET(CODE_TEMPLATE_HEADERS
+ ${dbcommon_SRC_DIR}/function/arith-cmp-func.h
+ ${dbcommon_SRC_DIR}/function/typecast-func.h
+ ${dbcommon_SRC_DIR}/function/func-kind.h
+)
+
+SET(CODE_TEMPLATE_SOURCES
+ ${dbcommon_SRC_DIR}/function/arith-func.cc
+ ${dbcommon_SRC_DIR}/function/cmp-func.cc
+ ${dbcommon_SRC_DIR}/function/typecast-func.cc
+ ${dbcommon_SRC_DIR}/function/func.cc
+ ${dbcommon_SRC_DIR}/type/type-util.cc
+)
+
+COGAPP_GENERATE(TEMPLATE_GENERATED_HEADERS ${CODE_TEMPLATE_HEADERS})
+COGAPP_GENERATE(TEMPLATE_GENERATED_SOURCES ${CODE_TEMPLATE_SOURCES})
+
+set(thrift_SRC_DIR ${CMAKE_BINARY_DIR}/codegen/src/dbcommon/thrift)
+set(thrift_HDRS
+ ${thrift_SRC_DIR}/fb303_types.h
+ ${thrift_SRC_DIR}/fb303_constants.h
+ ${thrift_SRC_DIR}/FacebookService.h
+ ${thrift_SRC_DIR}/hive_metastore_types.h
+ ${thrift_SRC_DIR}/hive_metastore_constants.h
+ ${thrift_SRC_DIR}/ThriftHiveMetastore.h)
+set(thrift_SRCS
+ ${thrift_SRC_DIR}/fb303_types.cpp
+ ${thrift_SRC_DIR}/fb303_constants.cpp
+ ${thrift_SRC_DIR}/FacebookService.cpp
+ ${thrift_SRC_DIR}/hive_metastore_types.cpp
+ ${thrift_SRC_DIR}/hive_metastore_constants.cpp
+ ${thrift_SRC_DIR}/ThriftHiveMetastore.cpp)
+
+file(MAKE_DIRECTORY ${thrift_SRC_DIR})
+add_custom_command(
+OUTPUT ${thrift_HDRS} ${thrift_SRCS}
+COMMAND thrift --gen cpp --recurse
+ -out ${thrift_SRC_DIR}/
+ -I "${dbcommon_SRC_DIR}/filesystem/hive/"
+ "${dbcommon_SRC_DIR}/filesystem/hive/hive_metastore.thrift"
+)
+
+AUTO_SOURCES(log_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/log")
+AUTO_SOURCES(network_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/network")
+AUTO_SOURCES(nodes_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/nodes")
+AUTO_SOURCES(utils_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/utils")
+AUTO_SOURCES(function_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/function")
+AUTO_SOURCES(filesystem_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/filesystem")
+AUTO_SOURCES(type_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/type")
+AUTO_SOURCES(common_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/common")
+AUTO_SOURCES(thread_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/thread")
+AUTO_SOURCES(checksum_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/checksum")
+AUTO_SOURCES(hash_SRC "*.cc" "RECURSE" "${dbcommon_SRC_DIR}/hash")
+
+LIST(APPEND dbcommon_SOURCES ${log_SRC})
+LIST(APPEND dbcommon_SOURCES ${network_SRC})
+LIST(APPEND dbcommon_SOURCES ${nodes_SRC})
+LIST(APPEND dbcommon_SOURCES ${utils_SRC})
+LIST(APPEND dbcommon_SOURCES ${function_SRC})
+LIST(APPEND dbcommon_SOURCES ${filesystem_SRC})
+LIST(APPEND dbcommon_SOURCES ${type_SRC})
+LIST(APPEND dbcommon_SOURCES ${common_SRC})
+LIST(APPEND dbcommon_SOURCES ${thread_SRC})
+LIST(APPEND dbcommon_SOURCES ${checksum_SRC})
+LIST(APPEND dbcommon_SOURCES ${hash_SRC})
+
+LIST(REMOVE_ITEM dbcommon_SOURCES ${CODE_TEMPLATE_SOURCES})
+
+AUTO_SOURCES(log_HEADER "*.h" "${dbcommon_SRC_DIR}/log")
+AUTO_SOURCES(network_HEADER "*.h" "${dbcommon_SRC_DIR}/network")
+AUTO_SOURCES(nodes_HEADER "*.h" "${dbcommon_SRC_DIR}/nodes")
+AUTO_SOURCES(utils_HEADER "*.h" "${dbcommon_SRC_DIR}/utils")
+AUTO_SOURCES(utils_mb_HEADER "*.h" "${dbcommon_SRC_DIR}/utils/mb")
+AUTO_SOURCES(utils_comp_HEADER "*.h" "${dbcommon_SRC_DIR}/utils/comp")
+AUTO_SOURCES(function_HEADER "*.h" "${dbcommon_SRC_DIR}/function")
+AUTO_SOURCES(filesystem_HEADER "*.h" "${dbcommon_SRC_DIR}/filesystem")
+AUTO_SOURCES(filesystem_hdfs_HEADER "*.h" "${dbcommon_SRC_DIR}/filesystem/hdfs")
+AUTO_SOURCES(filesystem_hive_HEADER "*.h" "${dbcommon_SRC_DIR}/filesystem/hive")
+AUTO_SOURCES(filesystem_local_HEADER "*.h" "${dbcommon_SRC_DIR}/filesystem/local")
+AUTO_SOURCES(type_HEADER "*.h" "${dbcommon_SRC_DIR}/type")
+AUTO_SOURCES(common_HEADER "*.h" "${dbcommon_SRC_DIR}/common")
+AUTO_SOURCES(common_vector_HEADER "*.h" "${dbcommon_SRC_DIR}/common/vector")
+AUTO_SOURCES(thread_HEADER "*.h" "${dbcommon_SRC_DIR}/thread")
+AUTO_SOURCES(checksum_HEADER "*.h" "${dbcommon_SRC_DIR}/checksum")
+AUTO_SOURCES(testutil_HEADER "*.h" "${dbcommon_SRC_DIR}/testutil")
+AUTO_SOURCES(hash_HEADER "*.h" "${dbcommon_SRC_DIR}/hash")
+
+LIST(REMOVE_ITEM function_HEADER ${CODE_TEMPLATE_HEADERS})
+
+SET(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "")
+SET(CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES "")
+INCLUDE_DIRECTORIES(${dbcommon_ROOT_DIR})
+INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/codegen/src)
+INCLUDE_DIRECTORIES(${GLOG_INCLUDE_DIRS})
+INCLUDE_DIRECTORIES(${DEPENDENCY_INSTALL_PREFIX}/package/include)
+INCLUDE_DIRECTORIES(/usr/local/include)
+LINK_DIRECTORIES(${DEPENDENCY_INSTALL_PREFIX}/package/lib)
+LINK_DIRECTORIES(/usr/local/lib)
+
+ADD_LIBRARY(dbcommon-shared SHARED
+ ${TEMPLATE_GENERATED_HEADERS}
+ ${TEMPLATE_GENERATED_SOURCES}
+ ${thrift_SRCS}
+ ${dbcommon_SOURCES}
+ )
+# ADD_LIBRARY(dbcommon-static STATIC ${dbcommon_SOURCES} ${TEMPLATE_GENERATED_HEADERS} ${TEMPLATE_GENERATED_SOURCES})
+
+TARGET_LINK_LIBRARIES(dbcommon-shared ${CLANG_LDFLAGS} ${GLOG_LIBRARIES} iconv pthread hdfs3 z lz4 thrift)
+# TARGET_LINK_LIBRARIES(dbcommon-static ${CLANG_LDFLAGS} ${GLOG_LIBRARIES} iconv pthread hdfs3)
+
+SET_TARGET_PROPERTIES(dbcommon-shared PROPERTIES OUTPUT_NAME "dbcommon")
+# SET_TARGET_PROPERTIES(dbcommon-static PROPERTIES OUTPUT_NAME "dbcommon")
+
+INSTALL(TARGETS dbcommon-shared
+ RUNTIME DESTINATION bin
+ LIBRARY DESTINATION lib
+ ARCHIVE DESTINATION lib)
+
+INSTALL(FILES ${log_HEADER} DESTINATION include/dbcommon/log)
+INSTALL(FILES ${network_HEADER} DESTINATION include/dbcommon/network)
+INSTALL(FILES ${nodes_HEADER} DESTINATION include/dbcommon/nodes)
+INSTALL(FILES ${utils_HEADER} DESTINATION include/dbcommon/utils)
+INSTALL(FILES ${common_HEADER} DESTINATION include/dbcommon/common)
+INSTALL(FILES ${common_vector_HEADER} DESTINATION include/dbcommon/common/vector)
+INSTALL(FILES ${function_HEADER} DESTINATION include/dbcommon/function)
+INSTALL(FILES ${TEMPLATE_GENERATED_HEADERS} DESTINATION include/dbcommon/function)
+INSTALL(FILES ${filesystem_HEADER} DESTINATION include/dbcommon/filesystem)
+INSTALL(FILES ${filesystem_hdfs_HEADER} DESTINATION include/dbcommon/filesystem/hdfs)
+INSTALL(FILES ${filesystem_hive_HEADER} DESTINATION include/dbcommon/filesystem/hive)
+INSTALL(FILES ${filesystem_local_HEADER} DESTINATION include/dbcommon/filesystem/local)
+INSTALL(FILES ${thread_HEADER} DESTINATION include/dbcommon/thread)
+INSTALL(FILES ${checksum_HEADER} DESTINATION include/dbcommon/checksum)
+INSTALL(FILES ${type_HEADER} DESTINATION include/dbcommon/type)
+INSTALL(FILES ${utils_mb_HEADER} DESTINATION include/dbcommon/utils/mb)
+INSTALL(FILES ${utils_comp_HEADER} DESTINATION include/dbcommon/utils/comp)
+INSTALL(FILES ${testutil_HEADER} DESTINATION include/dbcommon/testutil)
+INSTALL(FILES ${hash_HEADER} DESTINATION include/dbcommon/hash)
+
+SET(dbcommon_ROOT_DIR ${dbcommon_ROOT_DIR} PARENT_SCOPE)
diff --git a/depends/dbcommon/src/dbcommon/checksum/checksum-util.cc b/depends/dbcommon/src/dbcommon/checksum/checksum-util.cc
new file mode 100644
index 0000000..1246628
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/checksum/checksum-util.cc
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "dbcommon/checksum/checksum-util.h"
+#include <cassert>
+#include "dbcommon/checksum/hw-crc32c.h"
+#include "dbcommon/checksum/sw-crc32c.h"
+
+namespace dbcommon {
+
+ChecksumUtil::ChecksumUtil() {
+ if (HWCrc32c::available()) {
+ chk.reset(new HWCrc32c());
+ } else {
+ chk.reset(new SWCrc32c());
+ }
+ assert(chk);
+}
+
+uint32_t ChecksumUtil::getValue() { return chk->getValue(); }
+
+void ChecksumUtil::reset() { chk->reset(); }
+
+void ChecksumUtil::update(const void* b, int len) { chk->update(b, len); }
+
+} // namespace dbcommon
diff --git a/depends/dbcommon/src/dbcommon/checksum/checksum-util.h b/depends/dbcommon/src/dbcommon/checksum/checksum-util.h
new file mode 100644
index 0000000..2c09869
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/checksum/checksum-util.h
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_CHECKSUM_CHECKSUM_UTIL_H_
+#define DBCOMMON_SRC_DBCOMMON_CHECKSUM_CHECKSUM_UTIL_H_
+#include <memory>
+#include "dbcommon/checksum/checksum.h"
+
+namespace dbcommon {
+
+class ChecksumUtil {
+ public:
+ ChecksumUtil();
+ ~ChecksumUtil() {}
+
+ /**
+ * @return Returns the current checksum value.
+ */
+ uint32_t getValue();
+
+ /**
+ * Resets the checksum to its initial value.
+ */
+ void reset();
+
+ /**
+ * Updates the current checksum with the specified array of bytes.
+ * @param b The buffer of data.
+ * @param len The buffer length.
+ */
+ void update(const void* b, int len);
+
+ private:
+ std::unique_ptr<dbcommon::Checksum> chk;
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_CHECKSUM_CHECKSUM_UTIL_H_
diff --git a/depends/dbcommon/src/dbcommon/checksum/checksum.h b/depends/dbcommon/src/dbcommon/checksum/checksum.h
new file mode 100644
index 0000000..b05220f
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/checksum/checksum.h
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_CHECKSUM_CHECKSUM_H_
+#define DBCOMMON_SRC_DBCOMMON_CHECKSUM_CHECKSUM_H_
+
+#include <stdint.h>
+
+#define CHECKSUM_TYPE_SIZE 1
+#define CHECKSUM_BYTES_PER_CHECKSUM_SIZE 4
+#define CHECKSUM_TYPE_CRC32C 2
+
+namespace dbcommon {
+
+/**
+ * An abstract base CRC class.
+ */
+class Checksum {
+ public:
+ /**
+ * @return Returns the current checksum value.
+ */
+ virtual uint32_t getValue() = 0;
+
+ /**
+ * Resets the checksum to its initial value.
+ */
+ virtual void reset() = 0;
+
+ /**
+ * Updates the current checksum with the specified array of bytes.
+ * @param b The buffer of data.
+ * @param len The buffer length.
+ */
+ virtual void update(const void* b, int len) = 0;
+
+ /**
+ * Destroy the instance.
+ */
+ virtual ~Checksum() {}
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_CHECKSUM_CHECKSUM_H_
diff --git a/depends/dbcommon/src/dbcommon/checksum/hw-crc32c.cc b/depends/dbcommon/src/dbcommon/checksum/hw-crc32c.cc
new file mode 100644
index 0000000..53dab9a
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/checksum/hw-crc32c.cc
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <cassert>
+#include <cstdlib>
+
+#include "dbcommon/checksum/hw-crc32c.h"
+
+#if ((defined(__X86__) || defined(__i386__) || defined(i386) || \
+ defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || \
+ defined(_M_X64)))
+#include <cpuid.h>
+#endif
+
+#if ((defined(__X86__) || defined(__i386__) || defined(i386) || \
+ defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || \
+ defined(_M_X64)))
+#if !defined(__SSE4_2__)
+
+namespace dbcommon {
+
+#if defined(__LP64__)
+static inline uint64_t _mm_crc32_u64(uint64_t crc, uint64_t value) {
+ asm("crc32q %[value], %[crc]\n" : [crc] "+r"(crc) : [value] "rm"(value));
+ return crc;
+}
+#endif
+
+static inline uint32_t _mm_crc32_u16(uint32_t crc, uint16_t value) {
+ asm("crc32w %[value], %[crc]\n" : [crc] "+r"(crc) : [value] "rm"(value));
+ return crc;
+}
+
+static inline uint32_t _mm_crc32_u32(uint32_t crc, uint64_t value) {
+ asm("crc32l %[value], %[crc]\n" : [crc] "+r"(crc) : [value] "rm"(value));
+ return crc;
+}
+
+static inline uint32_t _mm_crc32_u8(uint32_t crc, uint8_t value) {
+ asm("crc32b %[value], %[crc]\n" : [crc] "+r"(crc) : [value] "rm"(value));
+ return crc;
+}
+
+} // namespace dbcommon
+
+#else
+
+#include <nmmintrin.h>
+
+#endif
+
+namespace dbcommon {
+
+bool HWCrc32c::available() {
+#if ((defined(__X86__) || defined(__i386__) || defined(i386) || \
+ defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || \
+ defined(_M_X64)))
+ uint32_t eax, ebx, ecx = 0, edx;
+ /*
+ * get the CPU features (level 1). ecx will have the SSE4.2 bit.
+ * This gcc routine automatically handles saving ebx in the case where we are
+ * -fpic or -fPIC
+ */
+ __get_cpuid(1, &eax, &ebx, &ecx, &edx);
+ return (ecx & (1 << 20)) != 0;
+#else
+ return false;
+#endif
+}
+
+void HWCrc32c::update(const void *b, int len) {
+ const char *p = static_cast<const char *>(b);
+#if defined(__LP64__)
+ const size_t bytes = sizeof(uint64_t);
+#else
+ const size_t bytes = sizeof(uint32_t);
+#endif
+ int align = bytes - reinterpret_cast<uint64_t>(p) % bytes;
+ align = bytes == static_cast<size_t>(align) ? 0 : align;
+
+ if (len < align) {
+ align = len;
+ }
+
+ updateInt64(p, align);
+ p = p + align;
+ len -= align;
+
+ if (len > 0) {
+ assert(0 == reinterpret_cast<uint64_t>(p) % bytes);
+
+ for (int i = len / bytes; i > 0; --i) {
+#if defined(__LP64__)
+ crc = _mm_crc32_u64(crc, *reinterpret_cast<const uint64_t *>(p));
+#else
+ crc = _mm_crc32_u32(crc, *reinterpret_cast<const uint32_t *>(p));
+#endif
+ p = p + bytes;
+ }
+
+ len &= bytes - 1;
+ updateInt64(p, len);
+ }
+}
+
+void HWCrc32c::updateInt64(const char *b, int len) {
+ assert(len < 8);
+
+ switch (len) {
+ case 7:
+ crc = _mm_crc32_u8(crc, *reinterpret_cast<const uint8_t *>(b++));
+
+ case 6:
+ crc = _mm_crc32_u16(crc, *reinterpret_cast<const uint16_t *>(b));
+ b += 2;
+
+ /* case 5 is below: 4 + 1 */
+ case 4:
+ crc = _mm_crc32_u32(crc, *reinterpret_cast<const uint32_t *>(b));
+ break;
+
+ case 3:
+ crc = _mm_crc32_u8(crc, *reinterpret_cast<const uint8_t *>(b++));
+
+ case 2:
+ crc = _mm_crc32_u16(crc, *reinterpret_cast<const uint16_t *>(b));
+ break;
+
+ case 5:
+ crc = _mm_crc32_u32(crc, *reinterpret_cast<const uint32_t *>(b));
+ b += 4;
+
+ case 1:
+ crc = _mm_crc32_u8(crc, *reinterpret_cast<const uint8_t *>(b));
+ break;
+
+ case 0:
+ break;
+ }
+}
+
+} // namespace dbcommon
+
+#endif
diff --git a/depends/dbcommon/src/dbcommon/checksum/hw-crc32c.h b/depends/dbcommon/src/dbcommon/checksum/hw-crc32c.h
new file mode 100644
index 0000000..474b89f
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/checksum/hw-crc32c.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_CHECKSUM_HW_CRC32C_H_
+#define DBCOMMON_SRC_DBCOMMON_CHECKSUM_HW_CRC32C_H_
+
+#include "dbcommon/checksum/checksum.h"
+
+namespace dbcommon {
+
+/**
+ * Calculate CRC with hardware support.
+ */
+class HWCrc32c : public Checksum {
+ public:
+ /**
+ * Constructor.
+ */
+ HWCrc32c() : crc(0xFFFFFFFF) {}
+
+ uint32_t getValue() { return ~crc; }
+
+ /**
+ * @ref Checksum#reset()
+ */
+ void reset() { crc = 0xFFFFFFFF; }
+
+ /**
+ * @ref Checksum#update(const void *, int)
+ */
+ void update(const void* b, int len);
+
+ /**
+ * Destory an HWCrc32 instance.
+ */
+ ~HWCrc32c() {}
+
+ /**
+ * To test if the hardware support this function.
+ * @return true if the hardware support to calculate the CRC.
+ */
+ static bool available();
+
+ private:
+ void updateInt64(const char* b, int len);
+
+ private:
+ uint32_t crc;
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_CHECKSUM_HW_CRC32C_H_
diff --git a/depends/dbcommon/src/dbcommon/checksum/sw-crc32c.cc b/depends/dbcommon/src/dbcommon/checksum/sw-crc32c.cc
new file mode 100644
index 0000000..2a96dbe
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/checksum/sw-crc32c.cc
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "dbcommon/checksum/sw-crc32c.h"
+
+namespace dbcommon {
+
+/*
+ * The following CRC lookup table was generated automagically
+ * using the following model parameters:
+ *
+ * Generator Polynomial = ................. 0x1EDC6F41
+ * Generator Polynomial Length = .......... 32 bits
+ * Reflected Bits = ....................... TRUE
+ * Table Generation Offset = .............. 32 bits
+ * Number of Slices = ..................... 8 slices
+ * Slice Lengths = ........................ 8 8 8 8 8 8 8 8
+ */
+
+static const uint32_t crc_tableil8_o32[256] = {
+ 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C,
+ 0x26A1E7E8, 0xD4CA64EB, 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
+ 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, 0x105EC76F, 0xE235446C,
+ 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,
+ 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, 0x5D1D08BF, 0xAF768BBC,
+ 0xBC267848, 0x4E4DFB4B, 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A,
+ 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, 0xAA64D611, 0x580F5512,
+ 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,
+ 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD,
+ 0x1642AE59, 0xE4292D5A, 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A,
+ 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, 0x417B1DBC, 0xB3109EBF,
+ 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,
+ 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F,
+ 0xED03A29B, 0x1F682198, 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927,
+ 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, 0xDBFC821C, 0x2997011F,
+ 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,
+ 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, 0xA65C047D, 0x5437877E,
+ 0x4767748A, 0xB50CF789, 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859,
+ 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, 0x7198540D, 0x83F3D70E,
+ 0x90A324FA, 0x62C8A7F9, 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,
+ 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE,
+ 0xDDE0EB2A, 0x2F8B6829, 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C,
+ 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, 0x082F63B7, 0xFA44E0B4,
+ 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,
+ 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B,
+ 0xB4091BFF, 0x466298FC, 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C,
+ 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, 0xA24BB5A6, 0x502036A5,
+ 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,
+ 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, 0xEF087A76, 0x1D63F975,
+ 0x0E330A81, 0xFC588982, 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D,
+ 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, 0x38CC2A06, 0xCAA7A905,
+ 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,
+ 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8,
+ 0xE52CC12C, 0x1747422F, 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF,
+ 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, 0xD3D3E1AB, 0x21B862A8,
+ 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,
+ 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78,
+ 0x7FAB5E8C, 0x8DC0DD8F, 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE,
+ 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, 0x69E9F0D5, 0x9B8273D6,
+ 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,
+ 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 0x34F4F86A, 0xC69F7B69,
+ 0xD5CF889D, 0x27A40B9E, 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
+ 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351};
+
+/*
+ * end of the CRC lookup table crc_tableil8_o32
+ */
+
+void SWCrc32c::update(const void *b, int len) {
+ const char *p = static_cast<const char *>(b);
+ const char *e = p + len;
+
+ while (p < e) {
+ crc = crc_tableil8_o32[(crc ^ *p++) & 0x000000FF] ^ (crc >> 8);
+ }
+}
+
+} // namespace dbcommon
diff --git a/depends/dbcommon/src/dbcommon/checksum/sw-crc32c.h b/depends/dbcommon/src/dbcommon/checksum/sw-crc32c.h
new file mode 100644
index 0000000..30015ad
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/checksum/sw-crc32c.h
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_CHECKSUM_SW_CRC32C_H_
+#define DBCOMMON_SRC_DBCOMMON_CHECKSUM_SW_CRC32C_H_
+
+#include "dbcommon/checksum/checksum.h"
+
+namespace dbcommon {
+
+class SWCrc32c : public Checksum {
+ public:
+ SWCrc32c() : crc(0xFFFFFFFF) {}
+
+ uint32_t getValue() { return ~crc; }
+
+ void reset() { crc = 0xFFFFFFFF; }
+
+ void update(const void* b, int len);
+
+ ~SWCrc32c() {}
+
+ private:
+ uint32_t crc;
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_CHECKSUM_SW_CRC32C_H_
diff --git a/depends/dbcommon/src/dbcommon/common/node-deserializer.cc b/depends/dbcommon/src/dbcommon/common/node-deserializer.cc
new file mode 100644
index 0000000..3ec3db1
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/node-deserializer.cc
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "dbcommon/common/node-deserializer.h"
+
+#include <cassert>
+#include <cstring>
+
+namespace dbcommon {
+
+const char *NodeDeserializer::readBytes(size_t length) {
+ assert(length >= 0 && cursor + length <= size && "index out of range");
+ cursor += length;
+ return buffer + cursor - length;
+}
+
+} // namespace dbcommon
diff --git a/depends/dbcommon/src/dbcommon/common/node-deserializer.h b/depends/dbcommon/src/dbcommon/common/node-deserializer.h
new file mode 100644
index 0000000..d15823c
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/node-deserializer.h
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_NODE_DESERIALIZER_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_NODE_DESERIALIZER_H_
+
+#include <cassert>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace dbcommon {
+
+#define BUFFER_CHECK_INDEX(type, nelement, cursor, size) \
+ assert(((cursor) + sizeof(type) * (nelement) <= (size)) && \
+ "index out of range")
+
+class NodeDeserializer {
+ public:
+ explicit NodeDeserializer(const std::string &buf)
+ : buffer(buf.data()), cursor(0), size(buf.length()) {}
+ NodeDeserializer(const char *buf, size_t len)
+ : buffer(buf), cursor(0), size(len) {}
+
+ NodeDeserializer(const NodeDeserializer &other) = delete;
+ NodeDeserializer &operator=(const NodeDeserializer &other) = delete;
+ NodeDeserializer(NodeDeserializer &&other) = delete;
+ NodeDeserializer &operator=(NodeDeserializer &&other) = delete;
+
+ ~NodeDeserializer() {}
+
+ void readBegin() {}
+
+ template <typename T>
+ T read() {
+ static_assert(
+ std::is_fundamental<T>::value == true || std::is_enum<T>::value == true,
+ "T2 is fundamental");
+
+ BUFFER_CHECK_INDEX(T, 1, cursor, size);
+ T retval = *reinterpret_cast<const T *>(buffer + cursor);
+ cursor += sizeof(T);
+ return retval;
+ }
+
+ const char *readBytes(size_t length);
+
+ size_t getCursor() { return cursor; }
+ void setCursor(size_t cursor_) { cursor = cursor_; }
+
+ private:
+ const char *buffer;
+ size_t cursor;
+ size_t size;
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_NODE_DESERIALIZER_H_
diff --git a/depends/dbcommon/src/dbcommon/common/node-serializer.h b/depends/dbcommon/src/dbcommon/common/node-serializer.h
new file mode 100644
index 0000000..f46ed08
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/node-serializer.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_NODE_SERIALIZER_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_NODE_SERIALIZER_H_
+
+#include <cassert>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace dbcommon {
+
+class NodeSerializer {
+ public:
+ explicit NodeSerializer(std::string *buf) : buffer(buf) {}
+ virtual ~NodeSerializer() {}
+
+ template <typename T>
+ void write(T val) {
+ buffer->append(reinterpret_cast<const char *>(&val), sizeof(T));
+ }
+
+ void writeBytes(const char *buf, size_t length) {
+ assert(length >= 0);
+ if (length > 0) buffer->append(buf, length);
+ }
+
+ private:
+ std::string *buffer;
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_NODE_SERIALIZER_H_
diff --git a/depends/dbcommon/src/dbcommon/common/tuple-batch-store.cc b/depends/dbcommon/src/dbcommon/common/tuple-batch-store.cc
new file mode 100644
index 0000000..1404b6c
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/tuple-batch-store.cc
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <fstream>
+
+#include "dbcommon/common/node-serializer.h"
+#include "dbcommon/common/tuple-batch-store.h"
+#include "dbcommon/utils/byte-buffer.h"
+
+#define EOF_MARK ((size_t)(0xFFFFFFFFFFFFFFFF))
+
+namespace dbcommon {
+
+NTupleBatchStore::~NTupleBatchStore() {
+ if (file_) {
+ file_->close();
+ }
+}
+
+void NTupleBatchStore::writeEOF() {
+ if (file_) {
+ if (mode_ == Mode::OUTPUT) {
+ size_t sizeOfBuffer = EOF_MARK;
+ filesystem_->write(file_.get(), reinterpret_cast<char *>(&sizeOfBuffer),
+ sizeof(sizeOfBuffer));
+ }
+ }
+}
+
+void NTupleBatchStore::reset() {
+ if (file_) {
+ filesystem_->seek(file_.get(), 0);
+ } else {
+ tbsOffset_ = 0;
+ }
+}
+
+void NTupleBatchStore::PutIntoNTupleBatchStore(dbcommon::TupleBatch::uptr tb) {
+ assert(mode_ == Mode::IN_MEMEORY || mode_ == Mode::OUTPUT);
+ totalRows_ += tb->getNumOfRows();
+ if (mode_ == Mode::IN_MEMEORY) {
+ tb->materialize();
+ tbs_.push_back(std::move(tb));
+ } else {
+ size_t sizeBuf;
+ std::string dataBuf;
+ tb->serialize(&dataBuf, tb->getTupleBatchSize());
+ sizeBuf = dataBuf.size();
+ filesystem_->write(file_.get(), reinterpret_cast<char *>(&sizeBuf),
+ sizeof(sizeBuf));
+ filesystem_->write(file_.get(), dataBuf.data(), dataBuf.size());
+ }
+}
+
+void NTupleBatchStore::PutIntoNTupleBatchStore(dbcommon::TupleBatch *tb) {
+ assert(mode_ == Mode::IN_MEMEORY || mode_ == Mode::OUTPUT);
+ totalRows_ += tb->getNumOfRows();
+ if (mode_ == Mode::IN_MEMEORY) {
+ tbs_.push_back(tb->clone());
+ } else {
+ size_t sizeBuf;
+ std::string dataBuf;
+ tb->serialize(&dataBuf, tb->getTupleBatchSize());
+ sizeBuf = dataBuf.size();
+ filesystem_->write(file_.get(), reinterpret_cast<char *>(&sizeBuf),
+ sizeof(sizeBuf));
+ filesystem_->write(file_.get(), dataBuf.data(), dataBuf.size());
+ }
+}
+
+dbcommon::TupleBatch::uptr NTupleBatchStore::GetFromNTupleBatchStore() {
+ if (mode_ == Mode::IN_MEMEORY) {
+ if (tbsOffset_ < tbs_.size()) {
+ auto tb = tbs_[tbsOffset_++]->clone();
+ return std::move(tb);
+ } else {
+ return nullptr;
+ }
+ } else {
+ size_t sizeOfBuffer;
+ filesystem_->read(file_.get(), reinterpret_cast<char *>(&sizeOfBuffer),
+ sizeof(sizeOfBuffer));
+ if (sizeOfBuffer == EOF_MARK) return nullptr;
+ buffer_.resize(sizeOfBuffer);
+ filesystem_->read(file_.get(), buffer_.data(), sizeOfBuffer);
+ TupleBatch::uptr tb(new TupleBatch);
+ tb->deserialize(std::string(buffer_.data(), buffer_.size()));
+ return std::move(tb);
+ }
+}
+
+} // namespace dbcommon
diff --git a/depends/dbcommon/src/dbcommon/common/tuple-batch-store.h b/depends/dbcommon/src/dbcommon/common/tuple-batch-store.h
new file mode 100644
index 0000000..6c8bea9
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/tuple-batch-store.h
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_TUPLE_BATCH_STORE_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_TUPLE_BATCH_STORE_H_
+
+#include <fstream>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "dbcommon/common/tuple-batch.h"
+#include "dbcommon/filesystem/file-system.h"
+#include "dbcommon/utils/byte-buffer.h"
+
+namespace dbcommon {
+
+class NTupleBatchStore {
+ public:
+ enum Mode { IN_MEMEORY, INPUT, OUTPUT };
+
+ NTupleBatchStore() : mode_(Mode::IN_MEMEORY), buffer_(false) {}
+
+ NTupleBatchStore(FileSystem *filesystem, const std::string &filename,
+ Mode mode)
+ : mode_(mode), buffer_(true), filesystem_(filesystem) {
+ if (mode == Mode::INPUT) {
+ file_ = filesystem_->open(filename, O_RDONLY);
+ } else if (mode == Mode::OUTPUT) {
+ filesystem_->removeIfExists(filename);
+ file_ = filesystem_->open(filename, O_WRONLY);
+ }
+ }
+
+ ~NTupleBatchStore();
+
+ void writeEOF();
+
+ void reset();
+
+ int64_t getTotalRows() const { return totalRows_; }
+
+ NTupleBatchStore(const NTupleBatchStore &other) = delete;
+ NTupleBatchStore &operator=(const NTupleBatchStore &other) = delete;
+ NTupleBatchStore(NTupleBatchStore &&other) = delete;
+ NTupleBatchStore &operator=(NTupleBatchStore &&other) = delete;
+
+ typedef std::unique_ptr<NTupleBatchStore> uptr;
+
+ void PutIntoNTupleBatchStore(dbcommon::TupleBatch::uptr tb);
+ void PutIntoNTupleBatchStore(dbcommon::TupleBatch *tb);
+
+ dbcommon::TupleBatch::uptr GetFromNTupleBatchStore();
+
+ private:
+ Mode mode_;
+ ByteBuffer buffer_;
+ FileSystem *filesystem_ = nullptr;
+ std::unique_ptr<File> file_ = nullptr;
+ std::vector<std::unique_ptr<TupleBatch>> tbs_;
+ int32_t tbsOffset_ = 0;
+ int64_t totalRows_ = 0;
+};
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_TUPLE_BATCH_STORE_H_
diff --git a/depends/dbcommon/src/dbcommon/common/tuple-batch.cc b/depends/dbcommon/src/dbcommon/common/tuple-batch.cc
new file mode 100644
index 0000000..b4c92d0
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/tuple-batch.cc
@@ -0,0 +1,860 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "dbcommon/common/tuple-batch.h"
+
+#include <algorithm>
+#include <cassert>
+
+#include "dbcommon/common/vector/decimal-vector.h"
+#include "dbcommon/common/vector/fixed-length-vector.h"
+
+namespace dbcommon {
+
+TupleBatch::TupleBatch(bool hasSysCol) {
+ if (hasSysCol) {
+ addSysColumn(dbcommon::TypeKind::MAGMATID, -1, true);
+ }
+}
+
+TupleBatch::TupleBatch(uint32_t nfield, bool hasSysCol)
+ : vectorRefs(nfield), vectors(nfield) {
+ if (hasSysCol) {
+ addSysColumn(dbcommon::TypeKind::MAGMATID, -1, true);
+ }
+}
+
+TupleBatch::TupleBatch(int32_t nfield, bool hasSysCol)
+ : vectorRefs(nfield), vectors(nfield) {
+ if (hasSysCol) {
+ addSysColumn(dbcommon::TypeKind::MAGMATID, -1, true);
+ }
+}
+
+TupleBatch::TupleBatch(size_t nfield, bool hasSysCol)
+ : vectorRefs(nfield), vectors(nfield) {
+ if (hasSysCol) {
+ addSysColumn(dbcommon::TypeKind::MAGMATID, -1, true);
+ }
+}
+
+TupleBatch::TupleBatch(const TupleDesc &desc, bool ownData, bool hasSysCol) {
+ rebuild(desc, ownData, hasSysCol);
+}
+
+void TupleBatch::rebuild(const TupleDesc &desc, bool ownData, bool hasSysCol) {
+ // add user vectors
+ size_t attNum = desc.getNumOfColumns();
+ for (size_t i = 0; i < attNum; i++) {
+ std::unique_ptr<Vector> vect = Vector::BuildVector(
+ desc.getColumnType(i), ownData, desc.getColumnTypeModifier(i));
+ std::unique_ptr<Vector> child;
+ switch (desc.getColumnType(i)) {
+ case SMALLINTARRAYID:
+ child = std::unique_ptr<Vector>(new SmallIntVector(ownData));
+ vect->addChildVector(std::move(child));
+ break;
+ case INTARRAYID:
+ child = std::unique_ptr<Vector>(new IntVector(ownData));
+ vect->addChildVector(std::move(child));
+ break;
+ case BIGINTARRAYID:
+ child = std::unique_ptr<Vector>(new BigIntVector(ownData));
+ vect->addChildVector(std::move(child));
+ break;
+ case FLOATARRAYID:
+ child = std::unique_ptr<Vector>(new FloatVector(ownData));
+ vect->addChildVector(std::move(child));
+ break;
+ case DOUBLEARRAYID:
+ child = std::unique_ptr<Vector>(new DoubleVector(ownData));
+ vect->addChildVector(std::move(child));
+ break;
+ default:
+ break;
+ }
+ vect->setParentTupleBatch(this);
+ vectorRefs.push_back(vect.get());
+ vectors.push_back(std::move(vect));
+ }
+
+ // add system vectors
+ if (hasSysCol) {
+ addSysColumn(dbcommon::TypeKind::MAGMATID, -1, true);
+ }
+}
+
+std::string TupleBatch::toString() const {
+ const TupleBatchReader &reader = this->getTupleBatchReader();
+ const TupleBatchReader &sysReader = this->getTupleBatchSysReader();
+ std::stringstream ss;
+
+ if (isScalarTupleBatch_) {
+ for (size_t i = 0; i < scalars_.size(); i++) {
+ // XXX(FIXME): Not support to print data types other than integers
+ std::string tmp;
+ if (scalars_[i].length == 0)
+ tmp = std::to_string(DatumGetValue<int64_t>(scalars_[i].value));
+ else
+ tmp.append(DatumGetValue<char *>(scalars_[i].value),
+ scalars_[i].length);
+
+ if (!scalars_[i].isnull) ss << tmp;
+ if (i < scalars_.size() - 1) ss << ",";
+ }
+ ss << "\n";
+ return ss.str();
+ }
+
+ for (size_t r = 0; r < this->getNumOfRows(); r++) {
+ for (size_t f = 0; f < this->getNumOfSysColumns(); f++) {
+ bool isNull;
+ if (reader[f] != nullptr) {
+ std::string tmp = sysReader[f]->read(r, &isNull);
+ if (!isNull) ss << tmp;
+ }
+ if (f <= this->getNumOfColumns() - 1) {
+ ss << ",";
+ }
+ }
+ for (size_t f = 0; f < this->getNumOfColumns(); f++) {
+ bool isNull;
+ if (reader[f] != nullptr) {
+ std::string tmp = reader[f]->read(r, &isNull);
+ if (!isNull) ss << tmp;
+ }
+ if (f < this->getNumOfColumns() - 1) {
+ ss << ",";
+ }
+ }
+ ss << "\n";
+ }
+ return ss.str();
+}
+
+int64_t TupleBatch::getTupleBatchSize() const {
+ int64_t size = 0;
+ for (uint32_t i = 0; i < vectorRefs.size(); i++) {
+ // std::unique_ptr<Vector> vec = vectors.at(i);
+ if (vectorRefs[i]) {
+ size += vectorRefs.at(i)->getMemorySizePlain();
+ }
+ }
+ return size;
+}
+
+void TupleBatch::addColumn(TypeKind dataType, int64_t typeMod, bool ownData) {
+ std::unique_ptr<Vector> vect =
+ Vector::BuildVector(dataType, ownData, typeMod);
+ vect->setParentTupleBatch(this);
+ vectorRefs.push_back(vect.get());
+ vectors.push_back(std::move(vect));
+}
+
+void TupleBatch::addSysColumn(TypeKind dataType, int64_t typeMod,
+ bool ownData) {
+ std::unique_ptr<Vector> vect =
+ Vector::BuildVector(dataType, ownData, typeMod);
+ vect->setParentTupleBatch(this);
+ sysVectorRefs.push_back(vect.get());
+ sysVectors.push_back(std::move(vect));
+}
+
+void TupleBatch::setColumn(int32_t index, std::unique_ptr<Vector> vect,
+ bool useParentSelected) {
+ if (index >= 0) {
+ assert(index < vectors.size() && "index out of range");
+ if (vect) vect->setParentTupleBatch(this);
+ vectorRefs[index] = (vect.get());
+ vectors[index].swap(vect);
+
+ if (useParentSelected) {
+ if (vectors[index])
+ vectors[index]->setSelected(this->getSelected(), true);
+ }
+ } else {
+ index = -index - 1;
+ assert(index < sysVectors.size() && "index out of range");
+ if (vect) vect->setParentTupleBatch(this);
+ sysVectorRefs[index] = vect.get();
+ sysVectors[index].swap(vect);
+
+ if (useParentSelected) {
+ if (sysVectors[index])
+ sysVectors[index]->setSelected(this->getSelected(), true);
+ }
+ }
+}
+
+void TupleBatch::setColumn(int32_t index, Vector *vect,
+ bool useParentSelected) {
+ if (index >= 0) {
+ assert(index < vectors.size() && "index out of range");
+ if (vect) vect->setParentTupleBatch(this);
+ vectorRefs[index] = vect;
+ vectors[index].reset(nullptr);
+
+ if (useParentSelected) {
+ if (vectors[index])
+ vectors[index]->setSelected(this->getSelected(), true);
+ }
+ } else {
+ index = -index - 1;
+ assert(index < sysVectors.size() && "index out of range");
+ if (vect) vect->setParentTupleBatch(this);
+ sysVectorRefs[index] = vect;
+ sysVectors[index].reset(nullptr);
+
+ if (useParentSelected) {
+ if (sysVectors[index])
+ sysVectors[index]->setSelected(this->getSelected(), true);
+ }
+ }
+}
+
+void TupleBatch::insertColumn(int32_t index, TypeKind dataType, int64_t typeMod,
+ bool ownData) {
+ auto vect = Vector::BuildVector(dataType, ownData, typeMod);
+
+ if (index >= 0) {
+ assert(index <= vectors.size() && "index out of range");
+ if (vect) vect->setParentTupleBatch(this);
+
+ vectorRefs.insert(vectorRefs.begin() + index, vect.get());
+ vectors.insert(vectors.begin() + index, std::move(vect));
+ } else {
+ index = -index - 1;
+ assert(index <= sysVectors.size() && "index out of range");
+
+ if (vect) vect->setParentTupleBatch(this);
+
+ sysVectorRefs.insert(sysVectorRefs.begin() + index, vect.get());
+ sysVectors.insert(sysVectors.begin() + index, std::move(vect));
+ }
+}
+
+void TupleBatch::insertColumn(int32_t index, std::unique_ptr<Vector> vect,
+ bool useParentSelected) {
+ if (index >= 0) {
+ assert(index <= vectors.size() && "index out of range");
+ if (vect) vect->setParentTupleBatch(this);
+
+ vectorRefs.insert(vectorRefs.begin() + index, vect.get());
+ vectors.insert(vectors.begin() + index, std::move(vect));
+
+ if (useParentSelected) {
+ if (vectors[index])
+ vectors[index]->setSelected(this->getSelected(), true);
+ }
+
+ } else {
+ index = -index - 1;
+ assert(index <= sysVectors.size() && "index out of range");
+
+ if (vect) vect->setParentTupleBatch(this);
+
+ sysVectorRefs.insert(sysVectorRefs.begin() + index, vect.get());
+ sysVectors.insert(sysVectors.begin() + index, std::move(vect));
+
+ if (useParentSelected) {
+ if (sysVectors[index])
+ sysVectors[index]->setSelected(this->getSelected(), true);
+ }
+ }
+}
+
+void TupleBatch::setNull(uint32_t index) {
+ for (auto &col : vectors) {
+ if (col != nullptr) col->setNull(index);
+ }
+ for (auto &col : sysVectors) {
+ if (col != nullptr) col->setNull(index);
+ }
+}
+
+void TupleBatch::appendNull(uint32_t tupleCount) {
+ for (auto &col : vectors) {
+ if (col != nullptr) {
+ for (uint32_t i = 0; i < tupleCount; ++i) col->append("", true);
+ }
+ }
+ for (auto &col : sysVectors) {
+ if (col != nullptr) {
+ for (uint32_t i = 0; i < tupleCount; ++i) col->append("", true);
+ }
+ }
+ this->numOfRowsPlain += tupleCount;
+}
+
+void TupleBatch::materialize() {
+ for (auto &col : vectors) {
+ if (col != nullptr) {
+ col->materialize();
+ }
+ }
+ for (auto &col : sysVectors) {
+ if (col != nullptr) {
+ col->materialize();
+ }
+ }
+ if (selected_) {
+ this->setNumOfRows(selected_->size());
+ selected_ = nullptr;
+ selectListOwner_.reset();
+ }
+}
+
+std::unique_ptr<TupleBatch> TupleBatch::cloneSelected(
+ const SelectList *sel) const {
+ int numCols = getNumOfColumns();
+ int numSysCols = getNumOfSysColumns();
+ auto retval =
+ std::unique_ptr<TupleBatch>(new TupleBatch(numCols, numSysCols > 0));
+ for (int i = 0; i < numCols; i++) {
+ if (vectorRefs[i])
+ retval->setColumn(i, std::move(vectorRefs[i]->cloneSelected(sel)), false);
+ }
+ for (int i = 0; i < numSysCols; i++) {
+ retval->setColumn((-i - 1), std::move(sysVectorRefs[i]->cloneSelected(sel)),
+ false);
+ }
+ if (sel && sel->size() > 0)
+ retval->setNumOfRows(sel->size());
+ else
+ retval->setNumOfRows(this->getNumOfRows());
+
+ return std::move(retval);
+}
+
+void TupleBatch::setSelected(const SelectList &selected) {
+ if (!selectListOwner_) {
+ selectListOwner_.reset(new SelectList);
+ selected_ = selectListOwner_.get();
+ }
+ *this->selected_ = selected;
+ for (auto &col : vectors) {
+ if (col) {
+ col->setSelected(selected_, true);
+ }
+ }
+ for (auto &col : sysVectors) {
+ if (col) {
+ col->setSelected(selected_, true);
+ }
+ }
+}
+
+void TupleBatch::setSelected(SelectList *selected) {
+ if (selected == selected_) return;
+ selectListOwner_.reset();
+ selected_ = selected;
+ for (auto &col : sysVectors) {
+ if (col) col->setSelected(selected_, true);
+ }
+ for (auto &col : vectors) {
+ if (col) col->setSelected(selected_, true);
+ }
+}
+
+void TupleBatch::unsetSelected() {
+ if (!selected_) return;
+ selected_ = nullptr;
+ selectListOwner_.reset();
+ for (auto &col : vectors) {
+ if (col) {
+ col->setSelected(nullptr, true);
+ }
+ }
+ for (auto &col : sysVectors) {
+ if (col) {
+ col->setSelected(nullptr, true);
+ }
+ }
+}
+
+void TupleBatch::reset() {
+ for (auto &vect : vectors) {
+ assert(vect != nullptr);
+ vect->reset();
+ }
+ vectorRefs.resize(vectors.size(), nullptr);
+
+ for (auto &vect : sysVectors) {
+ assert(vect != nullptr);
+ vect->reset();
+ }
+ sysVectorRefs.resize(sysVectors.size(), nullptr);
+
+ this->numOfRowsPlain = 0;
+ this->selected_ = nullptr;
+ this->selectListOwner_.reset();
+}
+
+void TupleBatch::append(TupleBatch *tupleBatch, SelectList *sel) {
+ if (tupleBatch->isScalarTupleBatch_) {
+ for (auto i = 0; i < getNumOfColumns(); i++)
+ vectors[i]->append(&tupleBatch->scalars_[i]);
+ return;
+ }
+ if (getNumOfRows() > 0) this->materialize();
+ int numCols = getNumOfColumns();
+ int numSysCols = getNumOfSysColumns();
+ if (this->selected_) {
+ int oldSize = this->getNumOfRowsPlain();
+ int appendSize = (sel ? sel->size() : tupleBatch->getNumOfRowsPlain());
+ for (size_t i = 0; i < appendSize; i++) {
+ this->selected_->push_back(oldSize++);
+ }
+ }
+ for (int i = -numSysCols; i < numCols; i++) {
+ if (getColumn(i) == nullptr) {
+ continue;
+ }
+ getColumn(i)->append(tupleBatch->getColumn(i), sel);
+ }
+ this->numOfRowsPlain += sel ? sel->size() : tupleBatch->getNumOfRows();
+}
+
+void TupleBatch::append(const TupleBatch &tb, uint32_t index) {
+ size_t numColumns = getNumOfColumns();
+ size_t numSysColumns = getNumOfSysColumns();
+ if (this->selected_) {
+ int oldSize = selected_->size();
+ selected_->push_back(oldSize);
+ }
+ for (int i = -numSysColumns; i < numColumns; i++) {
+ Vector *vect = getColumn(i);
+ Vector *src = tb.getColumn(i);
+
+ uint32_t len;
+ bool null;
+ vect->append(src, index);
+ }
+ this->numOfRowsPlain += 1;
+}
+
+dbcommon::TupleBatchWriter &TupleBatch::getTupleBatchWriter() {
+ assert(vectors.size() > 0);
+ return vectors;
+}
+
+dbcommon::TupleBatchWriter &TupleBatch::getTupleBatchSysWriter() {
+ assert(sysVectors.size() > 0);
+ return sysVectors;
+}
+
+bool TupleBatch::isValid() const {
+ for (auto &vect : vectors) {
+ assert(!vect || vect->isValid());
+ assert(!vect || vect->getSelected() == getSelected());
+ assert(!vect || vect->getNumOfRowsPlain() == getNumOfRowsPlain());
+ }
+ for (auto &vect : sysVectors) {
+ assert(!vect || vect->isValid());
+ assert(!vect || vect->getSelected() == getSelected());
+ assert(!vect || vect->getNumOfRowsPlain() == getNumOfRowsPlain());
+ }
+ return true;
+}
+
+void TupleBatch::swapColumn(uint32_t index, std::unique_ptr<Vector> *vec) {
+ assert(index < vectors.size() && "index out of range");
+ vectorRefs[index] = vec->get();
+ vectors[index].swap(*vec);
+ if (vectors[index] != nullptr) {
+ vectors[index]->setParentTupleBatch(this);
+ }
+}
+
+void TupleBatch::swapSysColumn(int32_t index, std::unique_ptr<Vector> *vec) {
+ uint32_t sysIdx = -index - 1;
+ assert(sysIdx < sysVectors.size() && "index out of range");
+ sysVectorRefs[sysIdx] = vec->get();
+ sysVectors[sysIdx].swap(*vec);
+ if (sysVectors[sysIdx] != nullptr) {
+ sysVectors[sysIdx]->setParentTupleBatch(this);
+ }
+}
+
+std::unique_ptr<TupleBatch> TupleBatch::replicateRowsToTB(
+ uint32_t rowIndex, int replicateNum) const {
+ TupleBatch::uptr retval(
+ new TupleBatch(getNumOfColumns(), getNumOfSysColumns() > 0));
+ int startIndex = getNumOfSysColumns() * -1;
+ int endIndex = getNumOfColumns();
+ for (int i = startIndex; i < endIndex; ++i) {
+ if (i < 0) {
+ if (sysVectors[i * -1 - 1] == nullptr) {
+ retval->setColumn(i, nullptr, false);
+ } else {
+ retval->setColumn(
+ i,
+ sysVectors[i * -1 - 1]->replicateItemToVec(rowIndex, replicateNum),
+ false);
+ }
+ } else {
+ if (vectors[i] == nullptr) {
+ retval->setColumn(i, nullptr, false);
+ } else {
+ retval->setColumn(
+ i, vectors[i]->replicateItemToVec(rowIndex, replicateNum), false);
+ }
+ }
+ }
+ retval->setNumOfRows(replicateNum);
+ return std::move(retval);
+}
+
+std::unique_ptr<TupleBatch> TupleBatch::getRow(uint32_t rowIndex) const {
+ TupleBatch::uptr retval(new TupleBatch);
+ retval->isScalarTupleBatch_ = true;
+ retval->scalars_.resize(getNumOfColumns());
+ retval->numOfRowsPlain = 1;
+ for (auto i = 0; i < getNumOfColumns(); i++)
+ vectorRefs[i]->readPlainScalar(rowIndex, &(retval->scalars_[i]));
+ return retval;
+}
+
+void TupleBatch::concatTB(std::unique_ptr<TupleBatch> other) {
+ assert(other != nullptr);
+ // for targetList is Null
+ if (other->getNumOfColumns() == 0) return;
+
+ assert(isScalarTupleBatch_ || getNumOfRows() == other->getNumOfRows());
+ assert(other->getNumOfSysColumns() == 0);
+
+ if (other->selected_) {
+ assert(this->selected_ == other->selected_ ||
+ *this->selected_ == *other->selected_);
+ }
+
+ TupleBatchWriter &writer = other->getTupleBatchWriter();
+ for (size_t i = 0; i < other->getNumOfColumns(); ++i) {
+ std::unique_ptr<Vector> vect(writer[i].release());
+ vect->setParentTupleBatch(this);
+ vectorRefs.push_back(vect.get());
+ vectors.push_back(std::move(vect));
+ }
+}
+
+std::unique_ptr<TupleBatch> TupleBatch::getTuple(uint32_t index) const {
+ return replicateRowsToTB(index, 1);
+}
+
+size_t TupleBatch::serialize(std::string *output, size_t reserved) {
+ std::unique_ptr<NodeSerializer> serializer(new NodeSerializer(output));
+ int colNum = getNumOfColumns();
+ int sysColNum = getNumOfSysColumns();
+
+ if (getNumOfRows() == 0)
+ LOG_ERROR(ERRCODE_INTERNAL_ERROR,
+ "Should not serialize an empty tuple batch");
+
+ assert(this->isValid());
+
+ std::string colBitMasks;
+ buildColBitMasks(&colBitMasks);
+
+ // caculate serializer size previously
+ uint64_t sSize =
+ colBitMasks.size() + // column vector availability mask
+ sizeof(uint32_t) * 4 +
+ sizeof(uint16_t); // num of row, row kv, user & sys col, range id
+
+ for (int i = -sysColNum; i < colNum; i++) {
+ Vector *vect = getColumn(i);
+ sSize += (vect == nullptr ? 0 : vect->getSerializeSize());
+ }
+ if (output->size() + sSize > reserved) {
+ reserved = output->size() + sSize;
+ output->reserve(reserved);
+ }
+
+ // write out number of rows, user columns, and system columns, range id
+ serializer->write<uint16_t>(static_cast<uint16_t>(rgId));
+ serializer->write<uint32_t>(getNumOfRows());
+ serializer->write<uint32_t>(colNum);
+ serializer->write<uint32_t>(sysColNum);
+ serializer->writeBytes(colBitMasks.data(), colBitMasks.size());
+ // directly serialize the content of each user column and system column
+ for (int i = -sysColNum; i < colNum; i++) {
+ Vector *vect = getColumn(i);
+ if (vect != nullptr) {
+ vect->serialize(serializer.get());
+ }
+ }
+
+ return reserved;
+}
+
+void TupleBatch::deserialize(const std::string &data, int32_t startPos,
+ uint32_t end) {
+ assert(startPos < end && end <= data.size());
+
+ int colNum = 0;
+ int sysColNum = 0;
+ if (end < startPos + 8) {
+ LOG_DEBUG("deserialize empty string to empty tuplebatch");
+ numOfRowsPlain = 0;
+ vectors.resize(0);
+ return;
+ }
+ int32_t cursor = startPos;
+ while (cursor < end) {
+ std::unique_ptr<NodeDeserializer> deserializer(new NodeDeserializer(data));
+ deserializer->setCursor(cursor);
+
+ if (this->getNumOfRows() == 0) {
+ rgId = deserializer->read<uint16_t>();
+ numOfRowsPlain = deserializer->read<uint32_t>();
+ colNum = deserializer->read<uint32_t>();
+ sysColNum = deserializer->read<uint32_t>();
+ vectorRefs.resize(colNum);
+ vectors.resize(colNum);
+ sysVectorRefs.resize(sysColNum);
+ sysVectors.resize(sysColNum);
+
+ // deserialize col mask bits
+ uint32_t sizeOfColMaskBits = calculateSizeOfMaskBits(colNum, sysColNum);
+ const char *colMaskBits = deserializer->readBytes(sizeOfColMaskBits);
+
+ for (int32_t i = -sysColNum; i < colNum; i++) {
+ if (isColMaskUnavailable(colMaskBits, sysColNum, i)) {
+ setColumn(i, nullptr, false);
+ continue;
+ }
+ std::unique_ptr<Vector> vect = Vector::deserialize(deserializer.get());
+ setColumn(i, std::move(vect), false);
+ }
+
+ // we should keep materialized tuplebatch after deserialize
+ this->materialize();
+ } else {
+ std::unique_ptr<TupleBatch> tb(new TupleBatch());
+ tb->rgId = deserializer->read<uint16_t>();
+ tb->numOfRowsPlain = deserializer->read<uint32_t>();
+ colNum = deserializer->read<uint32_t>();
+ sysColNum = deserializer->read<uint32_t>();
+ tb->vectorRefs.resize(colNum);
+ tb->vectors.resize(colNum);
+ tb->sysVectorRefs.resize(sysColNum);
+ tb->sysVectors.resize(sysColNum);
+
+ // deserialize col mask bits
+ int sizeOfColMaskBits = calculateSizeOfMaskBits(colNum, sysColNum);
+ const char *colMaskBits = deserializer->readBytes(sizeOfColMaskBits);
+
+ for (int32_t i = -sysColNum; i < colNum; i++) {
+ if (isColMaskUnavailable(colMaskBits, sysColNum, i)) {
+ setColumn(i, nullptr, false);
+ continue;
+ }
+ std::unique_ptr<Vector> vect = Vector::deserialize(deserializer.get());
+ tb->setColumn(i, std::move(vect), false);
+ }
+
+ this->append(tb.get());
+ }
+
+ assert(this->isValid());
+ cursor = deserializer->getCursor();
+ }
+}
+
+void TupleBatch::deserialize(const std::string &data) {
+ int colNum = 0;
+ int sysColNum = 0;
+ if (data.length() < 8) {
+ LOG_DEBUG("deserialize empty string to empty tuplebatch");
+ numOfRowsPlain = 0;
+ vectors.resize(0);
+ return;
+ }
+ int32_t cursor = 0;
+ while (cursor < data.length()) {
+ std::unique_ptr<NodeDeserializer> deserializer(new NodeDeserializer(data));
+ deserializer->setCursor(cursor);
+
+ if (this->getNumOfRows() == 0) {
+ rgId = deserializer->read<uint16_t>();
+ numOfRowsPlain = deserializer->read<uint32_t>();
+ colNum = deserializer->read<uint32_t>();
+ sysColNum = deserializer->read<uint32_t>();
+ vectorRefs.resize(colNum);
+ vectors.resize(colNum);
+ sysVectorRefs.resize(sysColNum);
+ sysVectors.resize(sysColNum);
+
+ // deserialize col mask bits
+ int sizeOfColMaskBits = calculateSizeOfMaskBits(colNum, sysColNum);
+ const char *colMaskBits = deserializer->readBytes(sizeOfColMaskBits);
+
+ for (int32_t i = -sysColNum; i < colNum; i++) {
+ if (isColMaskUnavailable(colMaskBits, sysColNum, i)) {
+ setColumn(i, nullptr, false);
+ continue;
+ }
+ std::unique_ptr<Vector> vect = Vector::deserialize(deserializer.get());
+ setColumn(i, std::move(vect), false);
+ }
+
+ // we should keep materialized tuplebatch after deserialize
+ this->materialize();
+ } else {
+ std::unique_ptr<TupleBatch> tb(new TupleBatch());
+ tb->rgId = deserializer->read<uint16_t>();
+ tb->numOfRowsPlain = deserializer->read<uint32_t>();
+ colNum = deserializer->read<uint32_t>();
+ sysColNum = deserializer->read<uint32_t>();
+ tb->vectorRefs.resize(colNum);
+ tb->vectors.resize(colNum);
+ tb->sysVectorRefs.resize(sysColNum);
+ tb->sysVectors.resize(sysColNum);
+
+ // deserialize col mask bits
+ int sizeOfColMaskBits = calculateSizeOfMaskBits(colNum, sysColNum);
+ const char *colMaskBits = deserializer->readBytes(sizeOfColMaskBits);
+
+ for (int32_t i = -sysColNum; i < colNum; i++) {
+ if (isColMaskUnavailable(colMaskBits, sysColNum, i)) {
+ setColumn(i, nullptr, false);
+ continue;
+ }
+ std::unique_ptr<Vector> vect = Vector::deserialize(deserializer.get());
+ tb->setColumn(i, std::move(vect), false);
+ }
+
+ this->append(tb.get());
+ }
+
+ assert(this->isValid());
+ cursor = deserializer->getCursor();
+ }
+}
+
+void TupleBatch::permutate(const std::vector<uint32_t> &projIdxs) {
+ if (isScalarTupleBatch_) {
+ assert(projIdxs.size() == scalars_.size());
+ std::vector<Scalar> tmp(scalars_.size());
+ for (auto i = 0; i < scalars_.size(); i++) tmp[projIdxs[i]] = scalars_[i];
+ scalars_ = std::move(tmp);
+ return;
+ }
+ assert(projIdxs.size() == vectors.size());
+ for (auto i = 0; i < vectors.size(); i++) {
+ vectorRefs[projIdxs[i]] = vectors[i].release();
+ }
+ for (auto i = 0; i < vectors.size(); i++) {
+ vectors[i].reset(vectorRefs[i]);
+ }
+}
+
+bool TupleBatch::ownData() const {
+ for (auto const &vec : vectors)
+ if (!vec->getOwnData()) return false;
+ return true;
+}
+
+void TupleBatch::trim() {
+ for (auto vec : vectorRefs) vec->trim();
+}
+
+std::unique_ptr<TupleBatch> TupleBatch::extract(const SelectList *sel,
+ uint64_t start,
+ uint64_t end) const {
+ assert(start <= end);
+ std::unique_ptr<TupleBatch> res(new TupleBatch());
+ if (sel && sel->size() < (end - start)) {
+ res->setSelected(*sel);
+ }
+ res->setNumOfRows(end - start);
+ res->isScalarTupleBatch_ = this->isScalarTupleBatch_;
+ res->rgId = this->rgId;
+ const SelectList *csel = res->getSelected();
+ for (auto &vect : vectors) {
+ if (!vect) {
+ res->addColumn(nullptr);
+ continue;
+ }
+ std::unique_ptr<Vector> tmp = vect->extract(csel, start, end);
+ res->addColumn(std::move(tmp));
+ }
+ for (auto &vect : sysVectors) {
+ if (!vect) {
+ res->addSysColumn(nullptr);
+ continue;
+ }
+ std::unique_ptr<Vector> tmp = vect->extract(csel, start, end);
+ res->addSysColumn(std::move(tmp));
+ }
+ return res;
+}
+
+std::unique_ptr<TupleBatch> TupleBatch::extract(uint64_t start,
+ uint64_t end) const {
+ if (selected_) {
+ std::unique_ptr<SelectList> sel = selected_->extract(start, end);
+ return extract(sel.get(), start, end);
+ } else {
+ return extract(nullptr, start, end);
+ }
+}
+
+std::unique_ptr<TupleDesc> TupleBatch::mockTupleDesc() {
+ assert(!vectors.empty());
+ std::unique_ptr<TupleDesc> td(new TupleDesc());
+ uint32_t i = 0;
+ for (auto &col : vectors) {
+ td->add(std::to_string(i++), col->getTypeKind(), col->getTypeModifier());
+ }
+ return td;
+}
+
+double TupleBatch::getMemUsed() {
+ double memUsed = 0;
+ for (const auto &vec : vectors)
+ if (vec) memUsed += vec->getSerializeSize();
+ return memUsed;
+}
+
+void TupleBatch::replaceDecimalVector() {
+ size_t numColumns = getNumOfColumns();
+ for (int i = 0; i < numColumns; i++) {
+ Vector *vect = getColumn(i);
+ if (vect && vect->getTypeKind() == dbcommon::TypeKind::DECIMALID) {
+ const uint64_t sz = vect->getNumOfRowsPlain();
+ std::unique_ptr<dbcommon::Vector> vector =
+ dbcommon::Vector::BuildVector(dbcommon::TypeKind::DECIMALNEWID, true);
+
+ vector->setHasNull(vect->hasNullValue());
+ dbcommon::DecimalVector *dvec =
+ dynamic_cast<dbcommon::DecimalVector *>(vector.get());
+
+ for (int j = 0; j < sz; j++) {
+ bool isnull = false;
+ std::string str = vect->readPlain(j, &isnull);
+ dvec->append(str, isnull);
+ }
+
+ setColumn(i, std::move(vector), false);
+ }
+ }
+}
+
+} // namespace dbcommon.
diff --git a/depends/dbcommon/src/dbcommon/common/tuple-batch.h b/depends/dbcommon/src/dbcommon/common/tuple-batch.h
new file mode 100644
index 0000000..15019e2
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/tuple-batch.h
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_TUPLE_BATCH_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_TUPLE_BATCH_H_
+
+#include <cassert>
+#include <string>
+#include <vector>
+
+#include "dbcommon/common/node-deserializer.h"
+#include "dbcommon/common/node-serializer.h"
+#include "dbcommon/common/tuple-desc.h"
+#include "dbcommon/common/vector.h"
+
+#include "dbcommon/log/logger.h"
+
+namespace dbcommon {
+
+class Vector;
+class JoinTupleBuffer;
+typedef std::vector<std::unique_ptr<Vector>> TupleBatchReader;
+typedef std::vector<std::unique_ptr<Vector>> TupleBatchWriter;
+
+class TupleBatch {
+ public:
+ explicit TupleBatch(bool hasSysCol = false);
+ explicit TupleBatch(uint32_t nfield, bool hasSysCol = false);
+ explicit TupleBatch(int32_t nfield, bool hasSysCol = false);
+ explicit TupleBatch(size_t nfield, bool hasSysCol = false);
+ TupleBatch(const TupleDesc &desc, bool ownData, bool hasSysCol = false);
+ ~TupleBatch() {}
+
+ TupleBatch(const TupleBatch &other) = delete;
+ TupleBatch &operator=(const TupleBatch &other) = delete;
+ TupleBatch(TupleBatch &&other) = delete;
+ TupleBatch &operator=(TupleBatch &&other) = delete;
+
+ // only call this when you get a clean tuple batch
+ // for example, you get an empty tuplebatch by new TupleBatch(),
+ // or you just called reset()
+ void rebuild(const TupleDesc &desc, bool ownData, bool hasSysCol);
+
+ // Get the tuple batch size in Bytes. The size is
+ // not accurate since it only contains data/offset/nullbitmap
+ // @return The tuple batch size in Bytes
+ int64_t getTupleBatchSize() const;
+
+ uint32_t getNumOfRows() const {
+ return selected_ ? selected_->size() : this->numOfRowsPlain;
+ }
+
+ uint32_t getNumOfRowsPlain() const { return this->numOfRowsPlain; }
+
+ uint16_t getRgId() const { return this->rgId; }
+
+ void setRgId(uint16_t rgId) { this->rgId = rgId; }
+
+ void setNumOfRows(uint32_t rows) { this->numOfRowsPlain = rows; }
+
+ void incNumOfRows(uint32_t inc) { this->numOfRowsPlain += inc; }
+
+ void addColumn(TypeKind dataType, int64_t typeMod, bool ownData);
+
+ void addColumn(std::unique_ptr<Vector> vec) {
+ vectorRefs.push_back(vec.get());
+ vectors.push_back(std::move(vec));
+ }
+
+ void addSysColumn(std::unique_ptr<Vector> vec) {
+ sysVectorRefs.push_back(vec.get());
+ sysVectors.push_back(std::move(vec));
+ }
+
+ void addSysColumn(TypeKind dataType, int64_t typeMod, bool ownData);
+
+ uint32_t getNumOfColumns() const { return vectors.size(); }
+
+ uint32_t getNumOfSysColumns() const { return sysVectors.size(); }
+
+ Vector *getColumn(int32_t index) const {
+ // Intend to provide scalar tuple for executor
+ if (isScalarTupleBatch_) {
+ return const_cast<Vector *>(
+ reinterpret_cast<const Vector *>(&scalars_[index]));
+ }
+
+ if (index >= 0) {
+ assert(index < vectors.size() && "index out of range");
+ return vectorRefs[index];
+ } else {
+ index = -index - 1;
+ assert(index < sysVectors.size() && "index out of range");
+ return sysVectorRefs[index];
+ }
+ }
+
+ std::unique_ptr<Vector> releaseColumn(int32_t index) {
+ if (index >= 0) {
+ assert(index < vectors.size() && "index out of range");
+ return std::unique_ptr<Vector>(vectors[index].release());
+ } else {
+ index = -index - 1;
+ assert(index < sysVectors.size() && "index out of range");
+ return std::unique_ptr<Vector>(sysVectors[index].release());
+ }
+ }
+
+ void clearColumnRef(int32_t index) {
+ if (index >= 0) {
+ assert(index < vectorRefs.size() && "index out of range");
+ assert(vectors[index] == nullptr); // only allow to clear ref when this
+ // vector is not owned
+ vectorRefs[index] = nullptr;
+ } else {
+ index = -index - 1;
+ assert(index < sysVectors.size() && "index out of range");
+ assert(sysVectors[index] == nullptr); // only allow to clear ref when
+ // this vector is not owned
+ sysVectorRefs[index] = nullptr;
+ }
+ }
+
+ void swapColumn(uint32_t index, std::unique_ptr<Vector> *vec);
+
+ void swapSysColumn(int32_t index, std::unique_ptr<Vector> *vec);
+
+ void setColumn(int32_t index, std::unique_ptr<Vector> vect,
+ bool useParentSelected);
+
+ void setColumn(int32_t index, Vector *vect, bool useParentSelected);
+
+ void insertColumn(int32_t index, TypeKind dataType, int64_t typeMod,
+ bool ownData);
+
+ void insertColumn(int32_t index, std::unique_ptr<Vector> vect,
+ bool useParentSelected);
+
+ void removeColumn(uint32_t idx) {
+ vectorRefs.erase(vectorRefs.begin() + idx);
+ vectors.erase(vectors.begin() + idx);
+ }
+
+ void removeSysColumn(int32_t idx) {
+ uint32_t sysIdx = -idx - 1;
+ assert(sysIdx < sysVectors.size());
+ sysVectorRefs.erase(sysVectorRefs.begin() + sysIdx);
+ sysVectors.erase(sysVectors.begin() + sysIdx);
+ }
+
+ // Materialize the tuple batch
+ // @return void
+ void materialize();
+
+ // Clone selected(plain index) tuples in the tuple batch
+ // @param sel The input select list
+ // @return The result tuple batch
+ std::unique_ptr<TupleBatch> cloneSelected(const SelectList *sel) const;
+
+ // Clone a plain tb according to the input select list in some range
+ // The result tb is a plain tuple batch
+ // @param sel The input select list, plain index
+ // @param start The start input plain index
+ // @param end The end input plain index (not included)
+ // @return The tb cloned
+ std::unique_ptr<TupleBatch> extract(const SelectList *sel, uint64_t start,
+ uint64_t end) const;
+ std::unique_ptr<TupleBatch> extract(uint64_t start, uint64_t end) const;
+
+ std::unique_ptr<TupleBatch> clone() const {
+ return cloneSelected(getSelected());
+ }
+
+ // Create a new TupleBatch that contains multiple replication of the specific
+ // row.
+ // @param rowIndex Plain index
+ std::unique_ptr<TupleBatch> replicateRowsToTB(uint32_t rowIndex,
+ int replicateNum) const;
+
+ // Create a new TupleBatch that contains the specific row.
+ // @param rowIndex Plain index
+ std::unique_ptr<TupleBatch> getRow(uint32_t rowIndex) const;
+
+ // Add Vectors from other TupleBatch to this TupleBatch
+ // The number of rows does not change in this function,
+ // But the number of columns is increased.
+ // @param other The input tuple batch
+ // @return void
+ void concatTB(std::unique_ptr<TupleBatch> other);
+
+ bool isUseSelected() const { return selected_ != nullptr; }
+
+ void setSelected(const SelectList &selected);
+
+ void setSelected(SelectList *selected);
+
+ void unsetSelected();
+
+ SelectList *getSelected() { return selected_; }
+
+ const SelectList *getSelected() const { return selected_; }
+
+ std::unique_ptr<SelectList> releaseSelected() {
+ return std::move(selectListOwner_);
+ }
+
+ void reset();
+
+ void append(const TupleBatch &tb, uint32_t index);
+
+ void append(TupleBatch *tb) { append(tb, tb->selected_); }
+
+ void append(TupleBatch *tupleBatch, SelectList *sel);
+
+ typedef std::unique_ptr<TupleBatch> uptr;
+
+ const TupleBatchReader &getTupleBatchReader() const { return vectors; }
+
+ TupleBatchWriter &getTupleBatchWriter();
+
+ const TupleBatchReader &getTupleBatchSysReader() const { return sysVectors; }
+
+ TupleBatchWriter &getTupleBatchSysWriter();
+
+ std::string toString() const;
+
+ // Used only for debug: validity check
+ // @return a bool value indicating whether it is valid
+ bool isValid() const;
+
+ std::unique_ptr<TupleBatch> getTuple(uint32_t index) const;
+
+ void setNull(uint32_t index);
+ void appendNull(uint32_t tupleCount);
+ void convert(const dbcommon::SupportedEncodingSet inputEncoding,
+ const dbcommon::SupportedEncodingSet outputEncoding) {
+ if (inputEncoding == outputEncoding) return;
+ for (int i = 0; i < vectors.size(); i++) {
+ vectors[i]->convert(inputEncoding, outputEncoding);
+ }
+ for (int i = 0; i < sysVectors.size(); i++) {
+ sysVectors[i]->convert(inputEncoding, outputEncoding);
+ }
+ materialize();
+ }
+ void deserialize(const std::string &data);
+ void deserialize(const std::string &data, int32_t startPos, uint32_t end);
+ size_t serialize(std::string *output, size_t reserved);
+
+ // @param projIdxs map each vector[i] into vector[projIdxs[i]]
+ void permutate(const std::vector<uint32_t> &projIdxs);
+
+ bool ownData() const;
+
+ void trim();
+
+ bool isScalarTupleBatch() { return isScalarTupleBatch_; }
+
+ std::unique_ptr<TupleDesc> mockTupleDesc();
+
+ double getMemUsed();
+
+ void replaceDecimalVector();
+
+ private:
+ inline uint32_t calculateSizeOfMaskBits(uint32_t colSize,
+ uint32_t sysColSize) {
+ return (colSize + sysColSize + 7) / 8; // use one byte to express 8 columns
+ }
+
+ inline void buildColBitMasks(std::string *res) {
+ uint32_t size = calculateSizeOfMaskBits(vectors.size(), sysVectors.size());
+ assert(size >= 0);
+ res->assign(size, '\0');
+ char *data = const_cast<char *>(res->data());
+ int totalCount = 0;
+ for (int i = 0; i < sysVectors.size(); ++i) {
+ if (sysVectors[i] != nullptr) {
+ data[totalCount / 8] |= (1 << (7 - (totalCount % 8)));
+ }
+ totalCount++;
+ }
+ for (int i = 0; i < vectors.size(); ++i) {
+ if (vectors[i] != nullptr) {
+ data[totalCount / 8] |= (1 << (7 - (totalCount % 8)));
+ }
+ totalCount++;
+ }
+ }
+
+ inline bool isColMaskUnavailable(const char *bits, uint32_t sysColSize,
+ int colIndex) {
+ int totalOffset = colIndex + sysColSize; // as syscol index is negative
+ return (bits[totalOffset / 8] & (1 << (7 - (totalOffset % 8)))) == 0;
+ }
+
+ private:
+ bool isScalarTupleBatch_ = false;
+ uint32_t numOfRowsPlain = 0;
+ uint16_t rgId = 0;
+ std::vector<std::unique_ptr<Vector>> vectors;
+ std::vector<std::unique_ptr<Vector>> sysVectors;
+ std::vector<Vector *> vectorRefs;
+ std::vector<Vector *> sysVectorRefs;
+ std::vector<Scalar> scalars_;
+ SelectList *selected_ = nullptr;
+ std::unique_ptr<SelectList> selectListOwner_;
+
+ friend JoinTupleBuffer;
+};
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_TUPLE_BATCH_H_
diff --git a/depends/dbcommon/src/dbcommon/common/tuple-desc.cc b/depends/dbcommon/src/dbcommon/common/tuple-desc.cc
new file mode 100644
index 0000000..f1e2d50
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/tuple-desc.cc
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <cassert>
+
+#include "dbcommon/common/tuple-desc.h"
+#include "dbcommon/type/type-modifier.h"
+
+namespace dbcommon {
+
+TupleDesc::TupleDesc() {}
+
+TupleDesc::~TupleDesc() {}
+
+size_t TupleDesc::getNumOfColumns() const { return columnNames.size(); }
+
+const std::string &TupleDesc::getColumnName(size_t idx) const {
+ return columnNames.at(idx);
+}
+
+TypeKind TupleDesc::getColumnType(size_t idx) const {
+ return columnTypes.at(idx);
+}
+
+uint64_t TupleDesc::getColumnTypeModifier(size_t idx) const {
+ return columnTypeModifiers.at(idx);
+}
+
+char TupleDesc::getColumnNullable(size_t idx) const {
+ return columnNullables.at(idx);
+}
+
+void TupleDesc::add(const std::string &field, dbcommon::TypeKind type,
+ int64_t typeMod, bool nullable) {
+ if (type == TypeKind::VARCHARID && typeMod == -1) type = TypeKind::STRINGID;
+ columnNames.push_back(field);
+ columnTypes.push_back(type);
+ columnNullables.push_back(nullable);
+ // todo: a trim workaround
+ if (field == "Dummy" && type == CHARID && typeMod == -1)
+ typeMod = TypeModifierUtil::getTypeModifierFromMaxLength(0);
+ columnTypeModifiers.push_back(typeMod);
+}
+
+std::vector<uint32_t> TupleDesc::getFixedLengths() const {
+ std::vector<uint32_t> ret;
+ for (auto i = 0; i < columnTypes.size(); i++) {
+ uint32_t fixedLength;
+ switch (columnTypes[i]) {
+ case dbcommon::TypeKind::BOOLEANID:
+ fixedLength = sizeof(char);
+ break;
+
+ case dbcommon::TypeKind::TINYINTID:
+ fixedLength = sizeof(int8_t);
+ break;
+
+ case dbcommon::TypeKind::SMALLINTID:
+ fixedLength = sizeof(int16_t);
+ break;
+
+ case dbcommon::TypeKind::INTID:
+ case dbcommon::TypeKind::DATEID:
+ fixedLength = sizeof(int32_t);
+ break;
+
+ case dbcommon::TypeKind::BIGINTID:
+ case dbcommon::TypeKind::TIMEID:
+ fixedLength = sizeof(int64_t);
+ break;
+
+ case dbcommon::TypeKind::TIMESTAMPID:
+ case dbcommon::TypeKind::TIMESTAMPTZID:
+ fixedLength = sizeof(int64_t) * 2;
+ break;
+
+ case dbcommon::TypeKind::FLOATID:
+ fixedLength = sizeof(float);
+ break;
+ case dbcommon::TypeKind::DOUBLEID:
+ fixedLength = sizeof(double);
+ break;
+ case dbcommon::TypeKind::DECIMALNEWID:
+ fixedLength = sizeof(uint64_t) * 3;
+ break;
+
+ case dbcommon::TypeKind::CHARID:
+ case dbcommon::TypeKind::VARCHARID:
+ case dbcommon::TypeKind::STRINGID:
+ case dbcommon::TypeKind::BINARYID:
+ case dbcommon::TypeKind::IOBASETYPEID:
+ case dbcommon::TypeKind::STRUCTEXID:
+ case dbcommon::TypeKind::DECIMALID:
+ fixedLength = 0;
+ break;
+
+ default:
+ assert(false); // not supported types
+ }
+ ret.push_back(fixedLength);
+ }
+ return ret;
+}
+
+} // namespace dbcommon
diff --git a/depends/dbcommon/src/dbcommon/common/tuple-desc.h b/depends/dbcommon/src/dbcommon/common/tuple-desc.h
new file mode 100644
index 0000000..6fbc4e8
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/tuple-desc.h
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_TUPLE_DESC_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_TUPLE_DESC_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "dbcommon/type/type-kind.h"
+
+namespace dbcommon {
+
+class TupleDesc {
+ public:
+ TupleDesc();
+
+ ~TupleDesc();
+
+ size_t getNumOfColumns() const;
+
+ const std::string &getColumnName(size_t idx) const;
+
+ dbcommon::TypeKind getColumnType(size_t idx) const;
+
+ uint64_t getColumnTypeModifier(size_t idx) const;
+
+ char getColumnNullable(size_t idx) const;
+
+ void add(const std::string &field, dbcommon::TypeKind type,
+ int64_t typeMod = -1, bool nullable = true);
+
+ std::vector<std::string> &getColumnNames() { return columnNames; }
+
+ std::vector<dbcommon::TypeKind> &getColumnTypes() { return columnTypes; }
+
+ std::vector<int64_t> &getColumnTypeModifiers() { return columnTypeModifiers; }
+
+ std::vector<char> &getColumnNullables() { return columnNullables; }
+
+ std::vector<uint32_t> getFixedLengths() const;
+
+ typedef std::unique_ptr<TupleDesc> uptr;
+
+ private:
+ std::vector<std::string> columnNames;
+ std::vector<dbcommon::TypeKind> columnTypes;
+ std::vector<int64_t> columnTypeModifiers;
+ std::vector<char> columnNullables;
+};
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_TUPLE_DESC_H_
diff --git a/depends/dbcommon/src/dbcommon/common/vector-transformer.h b/depends/dbcommon/src/dbcommon/common/vector-transformer.h
new file mode 100644
index 0000000..1fecae1
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector-transformer.h
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_TRANSFORMER_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_TRANSFORMER_H_
+
+#include "dbcommon/common/vector.h"
+#include "dbcommon/nodes/select-list.h"
+
+namespace dbcommon {
+
+template <class Operation>
+inline void transformVector(const size_t plainSize, const SelectList *sel,
+ const bool *__restrict__ nulls,
+ Operation operation) {
+ if (sel) {
+ if (nulls) {
+ for (auto plainIdx : *sel) {
+ if (!nulls[plainIdx]) operation(plainIdx);
+ }
+ } else {
+ for (auto plainIdx : *sel) {
+ operation(plainIdx);
+ }
+ }
+ } else {
+ if (nulls) {
+ for (uint64_t plainIdx = 0; plainIdx < plainSize; ++plainIdx) {
+ if (!nulls[plainIdx]) operation(plainIdx);
+ }
+ } else {
+ for (uint64_t plainIdx = 0; plainIdx < plainSize; ++plainIdx) {
+ operation(plainIdx);
+ }
+ }
+ }
+}
+
+template <class Operation>
+inline void transformVector(const size_t plainSize, const SelectList *sel,
+ const bool *__restrict__ nulls1,
+ const bool *__restrict__ nulls2,
+ Operation operation) {
+ if (sel) {
+ if (nulls1) {
+ if (nulls2) {
+ for (auto plainIdx : *sel)
+ if (!nulls1[plainIdx] && !nulls2[plainIdx]) operation(plainIdx);
+ } else {
+ for (auto plainIdx : *sel)
+ if (!nulls1[plainIdx]) operation(plainIdx);
+ }
+ } else {
+ if (nulls2) {
+ for (auto plainIdx : *sel)
+ if (!nulls2[plainIdx]) operation(plainIdx);
+ } else {
+ for (auto plainIdx : *sel) operation(plainIdx);
+ }
+ }
+ } else {
+ if (nulls1) {
+ if (nulls2) {
+ for (uint64_t plainIdx = 0; plainIdx < plainSize; ++plainIdx)
+ if (!nulls1[plainIdx] && !nulls2[plainIdx]) operation(plainIdx);
+ } else {
+ for (uint64_t plainIdx = 0; plainIdx < plainSize; ++plainIdx)
+ if (!nulls1[plainIdx]) operation(plainIdx);
+ }
+ } else {
+ if (nulls2) {
+ for (uint64_t plainIdx = 0; plainIdx < plainSize; ++plainIdx)
+ if (!nulls2[plainIdx]) operation(plainIdx);
+ } else {
+ for (uint64_t plainIdx = 0; plainIdx < plainSize; ++plainIdx)
+ operation(plainIdx);
+ }
+ }
+ }
+}
+
+template <class Operation>
+inline void transformDistinct(const size_t plainSize, const SelectList *sel,
+ const bool *__restrict__ nulls,
+ SelectList *selected, bool isnull,
+ Operation operation) {
+ SelectList::value_type *__restrict__ res = selected->begin();
+ uint64_t counter = 0;
+ if (isnull) {
+ if (sel) {
+ if (!nulls) {
+ for (const auto index : *sel) {
+ res[counter++] = index;
+ }
+ } else {
+ for (const auto index : *sel) {
+ if (!nulls[index]) res[counter++] = index;
+ }
+ }
+ } else {
+ if (!nulls) {
+ for (uint64_t i = 0; i < plainSize; ++i) {
+ res[counter++] = i;
+ }
+ } else {
+ for (uint64_t i = 0; i < plainSize; ++i) {
+ if (!nulls[i]) res[counter++] = i;
+ }
+ }
+ }
+ } else {
+ if (sel) {
+ if (!nulls) {
+ for (const auto index : *sel) {
+ if (operation(index)) res[counter++] = index;
+ }
+ } else {
+ for (const auto index : *sel) {
+ if (nulls[index] || operation(index)) res[counter++] = index;
+ }
+ }
+ } else {
+ if (!nulls) {
+ for (uint64_t i = 0; i < plainSize; ++i) {
+ if (operation(i)) res[counter++] = i;
+ }
+ } else {
+ for (uint64_t i = 0; i < plainSize; ++i) {
+ if (nulls[i] || operation(i)) res[counter++] = i;
+ }
+ }
+ }
+ }
+ selected->resize(counter);
+}
+
+template <class Operation>
+inline void transformDistinct(const size_t plainSize, const SelectList *sel,
+ const bool *__restrict__ nulls1,
+ const bool *__restrict__ nulls2,
+ SelectList *selected, Operation operation) {
+ SelectList::value_type *__restrict__ res = selected->begin();
+ uint64_t counter = 0;
+ if (sel) {
+ if (!nulls1) {
+ if (!nulls2) {
+ for (const auto index : *sel) {
+ if (operation(index)) {
+ res[counter++] = index;
+ }
+ }
+ } else {
+ for (const auto index : *sel) {
+ if (nulls2[index] || operation(index)) {
+ res[counter++] = index;
+ }
+ }
+ }
+ } else {
+ if (!nulls2) {
+ for (const auto index : *sel) {
+ if (nulls1[index] || operation(index)) {
+ res[counter++] = index;
+ }
+ }
+ } else {
+ for (const auto index : *sel) {
+ if ((nulls1[index] && !nulls2[index]) ||
+ (!nulls1[index] && nulls2[index]) ||
+ (!nulls1[index] && !nulls2[index] && operation(index))) {
+ res[counter++] = index;
+ }
+ }
+ }
+ }
+ } else {
+ if (!nulls1) {
+ if (!nulls2) {
+ for (uint64_t i = 0; i < plainSize; ++i) {
+ if (operation(i)) {
+ res[counter++] = i;
+ }
+ }
+ } else {
+ for (uint64_t i = 0; i < plainSize; ++i) {
+ if (nulls2[i] || operation(i)) {
+ res[counter++] = i;
+ }
+ }
+ }
+ } else {
+ if (!nulls2) {
+ for (uint64_t i = 0; i < plainSize; ++i) {
+ if (nulls1[i] || operation(i)) {
+ res[counter++] = i;
+ }
+ }
+ } else {
+ for (uint64_t i = 0; i < plainSize; ++i) {
+ if ((!nulls1[i] && nulls2[i]) || (nulls1[i] && !nulls2[i]) ||
+ (!nulls1[i] && !nulls2[i] && operation(i))) {
+ res[counter++] = i;
+ }
+ }
+ }
+ }
+ }
+ selected->resize(counter);
+}
+
+struct DecimalVectorRawData {
+ explicit DecimalVectorRawData(Vector *vec)
+ : plainSize(vec->getNumOfRowsPlain()),
+ sel(vec->getSelected()),
+ nulls((vec->getNulls())),
+ hightbits(reinterpret_cast<int64_t *>(
+ vec->getAuxiliaryValueBuffer()->data())),
+ lowbits(reinterpret_cast<uint64_t *>(vec->getValueBuffer()->data())),
+ scales(
+ reinterpret_cast<int64_t *>(vec->getScaleValueBuffer()->data())) {}
+
+ size_t plainSize;
+ const SelectList *sel;
+ const bool *__restrict__ nulls;
+ int64_t *__restrict__ hightbits;
+ uint64_t *__restrict__ lowbits;
+ int64_t *__restrict__ scales;
+};
+
+template <typename FixedSizeType>
+struct FixedSizeTypeVectorRawData {
+ explicit FixedSizeTypeVectorRawData(Vector *vec)
+ : plainSize(vec->getNumOfRowsPlain()),
+ sel(vec->getSelected()),
+ nulls((vec->getNulls())),
+ values(
+ reinterpret_cast<FixedSizeType *>(vec->getValueBuffer()->data())) {}
+
+ size_t plainSize;
+ const SelectList *sel;
+ const bool *__restrict__ nulls;
+ FixedSizeType *__restrict__ values;
+};
+
+struct VariableSizeTypeVectorRawData {
+ explicit VariableSizeTypeVectorRawData(Vector *vec)
+ : plainSize(vec->getNumOfRowsPlain()),
+ sel(vec->getSelected()),
+ nulls(vec->getNulls()),
+ valptrs(vec->getValPtrs()),
+ lengths(const_cast<uint64_t *>(vec->getLengths())) {}
+
+ size_t plainSize;
+ const SelectList *sel;
+ const bool *__restrict__ nulls;
+ const char **__restrict__ valptrs;
+ uint64_t *__restrict__ lengths;
+};
+
+struct TimestampVectorRawData {
+ explicit TimestampVectorRawData(Vector *vec)
+ : plainSize(vec->getNumOfRowsPlain()),
+ sel(vec->getSelected()),
+ nulls(vec->getNulls()),
+ seconds(reinterpret_cast<int64_t *>(vec->getValueBuffer()->data())),
+ nanoseconds(
+ reinterpret_cast<int64_t *>(vec->getNanosecondsBuffer()->data())) {}
+
+ size_t plainSize;
+ const SelectList *sel;
+ const bool *__restrict__ nulls;
+ int64_t *__restrict__ seconds;
+ int64_t *__restrict__ nanoseconds;
+};
+
+struct IntervalVectorRawData {
+ explicit IntervalVectorRawData(Vector *vec)
+ : plainSize(vec->getNumOfRowsPlain()),
+ sel(vec->getSelected()),
+ nulls(vec->getNulls()),
+ timeOffsets(reinterpret_cast<int64_t *>(vec->getValueBuffer()->data())),
+ days(reinterpret_cast<int32_t *>(vec->getDayBuffer()->data())),
+ months(reinterpret_cast<int32_t *>(vec->getMonthBuffer()->data())) {}
+
+ size_t plainSize;
+ const SelectList *sel;
+ const bool *__restrict__ nulls;
+ int64_t *__restrict__ timeOffsets;
+ int32_t *__restrict__ days;
+ int32_t *__restrict__ months;
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_TRANSFORMER_H_
diff --git a/depends/dbcommon/src/dbcommon/common/vector.cc b/depends/dbcommon/src/dbcommon/common/vector.cc
new file mode 100644
index 0000000..e453e83
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector.cc
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "dbcommon/common/vector.h"
+
+#include "dbcommon/common/tuple-batch.h"
+#include "dbcommon/common/vector/decimal-vector.h"
+#include "dbcommon/common/vector/fixed-length-vector.h"
+#include "dbcommon/common/vector/interval-vector.h"
+#include "dbcommon/common/vector/list-vector.h"
+#include "dbcommon/common/vector/struct-vector.h"
+#include "dbcommon/common/vector/timestamp-vector.h"
+#include "dbcommon/common/vector/variable-length-vector.h"
+#include "dbcommon/type/type-util.h"
+#include "dbcommon/utils/macro.h"
+
+namespace dbcommon {
+
+std::unique_ptr<Vector> Vector::BuildVector(TypeKind type, bool ownData,
+ int64_t typeModifier) {
+ std::unique_ptr<Vector> ret;
+ switch (type) {
+ case TINYINTID:
+ ret = std::unique_ptr<Vector>(new TinyIntVector(ownData));
+ break;
+ case SMALLINTID:
+ ret = std::unique_ptr<Vector>(new SmallIntVector(ownData));
+ break;
+ case INTID:
+ ret = std::unique_ptr<Vector>(new IntVector(ownData));
+ break;
+ case BIGINTID:
+ ret = std::unique_ptr<Vector>(new BigIntVector(ownData));
+ break;
+ case FLOATID:
+ ret = std::unique_ptr<Vector>(new FloatVector(ownData));
+ break;
+ case DOUBLEID:
+ ret = std::unique_ptr<Vector>(new DoubleVector(ownData));
+ break;
+ case TIMESTAMPID:
+ ret = std::unique_ptr<Vector>(new TimestampVector(ownData));
+ break;
+ case TIMESTAMPTZID:
+ ret = std::unique_ptr<Vector>(new TimestamptzVector(ownData));
+ break;
+ case DATEID:
+ ret = std::unique_ptr<Vector>(new DateVector(ownData));
+ break;
+ case TIMEID:
+ ret = std::unique_ptr<Vector>(new TimeVector(ownData));
+ break;
+ case INTERVALID:
+ ret = std::unique_ptr<Vector>(new IntervalVector(ownData));
+ break;
+ case STRINGID:
+ ret = std::unique_ptr<Vector>(new StringVector(ownData));
+ break;
+ case DECIMALID:
+ ret = std::unique_ptr<Vector>(new NumericVector(ownData));
+ break;
+ case DECIMALNEWID:
+ ret = std::unique_ptr<Vector>(new DecimalVector(ownData));
+ break;
+ case VARCHARID:
+ if (typeModifier == -1)
+ ret = std::unique_ptr<Vector>(new StringVector(ownData));
+ else
+ ret = std::unique_ptr<Vector>(
+ new VaryingCharVector(ownData, typeModifier));
+ break;
+ case CHARID:
+ if (typeModifier == -1)
+ ret = std::unique_ptr<Vector>(new StringVector(ownData));
+ else
+ ret = std::unique_ptr<Vector>(
+ new BlankPaddedCharVector(ownData, typeModifier));
+ break;
+ case BOOLEANID:
+ ret = std::unique_ptr<Vector>(new BooleanVector(ownData));
+ break;
+ case BINARYID:
+ ret = std::unique_ptr<Vector>(new BinaryVector(ownData));
+ break;
+ case STRUCTID:
+ ret = std::unique_ptr<Vector>(new StructVector(ownData));
+ break;
+ case SMALLINTARRAYID:
+ ret = std::unique_ptr<Vector>(new SmallIntArrayVector(ownData));
+ break;
+ case INTARRAYID:
+ ret = std::unique_ptr<Vector>(new IntArrayVector(ownData));
+ break;
+ case BIGINTARRAYID:
+ ret = std::unique_ptr<Vector>(new BigIntArrayVector(ownData));
+ break;
+ case FLOATARRAYID:
+ ret = std::unique_ptr<Vector>(new FloatArrayVector(ownData));
+ break;
+ case DOUBLEARRAYID:
+ ret = std::unique_ptr<Vector>(new DoubleArrayVector(ownData));
+ break;
+ case IOBASETYPEID:
+ ret = std::unique_ptr<Vector>(new IOBaseTypeVector(ownData));
+ break;
+ case STRUCTEXID:
+ ret = std::unique_ptr<Vector>(new StructExTypeVector(ownData));
+ break;
+ case MAGMATID:
+ ret = std::unique_ptr<Vector>(new MagmaTidVector(ownData));
+ break;
+ default:
+ LOG_ERROR(ERRCODE_INTERNAL_ERROR, "Not supported type %d", type);
+ }
+ return std::move(ret);
+}
+
+bool Vector::isValid() const {
+ if (this->getParentTupleBatch() != nullptr) {
+ assert(getNumOfRows() == this->getParentTupleBatch()->getNumOfRows());
+ if (selectListOwner == nullptr) {
+ // means no selected list, for example, just cloned from another vector
+ assert(this->selected == nullptr ||
+ this->selected == belongingTupleBatch->getSelected());
+ }
+ }
+
+ return true;
+}
+
+void Vector::setSelected(const SelectList *selected, bool fromBatch) {
+ if (fromBatch) {
+ assert(selectListOwner == nullptr && "expect plain vector");
+ this->selected = const_cast<SelectList *>(selected);
+ } else {
+ if (selectListOwner) {
+ assert(this->selected == selectListOwner.get());
+ selectListOwner->resize(0);
+ } else {
+ selectListOwner.reset(new SelectList);
+ this->selected = selectListOwner.get();
+ }
+ *(this->selected) = (*selected);
+ }
+ if (typeKind == TypeKind::STRUCTID) {
+ for (auto &child : childs) {
+ child->setSelected(selected, fromBatch);
+ }
+ }
+}
+
+bool Vector::equal(const Vector &a, const Vector &b) {
+ if (a.getTypeKind() != b.getTypeKind() ||
+ a.getNumOfRows() != b.getNumOfRows()) {
+ return false;
+ }
+
+ uint64_t nA = a.getNumOfRows();
+ for (uint64_t i = 0; i < nA; i++) {
+ if (!equal(a, i, b, i)) return false;
+ }
+
+ return true;
+}
+
+bool Vector::equal(const Vector &a, uint64_t index1, const Vector &b,
+ uint64_t index2) {
+ if (a.getTypeKind() != b.getTypeKind()) {
+ return false;
+ }
+
+ uint64_t l1, l2;
+ bool n1, n2;
+
+ auto *d1 = a.read(index1, &l1, &n1);
+ auto *d2 = b.read(index2, &l2, &n2);
+
+ if (n1 != n2 || l1 != l2) {
+ return false;
+ }
+
+ if (n1 || 0 == memcmp(d1, d2, l1)) {
+ return true;
+ }
+
+ return false;
+}
+
+std::string Vector::toString() const {
+ std::stringstream ss;
+
+ // FIXME(chiyang): Compromise decimal with double
+ if (TypeKind::DECIMALNEWID == typeKind) {
+ for (auto idx = 0; idx < this->getNumOfRows(); idx++) {
+ bool isNull;
+ std::string tmp = this->read(idx, &isNull);
+ if (!isNull) {
+ tmp = std::to_string(std::stod(tmp));
+ }
+ ss << tmp << " ";
+ }
+ return ss.str();
+ }
+
+ for (auto idx = 0; idx < this->getNumOfRows(); idx++) {
+ bool isNull;
+ std::string tmp = this->read(idx, &isNull);
+ ss << tmp << " ";
+ }
+ return ss.str();
+}
+
+void Vector::resize(size_t plainSize, const SelectList *sel,
+ const bool *__restrict__ nulls1,
+ const bool *__restrict__ nulls2,
+ const bool *__restrict__ nulls3) {
+ assert(this->getOwnData());
+ if (sel) this->setSelected(sel, true);
+
+ if (nulls1 || nulls2 || nulls3) {
+ this->setHasNull(true);
+ this->nulls.reset(plainSize, sel, nulls1, nulls2, nulls3);
+ } else {
+ this->setHasNull(false);
+ }
+
+ resizeInternal(plainSize);
+}
+
+void Vector::resize(size_t plainSize, const SelectList *sel, bool null) {
+ assert(this->getOwnData());
+ if (sel) this->setSelected(sel, true);
+
+ if (null) {
+ this->setHasNull(true);
+ this->nulls.reset(plainSize, sel, null);
+ } else {
+ this->setHasNull(false);
+ }
+
+ resizeInternal(plainSize);
+}
+
+} // namespace dbcommon
diff --git a/depends/dbcommon/src/dbcommon/common/vector.h b/depends/dbcommon/src/dbcommon/common/vector.h
new file mode 100644
index 0000000..c2d2408
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector.h
@@ -0,0 +1,763 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_H_
+
+#include <algorithm>
+#include <cassert>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "dbcommon/common/node-deserializer.h"
+#include "dbcommon/common/node-serializer.h"
+#include "dbcommon/hash/cdb-hash.h"
+#include "dbcommon/hash/fast-hash.h"
+#include "dbcommon/nodes/datum.h"
+#include "dbcommon/nodes/scalar.h"
+#include "dbcommon/nodes/select-list.h"
+#include "dbcommon/type/bool.h"
+#include "dbcommon/type/date.h"
+#include "dbcommon/type/float.h"
+#include "dbcommon/type/integer.h"
+#include "dbcommon/type/interval.h"
+#include "dbcommon/type/type-modifier.h"
+#include "dbcommon/type/varlen.h"
+#include "dbcommon/utils/bool-buffer.h"
+#include "dbcommon/utils/byte-buffer.h"
+#include "dbcommon/utils/macro.h"
+#include "dbcommon/utils/mb/mb-converter.h"
+
+namespace dbcommon {
+
+typedef struct VectorStatistics {
+ Datum minimum;
+ Datum maximum;
+ Datum sum;
+ uint64_t valueCount;
+ bool hasMinMaxStats;
+} VectorStatistics;
+
+class TupleBatch;
+class GroupKeys;
+
+// A vector typically represents a column.
+// It can own the data, or only reference the external data.
+//
+// A vector can be a "plain vector" or a "selected vector". A "plain vector"
+// is a vector without a "selected array".
+// A "selected" vector is a vector with a "selected array".
+// A "selected array" is used to accelerate
+// selection/filter operations. For example, if there is a filter "x > 3"
+// on column x, if there is only a small number of elements
+// satisfying the condition, then we can use the original vector
+// with a selected array which store the indexes of result elements,
+// instead of generating a new result vector.
+//
+// Now we distinguish two operations: "materialize" and "plainize".
+// "materialize" operation on a vector that does not own the data will make a
+// copy of the data, and it also make a "selected vector" as a "plain vector".
+//
+// "plainizing" a vector only convert a "selected vector" to a "plain vector".
+// So "plainizing" a plain vector will do nothing.
+// but "materializing" the vector will make it "materialized"
+//
+class Vector : public Object {
+ public:
+ typedef std::unique_ptr<Vector> uptr;
+
+ explicit Vector(bool ownData) : nulls(ownData), values(ownData) {
+ if (ownData) nulls.reserve(DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ }
+
+ virtual ~Vector() {}
+
+ virtual void convert(const dbcommon::SupportedEncodingSet inputEncoding,
+ const dbcommon::SupportedEncodingSet outputEncoding) {}
+
+ // Is the vector owning the data or just reference the real data?
+ // @return A bool value indicating whether the vector owns the data
+ // or not
+ bool getOwnData() const { return values.getOwnData(); }
+
+ // Get the element type of this vector
+ // @return The element type
+ TypeKind getTypeKind() const { return typeKind; }
+
+ // Set the element type of this vector
+ // @param typeKind The type for the vector
+ // @return void
+ void setTypeKind(TypeKind typeKind) { this->typeKind = typeKind; }
+
+ bool hasNullValue() const { return hasNulls; }
+ void setHasNull(bool hasNull) { this->hasNulls = hasNull; }
+
+ virtual bool checkZeroValue() const { return false; }
+
+ virtual bool checkLessZero() const { return false; }
+
+ virtual bool checkLogarithmAgrumentError() const { return false; }
+
+ virtual bool checkExponentialArgumentError() const { return false; }
+
+ // Is the element at "index" a null?
+ // @param index The position of the element
+ // @return A bool value indicating whether the element is null?
+ bool isNull(uint64_t index) const {
+ return isNullPlain(getPlainIndex(index));
+ }
+
+ // Is the element at the given plain "index" a null?
+ // @param index The plain index
+ // @return A bool value indicating whether the element is null?
+ bool isNullPlain(uint64_t index) const {
+ return hasNulls ? nulls.get(index) : false;
+ }
+
+ void setNull(uint64_t index) { nulls.set(getPlainIndex(index), true); }
+ void setNullPlain(uint64_t index) { nulls.set(index, true); }
+
+ // Append a null value to the "nulls" array
+ // @param null The input value
+ // @return void
+ inline void appendNull(bool null) {
+ nulls.append(null);
+ if (selectListOwner) {
+ selectListOwner->push_back(this->getNumOfRowsPlain() - 1);
+ }
+ }
+
+ // Get the size (in BYTEs) of bitmap array for nulls.
+ // A null value occupies only one bit.
+ // For example, 9 values use 2 bytes
+ // @return The size of the bitmap array for nulls.
+ uint64_t getNullBitMapNumBytesPlain() const {
+ return nulls.bitDataSizeInBytes();
+ }
+
+ // Get the number of elements in the null bitmap
+ // For example, having 9 values will return 9
+ // @return The number of elements in the null bitmap
+ uint64_t getNullsSize() const { return nulls.size(); }
+
+ const bool *getNulls() const { return hasNulls ? nulls.data() : nullptr; }
+
+ // Get the bitmap array for nulls. A null value occupies only one bit.
+ // @return The bitmap array for nulls.
+ const char *getNullBits() const { return nulls.getBitData(); }
+
+ // Set the null bitmap
+ // @param data The bitmap in byte array
+ // @param size The number of bool values in the array.
+ // @return void
+ void setNullBits(const char *nulls, uint64_t size) {
+ assert(!getOwnData());
+ this->nulls.setBitData(nulls, size);
+ }
+
+ void setNulls(const bool *nulls, uint64_t size) {
+ this->nulls.setBools(reinterpret_cast<const char *>(nulls),
+ size * sizeof(bool));
+ }
+
+ void setNotNulls(const char *notNulls, uint64_t size) {
+ this->nulls.setReserseBools(notNulls, size);
+ }
+
+ // Get the number of rows/elements in the vector
+ // @return The number of rows/elements in the vector
+ virtual uint64_t getNumOfRows() const {
+ return selected ? selected->size() : getNumOfRowsPlain();
+ }
+
+ // Get the number of rows/elements in the vector
+ // For a plain vector, return the number of rows.
+ // For a selected vector, does not consider the select vector
+ // @return The number of rows/elements in the vector
+ virtual uint64_t getNumOfRowsPlain() const = 0;
+
+ // Whether all elements are selected.
+ // @return A bool value indicating whether all elements are selected
+ bool allSelected() const {
+ return !selected || selected->size() == getNumOfRowsPlain();
+ }
+
+ // Get the select list
+ // @return The selected list
+ SelectList *getSelected() { return selected; }
+ const SelectList *getSelected() const { return selected; }
+
+ // Set the selected array. If it is from belonging tupleBatch,
+ // this vector only keeps a reference to the selected array.
+ // If it is not from belonging tupleBatch, this vector will
+ // swap the content of a local select array with the input select array.
+ // @param selected The selected array
+ // @param fromBatch Indicating whether it is from beloning tupleBatch
+ // @return void
+ void setSelected(const SelectList *selected, bool fromBatch);
+
+ // Get the value array. Please note that, if the vector
+ // is a "selected vector", then the values might contain
+ // some unnecessary data.
+ // @return The value array
+ const char *getValue() const { return values.data(); }
+
+ // Get the nanosecond value array. Only used in TimestampVector
+ // @return The nanosecode value array
+ virtual const char *getNanoseconds() const { return nullptr; }
+
+ // Get the auxiliary value array. Only used in DecimalVector
+ // @return The auxiliary value array
+ virtual const char *getAuxiliaryValue() const { return nullptr; }
+
+ // Get the scale value array. Only used in DecimalVector
+ // @return The scale value array
+ virtual const char *getScaleValue() const { return nullptr; }
+
+ // Get the day value array. Only used in IntervalVector
+ // @return The day value array
+ virtual const char *getDayValue() const { return nullptr; }
+
+ // Get the month value array. Only used in IntervalVector
+ // @return The month value array
+ virtual const char *getMonthValue() const { return nullptr; }
+
+ // Set the value array. This vector does not own the data.
+ // @param value The value array
+ // @param size The number of bytes in the array.
+ // @return void
+ virtual void setValue(const char *value, uint64_t size) = 0;
+
+ // Set the nanosecond value array. This vector does not own the data.
+ // Only used in TimestampVector
+ // @param value The value array
+ // @param size The number of bytes in the array.
+ // @return void
+ virtual void setNanoseconds(const char *value, uint64_t size) {}
+
+ // Set the auxiliary value array. This vector does not own the data.
+ // Only used in DecimalVector
+ // @param value The auxiliary value array
+ // @param size The number of bytes in the array.
+ // @return void
+ virtual void setAuxiliaryValue(const char *value, uint64_t size) {}
+ // Set the scale value array. This vector does not own the data.
+ // Only used in DecimalVector
+ // @param value The scale value array
+ // @param size The number of bytes in the array.
+ // @return void
+ virtual void setScaleValue(const char *value, uint64_t size) {}
+ // Set the day value array. This vector does not own the data.
+ // Only used in IntervalVector
+ // @param value The day value array
+ // @param size The number of bytes in the array.
+ // @return void
+ virtual void setDayValue(const char *value, uint64_t size) {}
+ // Set the month value array. This vector does not own the data.
+ // Only used in IntervalVector
+ // @param value The day value array
+ // @param size The number of bytes in the array.
+ // @return void
+ virtual void setMonthValue(const char *value, uint64_t size) {}
+
+ // Get the offset array, Please note that, if the vector
+ // is a "selected vector", then the data might contain
+ // some unnecessary data.
+ // @return the length array
+ virtual const uint64_t *getLengths() const { return nullptr; }
+
+ // Set the length array.
+ // @param lengthsPtr The length array
+ // @param size The number of elements (uint64_t) in the array.
+ // @return void
+ virtual void setLengths(const uint64_t *lengthsPtr, uint64_t size) {}
+
+ // Get the offset array
+ // @return The the offset array.
+ virtual const char **getValPtrs() const { return nullptr; }
+
+ // Set the value pointer array
+ // @param valPtrs The value pointer array
+ // @param size The number of elements in the value pointer array
+ // @return void
+ virtual void setValPtrs(const char **valPtrs, uint64_t size) {}
+
+ // Compute value pointers from length array and value array
+ // If valPtrs does not own the data, it will be changed to own the data
+ // @return void
+ virtual void computeValPtrs() {}
+
+ // Read a value at index of row vector and return as binary.
+ // @param index The row vector index at which position the value will be read.
+ // @param len The output variable which will be set to the returned number of
+ // bytes.
+ // @param isNull The output variable
+ // which will be set to true if the value is null.
+ // @return Return a constant buffer pointer to the data.
+ virtual const char *read(uint64_t index, uint64_t *len, bool *null) const = 0;
+
+ // Read a value at index of row vector and convert to sting
+ // @param index The row vector index at which position the value will be read.
+ // @param isNull The output variable
+ // which will be set to true if the value is null.
+ // @return Return the string representation of data at index.
+ std::string read(uint64_t index, bool *null) const {
+ return readPlain(getPlainIndex(index), null);
+ }
+
+ // Read a value at plain index of row vector and convert to sting
+ virtual std::string readPlain(uint64_t index, bool *null) const = 0;
+
+ // Print debug string
+ std::string toString() const;
+
+ // Read a value at plain index of row vector and convert to dbcommon::Scalar
+ // @param[in] index
+ // @param[out] scalar
+ virtual void readPlainScalar(uint64_t index, Scalar *scalar) const = 0;
+
+ // Append a value to the vector end.
+ // @param v The string representation of the value.
+ // @param v If appended value is null.
+ // @return void
+ virtual void append(const std::string &strValue, bool null) = 0;
+
+ // Append a value in binary format to the vector end.
+ // @param v The value to append.
+ // @param valueLen The size of the value
+ // @param isNull Indicate whether it is a null.
+ // @return void
+ virtual void append(const char *v, uint64_t valueLen, bool null) = 0;
+
+ // Append a datum to vector end.
+ // @param datum The input datum
+ // @param null The bool value indicating whether it is null.
+ // @return void
+ virtual void append(const Datum &datum, bool null) = 0;
+ virtual void append(const Datum &valDatum, const Datum &nanoDatum,
+ bool null) {}
+
+ // Append a Scalar to Vector.
+ virtual void append(const Scalar *scalar) = 0;
+
+ virtual void append(Vector *src, uint64_t index) = 0;
+
+ // Append the input vector
+ // All child class should call Vector::append too.
+ // @return void
+ virtual void append(Vector *vect) = 0;
+
+ // Append the subset of vector according to selected
+ // @param vector The source vector
+ // @param selected The list of rows selected to be appended.
+ // @return void
+ virtual void append(Vector *vect, SelectList *sel) {
+ assert(vect != nullptr);
+ assert(selected == nullptr);
+
+ int oldSize = this->getNumOfRowsPlain();
+
+ if (vect->hasNullValue()) {
+ if (!hasNullValue()) nulls.setReserseBools(false, oldSize);
+ setHasNull(true);
+
+ const dbcommon::BoolBuffer *inputNulls = vect->getNullBuffer();
+ if (!sel || sel->size() == vect->getNumOfRowsPlain()) {
+ nulls.append(inputNulls->data(), inputNulls->size());
+ } else {
+ for (auto i : *sel) {
+ nulls.append(inputNulls->get(i));
+ }
+ }
+
+ } else if (hasNullValue()) {
+ uint64_t newSize = sel ? sel->size() : vect->getNumOfRowsPlain();
+ for (uint64_t i = 0; i < newSize; i++) {
+ nulls.append(false);
+ }
+ }
+
+ if (selected && selectListOwner) {
+ assert(belongingTupleBatch == nullptr);
+ int numAppend = sel ? sel->size() : vect->getNumOfRowsPlain();
+ for (int i = 0; i < numAppend; i++) {
+ this->selected->push_back(oldSize++);
+ }
+ }
+ }
+
+ // Resize the Vector for subsequent operation that requires sufficient
+ // allocated buffer space and fill the NullVector according to the input
+ // NullVector.
+ void resize(size_t plainSize, const SelectList *sel,
+ const bool *__restrict__ nulls1,
+ const bool *__restrict__ nulls2 = nullptr,
+ const bool *__restrict__ nulls3 = nullptr);
+
+ // Resize the Vector for subsequent operation that requires sufficient
+ // allocated buffer space and fill the NullVector according to the input
+ // NullValue.
+ void resize(size_t plainSize, const SelectList *sel, bool null);
+
+ // Reset the vector to an empty Vector.
+ // All child class should call Vector::reset too.
+ // @param resetBelongingTupleBatch Bool value indicating whether
+ // resetting belonging tuple batch
+ // @return void
+ virtual void reset(bool resetBelongingTupleBatch = false) {
+ if (hasNulls) nulls.resize(0);
+
+ selected = nullptr;
+ selectListOwner.reset();
+
+ if (resetBelongingTupleBatch) this->belongingTupleBatch = nullptr;
+ }
+
+ void clear() override { this->reset(true); }
+
+ // Materialize the vector. Materialization means if the vector
+ // does not own data, it will make a copy of the data. And
+ // if the vector is a "selected vector", it will make it a "plain vector".
+ // @return void
+ virtual void materialize() = 0;
+
+ // Clone a vector, the result vector is a plain vector regardless
+ // of whether the input vector is a plain vector, or selected vector
+ // @return The vector cloned
+ std::unique_ptr<Vector> clone() const { return cloneSelected(selected); }
+
+ // Clone a plain vector according to the input select list(plain index).
+ // The result vector is a plain vector
+ // @param sel The input select list, plain index
+ // @return The vector cloned
+ virtual std::unique_ptr<Vector> cloneSelected(
+ const SelectList *sel) const = 0;
+
+ // Clone a plain vector according to the input select list in some range
+ // The result vector is a plain vector
+ // @param sel The input select list, plain index
+ // @param start The start input plain index
+ // @param end The end input plain index (not included)
+ // @return The vector cloned
+ virtual std::unique_ptr<Vector> extract(const SelectList *sel, uint64_t start,
+ uint64_t end) const = 0;
+
+ // Replicate the specified element at given index for given times
+ // @param index The element index
+ // @param replicateNum The number of replicates
+ // @return The replicated vector
+ virtual std::unique_ptr<Vector> replicateItemToVec(
+ uint64_t index, int replicateNum) const = 0;
+
+ // Materialize several selected vectors. Dedicated for CASE expression.
+ // @param[in] data Array of Vector*, SelectList* or Scalar*
+ // @param[in] sels Array of SelectList*, nullptr indicate no match
+ virtual void merge(size_t plainSize, const std::vector<Datum> &data,
+ const std::vector<SelectList *> &sels) {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED, "not implemented");
+ }
+
+ // Calculate hash value for each item
+ // @return The vector with the hash values
+ virtual void hash(std::vector<uint64_t> &retval) const = 0; // NOLINT
+
+ // Calculate hash value for each item (used for shuffle)
+ // @return The vector with the hash values
+ virtual void cdbHash(std::vector<uint64_t> &retval) const = 0; // NOLINT
+
+ // Get the memory size in Bytes for this vector
+ // It only includes the number of bytes for data, offsets and bitmap.
+ // Note that this only works for "plain vector"
+ // @return The data size
+ virtual uint64_t getMemorySizePlain() const {
+ return getSerializeSizeInternal();
+ }
+
+ // Get the containing tuple batch
+ // @return The containing tuple batch
+ virtual TupleBatch *getParentTupleBatch() const {
+ return this->belongingTupleBatch;
+ }
+
+ // Set the containing tuple batch for this vector
+ // @param parent The parent tuple batch
+ // @return void
+ virtual void setParentTupleBatch(TupleBatch *parent) {
+ this->belongingTupleBatch = parent;
+ }
+
+ // Get the plain index of an element.
+ // For example, a tuple batch has 10 tuples in its data array,
+ // and it has a select list of 3 tuples in position 0, 3, 5 respectively.
+ // So it means it has only 3 tuples.
+ // So if we want to get the real index of tuple 1,
+ // this function should return 3.
+ // @param The input index
+ // @return The real index
+ uint64_t getPlainIndex(uint64_t index) const {
+ assert(index >= 0 && index < getNumOfRows() && "invalid input");
+ return selected ? (*selected)[index] : index;
+ }
+
+ // Get the null buffer
+ // @return The null buffer. If this vector is not nullable, return nullptr
+ const dbcommon::BoolBuffer *getNullBuffer() const {
+ assert(hasNulls);
+ return &nulls;
+ }
+
+ // Get the null buffer
+ // @return The null buffer. If this vector is not nullable, return nullptr
+ dbcommon::BoolBuffer *getNullBuffer() {
+ assert(hasNulls);
+ return &nulls;
+ }
+
+ // Get the value buffer of this vector
+ // @return The value buffer
+ virtual dbcommon::ByteBuffer *getValueBuffer() = 0;
+
+ // Get the nanosecond value buffer. Only used in TimestampVector
+ // @return The nanosecond value buffer
+ virtual dbcommon::ByteBuffer *getNanosecondsBuffer() { return nullptr; }
+
+ // Get the auxiliary value buffer. Only used in DecimalVector
+ // to store the high bits of Int128 data
+ // @return The auxiliary value buffer
+ virtual dbcommon::ByteBuffer *getAuxiliaryValueBuffer() { return nullptr; }
+
+ // Get the scale value buffer. Only used in DecimalVector
+ // @return The scale value buffer
+ virtual dbcommon::ByteBuffer *getScaleValueBuffer() { return nullptr; }
+
+ // Get the day value buffer. Only used in IntervalVector
+ // @return The day value buffer
+ virtual dbcommon::ByteBuffer *getDayBuffer() { return nullptr; }
+
+ // Get the month value buffer. Only used in IntervalVector
+ // @return The month value buffer
+ virtual dbcommon::ByteBuffer *getMonthBuffer() { return nullptr; }
+
+ // Return true if the vector is valid, used only for validity check
+ // to find potential bugs
+ // @return True if the vector is valid, False otherwise
+ virtual bool isValid() const;
+
+ // Build a vector in given type
+ // @param type The vector element type
+ // @param ownData The bool value indicating whether the vector owns data
+ // @return The vector built
+ static std::unique_ptr<Vector> BuildVector(TypeKind type, bool ownData,
+ int64_t typeModifier = -1);
+
+ // Check whether the elements at the specified index are equal
+ // @param v1 The first input vector
+ // @param index1 The first index
+ // @param v2 The second input vector
+ // @param index2 The second index
+ // @return True if the two elements are equal, False if not
+ static bool equal(const Vector &v1, uint64_t index1, const Vector &v2,
+ uint64_t index2);
+
+ // Check whether the two vectors are equal
+ // @param v1 The first input vector
+ // @param v2 The second input vector
+ // @return True if the two vectors are equal, False if not
+ static bool equal(const Vector &v1, const Vector &v2);
+
+ // Swap the content of the two vectors
+ // @param The input vector to swap with this vector
+ // @return void
+ virtual void swap(Vector &vect) {
+ this->nulls.swap(vect.nulls);
+ std::swap(this->selectListOwner, vect.selectListOwner);
+ std::swap(this->selected, vect.selected);
+ std::swap(this->belongingTupleBatch, vect.belongingTupleBatch);
+ std::swap(this->typeKind, vect.typeKind);
+ }
+
+ // Check for group by, it will change the content of dataPtrSrc.
+ // dataPtrSrc will be set to point to next column if matched, otherwise
+ // nullptr.
+ // xxx Need to be familiar with the implementation of groupKeys in executor
+ virtual void checkGroupByKeys(
+ std::vector<const void *> &dataPtrSrc) const { // NOLINT
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED, "not implemented");
+ }
+
+ // This function is for comparing the data at specific position.
+ // @param dataPtr points to the data with the same data type of the vector
+ // @return The length that dataPtr have moved forward if matched, otherwise 0
+ // the data that dataPtr points to is organized as
+ // Fixed length column:
+ // |isNull(with padding to the size of the type) |data
+ // Variable length column:
+ // |[length(8 byte)] |isNull(1 byte without padding)|data
+ virtual uint64_t checkGroupByKey(uint64_t index, const char *dataPtr) const {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED, "not implemented");
+ }
+
+ virtual void addChildVector(std::unique_ptr<Vector> vec) {
+ if (childs.size() > 0) {
+ assert(vec->getNumOfRows() == childs[0]->getNumOfRows());
+ }
+ childs.push_back(std::move(vec));
+ }
+
+ virtual Vector *getChildVector(int index) { return childs[index].get(); }
+
+ virtual uint64_t getChildSize() const { return childs.size(); }
+
+ uint64_t getSerializeSize() { return getSerializeSizeInternal(); }
+
+ void serialize(NodeSerializer *serializer) { serializeInternal(serializer); }
+
+ static std::unique_ptr<Vector> deserialize(NodeDeserializer *deserializer) {
+ TypeKind type = deserializer->read<TypeKind>();
+ uint64_t typeModifier = deserializer->read<uint64_t>();
+ std::unique_ptr<Vector> vect =
+ Vector::BuildVector(type, false, typeModifier);
+ vect->deserializeInternal(deserializer);
+ return std::move(vect);
+ }
+
+ // Get estimated size of serialized bytes.
+ virtual uint64_t getSerializeSizeInternal() const {
+ uint64_t sSize = sizeof(typeKind) + sizeof(typeModifier) +
+ sizeof(uint64_t) + sizeof(hasNulls);
+ if (hasNulls) {
+ sSize += sizeof(uint64_t) + getNullBitMapNumBytesPlain();
+ }
+ return sSize;
+ }
+
+ // Serialize this vector
+ // @param serializer The serializer
+ // @return void
+ virtual void serializeInternal(NodeSerializer *serializer) {
+ // write column type
+ serializer->write<TypeKind>(getTypeKind());
+ serializer->write<uint64_t>(typeModifier);
+ assert(typeKind != CHARID || typeModifier != -1);
+
+ // write the number of rows
+ serializer->write<uint64_t>(getNumOfRows());
+
+ // write the bitmap
+ serializer->write<bool>(hasNulls);
+ if (hasNulls) {
+ if (allSelected()) {
+ uint64_t bsz = nulls.bitDataSizeInBytes();
+ serializer->write<uint64_t>(bsz);
+ serializer->writeBytes(getNullBits(), bsz);
+ } else {
+ dbcommon::BoolBuffer buf(true);
+ for (auto idx : *selected) buf.append(nulls.get(idx));
+ uint64_t bsz = buf.bitDataSizeInBytes();
+ serializer->write<uint64_t>(bsz);
+ serializer->writeBytes(buf.getBitData(), bsz);
+ }
+ }
+ }
+
+ // Deserialize this vector
+ // @param deserializer The deserializer
+ // @return void
+ virtual void deserializeInternal(NodeDeserializer *deserializer) {
+ uint64_t nRows = deserializer->read<uint64_t>();
+
+ this->hasNulls = deserializer->read<bool>();
+ if (this->hasNulls) {
+ uint64_t bsz = deserializer->read<uint64_t>();
+ assert((nRows + 7) / 8 == bsz);
+ if (bsz > 0) {
+ const char *bstr = deserializer->readBytes(bsz);
+ this->setNullBits(bstr, nRows);
+ }
+ }
+ }
+
+ void setVectorStatistics(const VectorStatistics &s) {
+ hasStats = true;
+ stats = s;
+ }
+
+ const VectorStatistics *getVectorStatistics() {
+ if (hasStats)
+ return &stats;
+ else
+ return nullptr;
+ }
+
+ int64_t getTypeModifier() { return typeModifier; }
+ void setTypeModifier(int64_t typeModifier) {
+ this->typeModifier = typeModifier;
+ }
+
+ virtual void setDirectEncoding(bool isDirectEncoding) {}
+
+ virtual void trim() {}
+
+ // Select the elements that "distinct from" the element stored in scalar
+ // "Distinct from" operator differs from "Not equal" in cases when either
+ // operand is NULL.
+ // @param selected The SelectList to be returned
+ // @param scalar The Scalar to be compared
+ // @return void
+ virtual void selectDistinct(SelectList *selected, Scalar *scalar) = 0;
+
+ // Select the elements that "distinct from" the elements
+ // at the same row in vect. "Distinct from" operator differs from "Not equal"
+ // in cases when either operand is NULL.
+ // @param selected The SelectList to be returned
+ // @param vect The second Vector to be compared
+ // @return void
+ virtual void selectDistinct(SelectList *selected, Vector *vect) = 0;
+
+ protected:
+ // Resize the Vector for subsequent operation that requires sufficient
+ // allocated buffer space.
+ virtual void resizeInternal(size_t plainSize) = 0;
+
+ TupleBatch *belongingTupleBatch = nullptr;
+
+ SelectList *selected = nullptr;
+ std::unique_ptr<SelectList> selectListOwner;
+
+ int64_t typeModifier = -1; // -1 for no type modifier
+
+ // whether we only hava stripe statistics
+ bool hasStats = false;
+ VectorStatistics stats;
+
+ // whether there are any null values
+ bool hasNulls = true;
+ dbcommon::BoolBuffer nulls;
+
+ dbcommon::ByteBuffer values;
+ TypeKind typeKind = INVALIDTYPEID;
+ std::vector<std::unique_ptr<Vector>> childs;
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_H_
diff --git a/depends/dbcommon/src/dbcommon/common/vector/decimal-vector.cc b/depends/dbcommon/src/dbcommon/common/vector/decimal-vector.cc
new file mode 100644
index 0000000..027097d
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector/decimal-vector.cc
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "dbcommon/common/vector/decimal-vector.h"
+
+#include <algorithm>
+
+#include "dbcommon/common/vector-transformer.h"
+#include "dbcommon/utils/string-util.h"
+namespace dbcommon {
+
+const char *DecimalVector::read(uint64_t index, uint64_t *len,
+ bool *null) const {
+ assert(index < getNumOfRows() && "invalid input");
+ assert(nullptr != null && "invalid input");
+ assert(nullptr != len && "invalid input");
+
+ uint64_t i = getPlainIndex(index);
+ *null = isNullPlain(i);
+
+ assert(nextBuffer == 0 || nextBuffer == 1);
+ nextBuffer = 1 - nextBuffer;
+ readBuffer[nextBuffer].lowbits =
+ *reinterpret_cast<uint64_t *>(values.data() + (i * (sizeof(uint64_t))));
+ readBuffer[nextBuffer].highbits = *reinterpret_cast<int64_t *>(
+ auxiliaryValues.data() + (i * (sizeof(int64_t))));
+ readBuffer[nextBuffer].scale = *reinterpret_cast<int64_t *>(
+ scaleValues.data() + (i * (sizeof(int64_t))));
+ *len = sizeof(int64_t) + sizeof(uint64_t) + sizeof(int64_t);
+ return reinterpret_cast<const char *>(&readBuffer[nextBuffer]);
+}
+
+std::string DecimalVector::readPlain(uint64_t index, bool *null) const {
+ *null = isNullPlain(index);
+ if (*null) return "";
+ return std::move(DecimalType::toString(auxiliaryValues.get<int64_t>(index),
+ values.get<uint64_t>(index),
+ scaleValues.get<int64_t>(index)));
+}
+
+void DecimalVector::readPlainScalar(uint64_t index, Scalar *scalar) const {
+ assert(nextBuffer == 0 || nextBuffer == 1);
+ nextBuffer = 1 - nextBuffer;
+ readBuffer[nextBuffer].lowbits = *reinterpret_cast<uint64_t *>(
+ values.data() + (index * (sizeof(uint64_t))));
+ readBuffer[nextBuffer].highbits = *reinterpret_cast<int64_t *>(
+ auxiliaryValues.data() + (index * (sizeof(int64_t))));
+ readBuffer[nextBuffer].scale = *reinterpret_cast<int64_t *>(
+ scaleValues.data() + (index * (sizeof(int64_t))));
+ scalar->value =
+ CreateDatum(reinterpret_cast<const char *>(&readBuffer[nextBuffer]));
+ scalar->length = sizeof(DecimalVar);
+ scalar->isnull = isNullPlain(index);
+}
+
+void DecimalVector::append(const std::string &strValue, bool null) {
+ assert(getOwnData());
+ DecimalVar value = {0, 0, 0};
+ if (!null) {
+ value = DecimalType::fromString(strValue);
+ }
+
+ append(reinterpret_cast<char *>(&value), sizeof(DecimalVar), null);
+}
+
+void DecimalVector::append(const char *v, uint64_t valueLen, bool null) {
+ assert(getOwnData() && v != nullptr);
+ assert(valueLen == sizeof(int64_t) + sizeof(uint64_t) + sizeof(int64_t));
+
+ int64_t auxVal = *(reinterpret_cast<const int64_t *>(v));
+ uint64_t val = *(reinterpret_cast<const uint64_t *>(v + 8));
+ int64_t scale = *(reinterpret_cast<const int64_t *>(v + 16));
+ values.append<uint64_t>(val);
+ auxiliaryValues.append<int64_t>(auxVal);
+ scaleValues.append<int64_t>(scale);
+ appendNull(null);
+}
+
+void DecimalVector::append(const Datum &datum, bool null) {
+ dbcommon::DecimalVar *val = DatumGetValue<DecimalVar *>(datum);
+ if (val) {
+ values.append(val->lowbits);
+ auxiliaryValues.append(val->highbits);
+ scaleValues.append(val->scale);
+ } else {
+ assert(null == true);
+ values.append(uint64_t(0));
+ auxiliaryValues.append(int64_t(0));
+ scaleValues.append(int64_t(0));
+ }
+ appendNull(null);
+}
+
+void DecimalVector::append(const Scalar *scalar) {
+ dbcommon::DecimalVar *val = DatumGetValue<DecimalVar *>(scalar->value);
+ if (val) {
+ values.append(val->lowbits);
+ auxiliaryValues.append(val->highbits);
+ scaleValues.append(val->scale);
+ }
+ appendNull(scalar->isnull);
+}
+
+void DecimalVector::append(Vector *src, uint64_t index) {
+ bool isnull = false;
+ uint64_t len = 0;
+ const char *ret = src->read(index, &len, &isnull);
+ append(ret, len, isnull);
+}
+
+void DecimalVector::append(Vector *vect) {
+ assert(getOwnData());
+ this->append(vect, vect->getSelected());
+}
+
+void DecimalVector::append(Vector *vect, SelectList *sel) {
+ Vector::append(vect, sel);
+
+ const dbcommon::ByteBuffer *newValues = vect->getValueBuffer();
+ const dbcommon::ByteBuffer *newAuxiliaryValues =
+ vect->getAuxiliaryValueBuffer();
+ const dbcommon::ByteBuffer *newScaleValues = vect->getScaleValueBuffer();
+ if (!sel || sel->size() == vect->getNumOfRowsPlain()) {
+ values.append(newValues->data(), newValues->size());
+ auxiliaryValues.append(newAuxiliaryValues->data(),
+ newAuxiliaryValues->size());
+ scaleValues.append(newScaleValues->data(), newScaleValues->size());
+ } else {
+ uint64_t newSize = sel->size();
+ for (uint64_t i = 0; i < newSize; i++) {
+ values.append(newValues->get<uint64_t>((*sel)[i]));
+ auxiliaryValues.append(newAuxiliaryValues->get<int64_t>((*sel)[i]));
+ scaleValues.append(newScaleValues->get<int64_t>((*sel)[i]));
+ }
+ }
+}
+
+void DecimalVector::materialize() {
+ values.materialize<uint64_t>(selected);
+ auxiliaryValues.materialize<int64_t>(selected);
+ scaleValues.materialize<int64_t>(selected);
+ if (hasNulls) nulls.materialize(selected);
+ selected = nullptr;
+}
+
+std::unique_ptr<Vector> DecimalVector::cloneSelected(
+ const SelectList *sel) const {
+ assert(isValid());
+
+ std::unique_ptr<Vector> retval = Vector::BuildVector(getTypeKind(), false);
+ DecimalVector *ret = static_cast<DecimalVector *>(retval.get());
+
+ ret->setTypeKind(typeKind);
+ ret->setHasNull(hasNulls);
+ assert(values.size() == auxiliaryValues.size());
+ if (hasNulls) ret->setNulls(getNulls(), getNumOfRowsPlain());
+ ret->setValue(values.data(), values.size());
+ ret->setAuxiliaryValue(auxiliaryValues.data(), auxiliaryValues.size());
+ ret->setScaleValue(scaleValues.data(), scaleValues.size());
+
+ ret->setSelected(sel, true);
+ ret->materialize();
+
+ assert(ret->isValid());
+ return std::move(retval);
+}
+
+std::unique_ptr<Vector> DecimalVector::extract(const SelectList *sel,
+ uint64_t start,
+ uint64_t end) const {
+ assert(isValid() && start < end);
+ uint64_t total = getNumOfRowsPlain();
+ assert(end <= total);
+ uint64_t extSz = end - start;
+
+ std::unique_ptr<Vector> retval = Vector::BuildVector(getTypeKind(), false);
+ DecimalVector *ret = static_cast<DecimalVector *>(retval.get());
+
+ ret->setTypeKind(typeKind);
+ ret->setHasNull(hasNulls);
+ if (hasNulls) ret->setNulls(getNulls() + start, extSz);
+
+ uint64_t typLen = sizeof(int64_t) + sizeof(uint64_t) + sizeof(int64_t);
+ if (values.data()) {
+ ret->setValue(values.data() + start * typLen, extSz * typLen);
+ }
+ if (auxiliaryValues.data()) {
+ ret->setAuxiliaryValue(auxiliaryValues.data() + start * typLen,
+ extSz * typLen);
+ }
+ if (scaleValues.data()) {
+ ret->setScaleValue(scaleValues.data() + start * typLen, extSz * typLen);
+ }
+ ret->setSelected(sel, true);
+
+ assert(ret->isValid());
+ return std::move(retval);
+}
+
+std::unique_ptr<Vector> DecimalVector::replicateItemToVec(
+ uint64_t index, int replicateNum) const {
+ assert(isValid());
+ auto retval = Vector::BuildVector(getTypeKind(), true);
+ auto ret = static_cast<DecimalVector *>(retval.get());
+
+ ret->setTypeKind(typeKind);
+ ret->setHasNull(hasNulls);
+
+ dbcommon::ByteBuffer valbuf(true);
+ dbcommon::ByteBuffer auxbuf(true);
+ dbcommon::ByteBuffer scbuf(true);
+ uint64_t idx = getPlainIndex(index);
+ auto value = values.get<uint64_t>(idx);
+ auto auxValue = auxiliaryValues.get<int64_t>(idx);
+ auto scaleVal = scaleValues.get<int64_t>(idx);
+ auto null = isNullPlain(idx);
+
+ if (hasNulls) {
+ for (int i = 0; i < replicateNum; ++i) {
+ valbuf.append(value);
+ auxbuf.append(auxValue);
+ scbuf.append(scaleVal);
+ ret->nulls.append(null);
+ }
+ } else {
+ for (int i = 0; i < replicateNum; ++i) {
+ valbuf.append(values.get<uint64_t>(idx));
+ auxbuf.append(auxiliaryValues.get<int64_t>(idx));
+ scbuf.append(scaleValues.get<int64_t>(idx));
+ }
+ }
+ ret->values.swap(valbuf);
+ ret->auxiliaryValues.swap(auxbuf);
+ ret->scaleValues.swap(scbuf);
+ assert(ret->isValid());
+ return std::move(retval);
+}
+
+void DecimalVector::merge(size_t plainSize, const std::vector<Datum> &data,
+ const std::vector<SelectList *> &sels) {
+ assert(data.size() == sels.size());
+
+ values.resize(plainSize * sizeof(uint64_t));
+ auxiliaryValues.resize(plainSize * sizeof(int64_t));
+ scaleValues.resize(plainSize * sizeof(int64_t));
+ nulls.resize(plainSize);
+ uint64_t *__restrict__ lvals = reinterpret_cast<uint64_t *>(values.data());
+ int64_t *__restrict__ hvals =
+ reinterpret_cast<int64_t *>(auxiliaryValues.data());
+ int64_t *__restrict__ svals = reinterpret_cast<int64_t *>(scaleValues.data());
+ bool *__restrict__ nulls = this->nulls.data();
+
+ for (auto inputIdx = 0; inputIdx < data.size(); inputIdx++) {
+ auto sel = sels[inputIdx];
+ if (sel == nullptr) continue;
+ auto datum = DatumGetValue<Object *>(data[inputIdx]);
+
+ if (auto vec = dynamic_cast<Vector *>(datum)) {
+ assert(dynamic_cast<DecimalVector *>(vec));
+ DecimalVector *dvec = dynamic_cast<DecimalVector *>(vec);
+ auto lvalsIn = reinterpret_cast<const uint64_t *>(dvec->getValue());
+ auto hvalsIn =
+ reinterpret_cast<const int64_t *>(dvec->getAuxiliaryValue());
+ auto svalsIn = reinterpret_cast<const int64_t *>(dvec->getScaleValue());
+ for (auto idx : *sel) {
+ lvals[idx] = lvalsIn[idx];
+ hvals[idx] = hvalsIn[idx];
+ svals[idx] = svalsIn[idx];
+ }
+ if (dvec->hasNullValue()) {
+ auto nullsIn = dvec->getNulls();
+ for (auto idx : *sel) nulls[idx] = nullsIn[idx];
+ } else {
+ for (auto idx : *sel) nulls[idx] = false;
+ }
+ } else {
+ auto scalar = reinterpret_cast<Scalar *>(datum);
+ DecimalVar *val = DatumGetValue<DecimalVar *>(scalar->value);
+ auto null = scalar->isnull;
+ if (null) {
+ for (auto idx : *sel) {
+ nulls[idx] = true;
+ }
+ } else {
+ for (auto idx : *sel) {
+ lvals[idx] = val->lowbits;
+ hvals[idx] = val->highbits;
+ svals[idx] = val->scale;
+ nulls[idx] = false;
+ }
+ }
+ }
+ }
+}
+
+void DecimalVector::hash(std::vector<uint64_t> &retval) const {
+ // TODO(wshao) :Optimize the performance of hash function in terms of the
+ // both highbits and lowbits value.
+ auto base = reinterpret_cast<const int64_t *>(getValue());
+
+ if (selected) {
+ fast_hash(base, *selected, retval);
+ } else {
+ fast_hash(base, this->getNumOfRows(), retval);
+ }
+
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ if (nulls.get(getPlainIndex(i))) {
+ retval[i] = 0;
+ }
+ }
+ }
+}
+
+void DecimalVector::cdbHash(std::vector<uint64_t> &retval) const {
+ auto base = reinterpret_cast<const int64_t *>(getValue());
+ size_t len = sizeof(uint64_t);
+
+ if (selected) {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = nulls.get(idx) ? cdbhashnull(retval[i])
+ : cdbhash(&base[idx], len, retval[i]);
+ }
+ } else {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = cdbhash(&base[idx], len, retval[i]);
+ }
+ }
+ } else {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = nulls.get(i) ? cdbhashnull(retval[i])
+ : cdbhash(&base[i], len, retval[i]);
+ }
+ } else {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = cdbhash(&base[i], len, retval[i]);
+ }
+ }
+ }
+}
+
+void DecimalVector::swap(Vector &vect) {
+ Vector::swap(vect);
+ auto &v = dynamic_cast<DecimalVector &>(vect);
+ values.swap(v.values);
+ auxiliaryValues.swap(v.auxiliaryValues);
+ scaleValues.swap(v.scaleValues);
+}
+
+void DecimalVector::checkGroupByKeys(
+ std::vector<const void *> &dataPtrSrc) const {
+ const uint64_t *data = reinterpret_cast<const uint64_t *>(values.data());
+ const int64_t *auxdata =
+ reinterpret_cast<const int64_t *>(auxiliaryValues.data());
+ const int64_t *scdata = reinterpret_cast<const int64_t *>(scaleValues.data());
+ const uint64_t size = getNumOfRows();
+ const char **dataPtr = reinterpret_cast<const char **>(&dataPtrSrc[0]);
+ if (selected) {
+ auto &sel = *selected;
+
+#pragma clang loop unroll_count(4)
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ uint64_t index = sel[i];
+ char null = hasNulls ? nulls.getChar(index) : 0;
+ const uint64_t d = data[index];
+ const int64_t a = auxdata[index];
+ const int64_t s = scdata[index];
+
+ const DecimalDataInfo *info =
+ reinterpret_cast<const DecimalDataInfo *>(dataPtr[i]);
+ if (null == info->isNull &&
+ (null || (a == info->highbitsData && d == info->lowbitsData &&
+ s == info->scaleData)))
+ dataPtr[i] = alignedAddress(dataPtr[i] + sizeof(DecimalDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ } else {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ char null = nulls.getChar(i);
+ const uint64_t d = data[i];
+ const int64_t a = auxdata[i];
+ const int64_t s = scdata[i];
+
+ const DecimalDataInfo *info =
+ reinterpret_cast<const DecimalDataInfo *>(dataPtr[i]);
+ if (null == info->isNull &&
+ (null || (a == info->highbitsData && d == info->lowbitsData &&
+ s == info->scaleData)))
+ dataPtr[i] = alignedAddress(dataPtr[i] + sizeof(DecimalDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ } else {
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ char null = false;
+ const uint64_t d = data[i];
+ const int64_t a = auxdata[i];
+ const int64_t s = scdata[i];
+
+ const DecimalDataInfo *info =
+ reinterpret_cast<const DecimalDataInfo *>(dataPtr[i]);
+ if (null == info->isNull &&
+ (null || (a == info->highbitsData && d == info->lowbitsData &&
+ s == info->scaleData)))
+ dataPtr[i] = alignedAddress(dataPtr[i] + sizeof(DecimalDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ }
+ }
+}
+
+uint64_t DecimalVector::checkGroupByKey(uint64_t index,
+ const char *dataPtr) const {
+ assert(index < getNumOfRows() && "invalid input");
+ assert(getNumOfRows() > 0);
+ uint64_t i = getPlainIndex(index);
+ const DecimalDataInfo *info =
+ reinterpret_cast<const DecimalDataInfo *>(dataPtr);
+ if (isNullPlain(i) != info->isNull) return 0;
+ if (!info->isNull) {
+ if (*reinterpret_cast<uint64_t *>(values.data() + i * (sizeof(uint64_t))) !=
+ info->lowbitsData)
+ return 0;
+ if (*reinterpret_cast<int64_t *>(auxiliaryValues.data() +
+ i * (sizeof(int64_t))) !=
+ info->highbitsData)
+ return 0;
+ if (*reinterpret_cast<int64_t *>(scaleValues.data() +
+ i * (sizeof(int64_t))) != info->scaleData)
+ return 0;
+ }
+ return sizeof(DecimalDataInfo);
+}
+
+uint64_t DecimalVector::getSerializeSizeInternal() const {
+ size_t sSize = Vector::getSerializeSizeInternal();
+ sSize += sizeof(uint64_t);
+ sSize += values.size();
+ sSize += auxiliaryValues.size();
+ sSize += scaleValues.size();
+ return sSize;
+}
+
+void DecimalVector::serializeInternal(NodeSerializer *serializer) {
+ Vector::serializeInternal(serializer);
+
+ uint64_t vsz = getNumOfRows() * sizeof(int64_t);
+ serializer->write<uint64_t>(vsz * 3);
+ if (vsz > 0) {
+ if (allSelected()) {
+ serializer->writeBytes(getValue(), vsz);
+ serializer->writeBytes(getAuxiliaryValue(), vsz);
+ serializer->writeBytes(getScaleValue(), vsz);
+ } else {
+ auto values = reinterpret_cast<const uint64_t *>(getValue());
+ for (auto i : *selected) {
+ serializer->write(values[i]);
+ }
+ auto auxValues = reinterpret_cast<const int64_t *>(getAuxiliaryValue());
+ for (auto i : *selected) {
+ serializer->write(auxValues[i]);
+ }
+ auto scaleValues = reinterpret_cast<const int64_t *>(getScaleValue());
+ for (auto i : *selected) {
+ serializer->write(scaleValues[i]);
+ }
+ }
+ }
+}
+
+void DecimalVector::deserializeInternal(NodeDeserializer *deserializer) {
+ Vector::deserializeInternal(deserializer);
+
+ uint64_t vsz = deserializer->read<uint64_t>();
+ assert(vsz % 3 == 0);
+ if (vsz > 0) {
+ const char *dstr = deserializer->readBytes(vsz);
+ setValue(dstr, vsz / 3);
+ setAuxiliaryValue(dstr + vsz / 3, vsz / 3);
+ setScaleValue(dstr + vsz / 3 + vsz / 3, vsz / 3);
+ }
+
+ assert(isValid());
+}
+
+void DecimalVector::selectDistinct(SelectList *selected, Scalar *scalar) {
+ SelectList *sel = getSelected();
+ uint64_t sz = getNumOfRows();
+ const uint64_t *lowbit = reinterpret_cast<const uint64_t *>(values.data());
+ const int64_t *highbit =
+ reinterpret_cast<const int64_t *>(getAuxiliaryValue());
+ const int64_t *scale = reinterpret_cast<const int64_t *>(getScaleValue());
+
+ auto operation = [&](uint64_t plainIdx) { // NOLINT
+ if (!scalar->isnull) {
+ DecimalVar *value = DatumGetValue<DecimalVar *>(scalar->value);
+ return (lowbit[plainIdx] != value->lowbits ||
+ highbit[plainIdx] != value->highbits ||
+ scale[plainIdx] != value->scale);
+ }
+ return false;
+ };
+ const bool *nulls = getNulls();
+ dbcommon::transformDistinct(sz, sel, nulls, selected, scalar->isnull,
+ operation);
+}
+
+void DecimalVector::selectDistinct(SelectList *selected, Vector *vect) {
+ SelectList *sel1 = getSelected();
+ SelectList *sel2 = vect->getSelected();
+ uint64_t sz = getNumOfRows();
+ assert(sz == vect->getNumOfRows() && "invalid input");
+
+ const uint64_t *lowbit1 = reinterpret_cast<const uint64_t *>(values.data());
+ const int64_t *highbit1 =
+ reinterpret_cast<const int64_t *>(getAuxiliaryValue());
+ const int64_t *scale1 = reinterpret_cast<const int64_t *>(getScaleValue());
+ const uint64_t *lowbit2 =
+ reinterpret_cast<const uint64_t *>(vect->getValue());
+ const int64_t *highbit2 =
+ reinterpret_cast<const int64_t *>(vect->getAuxiliaryValue());
+ const int64_t *scale2 =
+ reinterpret_cast<const int64_t *>(vect->getScaleValue());
+
+ auto operation = [&](uint64_t plainIdx) { // NOLINT
+ return (lowbit1[plainIdx] != lowbit2[plainIdx] ||
+ highbit1[plainIdx] != highbit2[plainIdx] ||
+ scale1[plainIdx] != scale2[plainIdx]);
+ };
+ const bool *nulls1 = getNulls();
+ const bool *nulls2 = vect->getNulls();
+ dbcommon::transformDistinct(sz, sel1, nulls1, nulls2, selected, operation);
+}
+
+const char *DecimalVector::readRawValue(uint64_t index, uint64_t *len,
+ bool *null) const {
+ assert(index < getNumOfRows() && "invalid input");
+ assert(getNumOfRows() > 0);
+
+ assert(nullptr != null && "invalid input");
+ assert(nullptr != len && "invalid input");
+
+ uint64_t i = getPlainIndex(index);
+
+ *null = isNullPlain(i);
+
+ *len = lengths.get<uint64_t>(i);
+
+ return valPtrs.get<const char *>(i);
+}
+
+void DecimalVector::computeRawValueAndValPtrs() {
+ assert(rawValues.size() == 0);
+ assert(lengths.size() == 0);
+ assert(valPtrs.size() == 0);
+ int sz = getNumOfRowsPlain();
+ rawValues.reserve(DEFAULT_RESERVED_SIZE_OF_STRING * sz);
+ lengths.resize(sizeof(uint64_t) * sz);
+ uint64_t *lenVal = reinterpret_cast<uint64_t *>(lengths.data());
+ valPtrs.resize(lengths.size());
+ for (int i = 0; i < sz; ++i) {
+ bool isNull;
+ std::string plain = readPlain(i, &isNull);
+ rawValues.append(plain.c_str(), plain.length());
+ lenVal[i] = plain.length();
+ }
+ auto retrievedSize = this->computeValPtrs(
+ rawValues.data(), reinterpret_cast<const uint64_t *>(lengths.data()),
+ hasNulls ? getNulls() : nullptr, sz, (const char **)valPtrs.data());
+ assert(rawValues.size() == retrievedSize);
+}
+
+void DecimalVector::resizeInternal(size_t plainSize) {
+ auxiliaryValues.resize(plainSize * sizeof(uint64_t));
+ values.resize(plainSize * sizeof(uint64_t));
+ scaleValues.resize(plainSize * sizeof(uint64_t));
+}
+
+bool DecimalVector::checkZeroValue() const {
+ bool hasZero = false;
+ DecimalVectorRawData vec(
+ const_cast<Vector *>(reinterpret_cast<const Vector *>(this)));
+ auto checkZero = [&](SelectList::value_type plainIdx) {
+ hasZero |= (vec.hightbits[plainIdx] == 0 && vec.lowbits[plainIdx] == 0);
+ };
+ transformVector(vec.plainSize, vec.sel, vec.nulls, checkZero);
+ return hasZero;
+}
+
+} // namespace dbcommon
diff --git a/depends/dbcommon/src/dbcommon/common/vector/decimal-vector.h b/depends/dbcommon/src/dbcommon/common/vector/decimal-vector.h
new file mode 100644
index 0000000..bdb38b4
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector/decimal-vector.h
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_DECIMAL_VECTOR_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_DECIMAL_VECTOR_H_
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "dbcommon/common/vector.h"
+#include "dbcommon/type/decimal.h"
+
+namespace dbcommon {
+
+class DecimalVector : public Vector {
+ public:
+ explicit DecimalVector(bool ownData)
+ : Vector(ownData),
+ auxiliaryValues(ownData),
+ scaleValues(ownData),
+ rawValues(true),
+ lengths(true),
+ valPtrs(true) {
+ if (ownData) {
+ values.reserve(sizeof(uint64_t) * DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ auxiliaryValues.reserve(sizeof(int64_t) *
+ DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ scaleValues.reserve(sizeof(int64_t) * DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ }
+ setTypeKind(DECIMALNEWID);
+ }
+
+ ~DecimalVector() override {}
+
+ uint64_t getNumOfRowsPlain() const override {
+ return values.size() / sizeof(uint64_t);
+ }
+
+ dbcommon::ByteBuffer *getAuxiliaryValueBuffer() override {
+ return &auxiliaryValues;
+ }
+
+ dbcommon::ByteBuffer *getScaleValueBuffer() override { return &scaleValues; }
+
+ const char *getAuxiliaryValue() const override {
+ return auxiliaryValues.data();
+ }
+
+ const char *getScaleValue() const override { return scaleValues.data(); }
+
+ void setValue(const char *value, uint64_t size) override {
+ assert(!getOwnData() && value != nullptr && size >= 0);
+ values.setData(value, size);
+ }
+
+ void setAuxiliaryValue(const char *value, uint64_t size) override {
+ assert(!getOwnData() && value != nullptr && size >= 0);
+ auxiliaryValues.setData(value, size);
+ }
+
+ void setScaleValue(const char *value, uint64_t size) override {
+ assert(!getOwnData() && value != nullptr && size >= 0);
+ scaleValues.setData(value, size);
+ }
+
+ bool checkZeroValue() const override;
+
+ const char *read(uint64_t index, uint64_t *len, bool *null) const override;
+
+ std::string readPlain(uint64_t index, bool *null) const override;
+
+ void readPlainScalar(uint64_t index, Scalar *scalar) const override;
+
+ void append(const std::string &strValue, bool null) override;
+
+ void append(const char *v, uint64_t valueLen, bool null) override;
+
+ void append(const Datum &datum, bool null) override;
+
+ void append(const Scalar *scalar) override;
+
+ void append(Vector *src, uint64_t index) override;
+
+ void append(Vector *vect) override;
+
+ void append(Vector *vect, SelectList *sel) override;
+
+ void materialize() override;
+
+ void reset(bool resetBelongingTupleBatch = false) override {
+ Vector::reset(resetBelongingTupleBatch);
+ values.resize(0);
+ auxiliaryValues.resize(0);
+ scaleValues.resize(0);
+ rawValues.resize(0);
+ lengths.resize(0);
+ valPtrs.resize(0);
+ }
+
+ std::unique_ptr<Vector> cloneSelected(const SelectList *sel) const override;
+
+ std::unique_ptr<Vector> extract(const SelectList *sel, uint64_t start,
+ uint64_t end) const override;
+
+ std::unique_ptr<Vector> replicateItemToVec(uint64_t index,
+ int replicateNum) const override;
+
+ void merge(size_t plainSize, const std::vector<Datum> &data,
+ const std::vector<SelectList *> &sels) override;
+
+ bool isValid() const override {
+ Vector::isValid();
+
+ if (hasNulls) {
+ assert(nulls.size() == values.size() / sizeof(uint64_t));
+ assert(nulls.size() == auxiliaryValues.size() / sizeof(int64_t));
+ assert(nulls.size() == scaleValues.size() / sizeof(int64_t));
+ }
+ return true;
+ }
+
+ void hash(std::vector<uint64_t> &retval) const override;
+
+ void cdbHash(std::vector<uint64_t> &retval) const override;
+
+ void swap(Vector &vect) override;
+
+ dbcommon::ByteBuffer *getValueBuffer() override { return &values; }
+
+ const char **getValPtrs() const override {
+ return reinterpret_cast<const char **>(valPtrs.data());
+ }
+
+ const uint64_t *getLengths() const override {
+ return reinterpret_cast<const uint64_t *>(lengths.data());
+ }
+
+ void checkGroupByKeys(std::vector<const void *> &dataPtrSrc) const override;
+
+ uint64_t checkGroupByKey(uint64_t index, const char *dataPtr) const override;
+
+ uint64_t getSerializeSizeInternal() const override;
+
+ void serializeInternal(NodeSerializer *serializer) override;
+
+ void deserializeInternal(NodeDeserializer *deserializer) override;
+
+ void selectDistinct(SelectList *selected, Scalar *scalar) override;
+
+ void selectDistinct(SelectList *selected, Vector *vect) override;
+
+ const char *readRawValue(uint64_t index, uint64_t *len, bool *null) const;
+
+ void computeRawValueAndValPtrs();
+
+ protected:
+ void resizeInternal(size_t plainSize) override;
+
+ static uint64_t computeValPtrs(const char *vals, const uint64_t *lens,
+ const bool *nulls, uint64_t size,
+ const char **valPtrs) {
+ const char *ptr = vals;
+
+ if (nulls) {
+ for (uint64_t i = 0; i < size; i++) {
+ valPtrs[i] = ptr;
+ ptr += nulls[i] ? 0 : lens[i];
+ }
+ } else {
+ for (uint64_t i = 0; i < size; i++) {
+ valPtrs[i] = ptr;
+ ptr += lens[i];
+ }
+ }
+
+ return ptr - vals;
+ }
+ dbcommon::ByteBuffer auxiliaryValues;
+ dbcommon::ByteBuffer scaleValues;
+ dbcommon::ByteBuffer rawValues;
+ dbcommon::ByteBuffer lengths;
+ dbcommon::ByteBuffer valPtrs;
+ mutable DecimalVar readBuffer[2];
+ mutable uint8_t nextBuffer = 0;
+ struct DecimalDataInfo {
+ char isNull;
+ char padding[sizeof(DecimalVar) - 1];
+ int64_t highbitsData;
+ uint64_t lowbitsData;
+ int64_t scaleData;
+ };
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_DECIMAL_VECTOR_H_
diff --git a/depends/dbcommon/src/dbcommon/common/vector/fixed-length-vector.h b/depends/dbcommon/src/dbcommon/common/vector/fixed-length-vector.h
new file mode 100644
index 0000000..966c07d
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector/fixed-length-vector.h
@@ -0,0 +1,702 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_FIXED_LENGTH_VECTOR_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_FIXED_LENGTH_VECTOR_H_
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "dbcommon/common/vector-transformer.h"
+#include "dbcommon/common/vector.h"
+#include "dbcommon/type/magma-tid.h"
+
+namespace dbcommon {
+
+template <typename ValueType, typename ADT>
+class FixedSizeTypeVector : public Vector {
+ public:
+ typedef ValueType value_type;
+ typedef ADT abstract_type;
+
+ explicit FixedSizeTypeVector(bool ownData) : Vector(ownData) {
+ if (ownData)
+ values.reserve(sizeof(ValueType) * DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ }
+
+ ~FixedSizeTypeVector() override {}
+
+ uint64_t getNumOfRowsPlain() const override {
+ return values.size() / sizeof(value_type);
+ }
+
+ void setValue(const char *value, uint64_t size) override {
+ assert(!getOwnData() && value != nullptr && size >= 0);
+ values.setData(value, size);
+ }
+
+ void materialize() override {
+ values.materialize<value_type>(selected);
+ if (hasNulls) nulls.materialize(selected);
+ selected = nullptr;
+ }
+
+ std::unique_ptr<Vector> cloneSelected(const SelectList *sel) const override {
+ assert(isValid());
+
+ std::unique_ptr<Vector> retval = Vector::BuildVector(getTypeKind(), false);
+ FixedSizeTypeVector<value_type, abstract_type> *ret =
+ static_cast<FixedSizeTypeVector<value_type, abstract_type> *>(
+ retval.get());
+
+ ret->setTypeKind(typeKind);
+ ret->setHasNull(hasNulls);
+
+ if (hasNulls) ret->setNulls(getNulls(), getNumOfRowsPlain());
+ ret->setValue(values.data(), values.size());
+
+ ret->setSelected(sel, true);
+ ret->materialize();
+
+ assert(ret->isValid());
+ return std::move(retval);
+ }
+
+ std::unique_ptr<Vector> extract(const SelectList *sel, uint64_t start,
+ uint64_t end) const override {
+ assert(isValid() && start < end);
+ uint64_t total = getNumOfRowsPlain();
+ assert(end <= total);
+ uint64_t extSz = end - start;
+
+ std::unique_ptr<Vector> retval = Vector::BuildVector(getTypeKind(), false);
+ FixedSizeTypeVector<value_type, abstract_type> *ret =
+ static_cast<FixedSizeTypeVector<value_type, abstract_type> *>(
+ retval.get());
+
+ ret->setTypeKind(typeKind);
+ ret->setHasNull(hasNulls);
+
+ if (hasNulls) ret->setNulls(getNulls() + start, extSz);
+
+ uint64_t typLen = sizeof(value_type);
+ if (values.data()) {
+ ret->setValue(values.data() + start * typLen, extSz * typLen);
+ }
+
+ ret->setSelected(sel, true);
+
+ assert(ret->isValid());
+ return std::move(retval);
+ }
+
+ std::unique_ptr<Vector> replicateItemToVec(uint64_t index,
+ int replicateNum) const override {
+ assert(isValid());
+ auto retval = Vector::BuildVector(getTypeKind(), true);
+ auto rt = static_cast<FixedSizeTypeVector<value_type, abstract_type> *>(
+ retval.get());
+
+ rt->setTypeKind(typeKind);
+ rt->setHasNull(hasNulls);
+
+ dbcommon::ByteBuffer buf(true);
+ uint64_t idx = getPlainIndex(index);
+ auto value = values.get<value_type>(idx);
+ auto null = isNullPlain(idx);
+
+ if (hasNulls) {
+ for (int i = 0; i < replicateNum; ++i) {
+ buf.append(value);
+ rt->nulls.append(null);
+ }
+ } else {
+ for (int i = 0; i < replicateNum; ++i) {
+ buf.append(values.get<value_type>(idx));
+ }
+ }
+ rt->values.swap(buf);
+ assert(rt->isValid());
+
+ return std::move(retval);
+ }
+
+ void merge(size_t plainSize, const std::vector<Datum> &data,
+ const std::vector<SelectList *> &sels) override {
+ assert(data.size() == sels.size());
+
+ values.resize(plainSize * sizeof(value_type));
+ nulls.resize(plainSize);
+ value_type *__restrict__ vals =
+ reinterpret_cast<value_type *>(values.data());
+ bool *__restrict__ nulls = this->nulls.data();
+
+ for (auto inputIdx = 0; inputIdx < data.size(); inputIdx++) {
+ auto sel = sels[inputIdx];
+ if (sel == nullptr) continue;
+ auto datum = DatumGetValue<Object *>(data[inputIdx]);
+
+ if (auto vec = dynamic_cast<Vector *>(datum)) {
+ auto valsIn = reinterpret_cast<const value_type *>(vec->getValue());
+ for (auto idx : *sel) vals[idx] = valsIn[idx];
+ if (vec->hasNullValue()) {
+ auto nullsIn = vec->getNulls();
+ for (auto idx : *sel) nulls[idx] = nullsIn[idx];
+ } else {
+ for (auto idx : *sel) nulls[idx] = false;
+ }
+ } else if (auto boolSel = dynamic_cast<SelectList *>(datum)) {
+ assert(typeKind == TypeKind::BOOLEANID);
+ for (auto idx : *sel) {
+ vals[idx] = false;
+ nulls[idx] = false;
+ }
+ for (auto idx : *boolSel) {
+ vals[idx] = true;
+ }
+ } else {
+ auto scalar = reinterpret_cast<Scalar *>(datum);
+ auto val = DatumGetValue<value_type>(scalar->value);
+ auto null = scalar->isnull;
+ if (null) {
+ for (auto idx : *sel) {
+ nulls[idx] = true;
+ }
+ } else {
+ for (auto idx : *sel) {
+ vals[idx] = val;
+ nulls[idx] = false;
+ }
+ }
+ }
+ } // End of loop each input data
+ }
+
+ const char *read(uint64_t index, uint64_t *len, bool *null) const override {
+ assert(index < getNumOfRows() && "invalid input");
+ assert(nullptr != null && "invalid input");
+ assert(nullptr != len && "invalid input");
+
+ uint64_t i = getPlainIndex(index);
+
+ *null = isNullPlain(i);
+ *len = sizeof(value_type);
+
+ return values.data() + (i * (*len));
+ }
+
+ std::string readPlain(uint64_t index, bool *null) const override {
+ *null = isNullPlain(index);
+ if (*null) return "NULL";
+ return std::move(abstract_type::toString(values.get<value_type>(index)));
+ }
+
+ void readPlainScalar(uint64_t index, Scalar *scalar) const override {
+ scalar->value = CreateDatum(values.get<value_type>(index));
+ scalar->isnull = isNullPlain(index);
+ }
+
+ void append(const char *v, uint64_t valueLen, bool null) override {
+ assert(getOwnData() && v != nullptr);
+ assert(valueLen == sizeof(value_type));
+
+ value_type value = *(reinterpret_cast<const value_type *>(v));
+ values.append<value_type>(value);
+
+ appendNull(null);
+ }
+
+ void append(Vector *src, uint64_t index) override {
+ bool isnull = false;
+ uint64_t len = 0;
+ const char *ret = src->read(index, &len, &isnull);
+ append(ret, len, isnull);
+ }
+
+ void append(const std::string &strValue, bool null) override {
+ assert(getOwnData());
+
+ value_type value(0);
+
+ if (!null) {
+ value = abstract_type::fromString(strValue);
+ }
+
+ append(reinterpret_cast<const char *>(&value), sizeof(value_type), null);
+ }
+
+ void append(const Datum &datum, bool null) override {
+ /*
+ std::string buffer;
+ if (!null) {
+ buffer = DatumToBinary(datum, typeKind);
+ } else {
+ buffer.resize(abstract_type::kWidth);
+ }
+
+ append(buffer.data(), buffer.size(), null);
+ */
+
+ values.append(*reinterpret_cast<const value_type *>(&datum));
+ appendNull(null);
+ }
+
+ void append(const Scalar *scalar) override {
+ values.append(DatumGetValue<const value_type>(scalar->value));
+ nulls.append(scalar->isnull);
+ }
+
+ void append(Vector *vect, SelectList *sel) override {
+ Vector::append(vect, sel);
+
+ const dbcommon::ByteBuffer *newValues = vect->getValueBuffer();
+
+ if (!sel || sel->size() == vect->getNumOfRowsPlain()) {
+ values.append(newValues->data(), newValues->size());
+ } else {
+ uint64_t newSize = sel->size();
+ for (uint64_t i = 0; i < newSize; i++) {
+ values.append(newValues->get<value_type>((*sel)[i]));
+ }
+ }
+ }
+
+ void reset(bool resetBelongingTupleBatch = false) override {
+ Vector::reset(resetBelongingTupleBatch);
+ values.resize(0);
+ }
+
+ void append(Vector *vect) override {
+ assert(getOwnData());
+
+ this->append(vect, vect->getSelected());
+ }
+
+ bool isValid() const override {
+ Vector::isValid();
+
+ if (hasNulls) assert(nulls.size() == values.size() / sizeof(value_type));
+ return true;
+ }
+
+ void hash(std::vector<uint64_t> &retval) const override {
+ auto base = reinterpret_cast<const value_type *>(getValue());
+
+ if (selected) {
+ fast_hash(base, *selected, retval);
+ } else {
+ fast_hash(base, this->getNumOfRows(), retval);
+ }
+
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ if (nulls.get(getPlainIndex(i))) {
+ retval[i] = 0;
+ }
+ }
+ }
+ }
+
+ void cdbHash(std::vector<uint64_t> &retval) const override {
+ if (std::is_same<value_type, int32_t>::value ||
+ std::is_same<value_type, int16_t>::value ||
+ std::is_same<value_type, int8_t>::value) {
+ auto base = reinterpret_cast<const value_type *>(getValue());
+ size_t len = sizeof(int64_t);
+
+ if (selected) {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ int64_t value = static_cast<int64_t>(base[idx]);
+ retval[i] = nulls.get(idx) ? cdbhashnull(retval[i])
+ : cdbhash(&value, len, retval[i]);
+ }
+ } else {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ int64_t value = static_cast<int64_t>(base[idx]);
+ retval[i] = cdbhash(&value, len, retval[i]);
+ }
+ }
+ } else {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ int64_t value = static_cast<int64_t>(base[i]);
+ retval[i] = nulls.get(i) ? cdbhashnull(retval[i])
+ : cdbhash(&value, len, retval[i]);
+ }
+ } else {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ int64_t value = static_cast<int64_t>(base[i]);
+ retval[i] = cdbhash(&value, len, retval[i]);
+ }
+ }
+ }
+ return;
+ }
+
+ auto base = reinterpret_cast<const value_type *>(getValue());
+ size_t len = sizeof(value_type);
+
+ if (selected) {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = nulls.get(idx) ? cdbhashnull(retval[i])
+ : cdbhash(&base[idx], len, retval[i]);
+ }
+ } else {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = cdbhash(&base[idx], len, retval[i]);
+ }
+ }
+ } else {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = nulls.get(i) ? cdbhashnull(retval[i])
+ : cdbhash(&base[i], len, retval[i]);
+ }
+ } else {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = cdbhash(&base[i], len, retval[i]);
+ }
+ }
+ }
+ }
+
+ dbcommon::ByteBuffer *getValueBuffer() override { return &values; }
+
+ void swap(Vector &vect) override {
+ Vector::swap(vect);
+ auto &v =
+ dynamic_cast<FixedSizeTypeVector<value_type, abstract_type> &>(vect);
+ this->values.swap(v.values);
+ }
+
+ bool checkZeroValue() const override {
+ // zero value is not a common case, so we go through all the values to
+ // reduce CPU branch predict miss
+ bool hasZero = false;
+ FixedSizeTypeVectorRawData<value_type> vec(
+ const_cast<Vector *>(reinterpret_cast<const Vector *>(this)));
+ auto checkZero = [&](SelectList::value_type plainIdx) {
+ hasZero |= (vec.values[plainIdx] == 0);
+ };
+ transformVector(vec.plainSize, vec.sel, vec.nulls, checkZero);
+ return hasZero;
+ }
+
+ bool checkLessZero() const override {
+ bool lessZero = false;
+ FixedSizeTypeVectorRawData<value_type> vec(
+ const_cast<Vector *>(reinterpret_cast<const Vector *>(this)));
+ auto checkZero = [&](SelectList::value_type plainIdx) {
+ lessZero |= (vec.values[plainIdx] < 0);
+ };
+ transformVector(vec.plainSize, vec.sel, vec.nulls, checkZero);
+ return lessZero;
+ }
+
+ bool checkLogarithmAgrumentError() const override {
+ bool lessZero = false;
+ bool isZero = false;
+ FixedSizeTypeVectorRawData<value_type> vec(
+ const_cast<Vector *>(reinterpret_cast<const Vector *>(this)));
+ auto checkIfValid = [&](SelectList::value_type plainIdx) {
+ lessZero |= (vec.values[plainIdx] < 0);
+ isZero |= (vec.values[plainIdx] == 0);
+ };
+ transformVector(vec.plainSize, vec.sel, vec.nulls, checkIfValid);
+ if (lessZero)
+ LOG_ERROR(ERRCODE_INVALID_ARGUMENT_FOR_LOG,
+ "cannot take logarithm of a negative number");
+ if (isZero)
+ LOG_ERROR(ERRCODE_INVALID_ARGUMENT_FOR_LOG,
+ "cannot take logarithm of zero");
+ return false;
+ }
+
+ bool checkExponentialArgumentError() const override {
+ bool isInvalid = false;
+ FixedSizeTypeVectorRawData<value_type> vec(
+ const_cast<Vector *>(reinterpret_cast<const Vector *>(this)));
+ auto checkIfValid = [&](SelectList::value_type plainIdx) {
+ isInvalid |=
+ (vec.values[plainIdx] < -708.4 || vec.values[plainIdx] > 709.8);
+ };
+ transformVector(vec.plainSize, vec.sel, vec.nulls, checkIfValid);
+ if (isInvalid)
+ LOG_ERROR(ERRCODE_INTERVAL_FIELD_OVERFLOW,
+ "value out of range: overflow");
+ return false;
+ }
+
+ uint64_t checkGroupByKey(uint64_t index, const char *dataPtr) const override {
+ assert(index < getNumOfRows() && "invalid input");
+ assert(getNumOfRows() > 0);
+ uint64_t i = getPlainIndex(index);
+ const FixedLenDataInfo *info =
+ reinterpret_cast<const FixedLenDataInfo *>(dataPtr);
+ if (isNullPlain(i) != info->isNull) return 0;
+ if (!info->isNull) {
+ if (*reinterpret_cast<ValueType *>(values.data() +
+ i * (sizeof(ValueType))) != info->data)
+ return 0;
+ }
+ return sizeof(ValueType) + sizeof(ValueType);
+ }
+
+ void checkGroupByKeys(std::vector<const void *> &dataPtrSrc) const override {
+ const value_type *data =
+ reinterpret_cast<const value_type *>(FixedSizeTypeVector::getValue());
+
+ const uint64_t size = FixedSizeTypeVector::getNumOfRows();
+ const char **dataPtr = reinterpret_cast<const char **>(&dataPtrSrc[0]);
+ if (selected) {
+ auto &sel = *selected;
+
+#pragma clang loop unroll_count(4)
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ uint64_t index = sel[i];
+ char null = hasNulls ? nulls.getChar(index) : 0;
+ const value_type d = data[index];
+
+ const FixedLenDataInfo *info =
+ reinterpret_cast<const FixedLenDataInfo *>(dataPtr[i]);
+ if (null == info->isNull && (null || d == info->data))
+ dataPtr[i] = alignedAddress(dataPtr[i] + sizeof(FixedLenDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ } else {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ char null = nulls.getChar(i);
+ const value_type d = data[i];
+
+ const FixedLenDataInfo *info =
+ reinterpret_cast<const FixedLenDataInfo *>(dataPtr[i]);
+ if (null == info->isNull && (null || d == info->data))
+ dataPtr[i] =
+ alignedAddress(dataPtr[i] + sizeof(FixedLenDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ } else {
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ char null = false;
+ const value_type d = data[i];
+
+ const FixedLenDataInfo *info =
+ reinterpret_cast<const FixedLenDataInfo *>(dataPtr[i]);
+ if (null == info->isNull && (null || d == info->data))
+ dataPtr[i] =
+ alignedAddress(dataPtr[i] + sizeof(FixedLenDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ }
+ }
+ }
+
+ uint64_t getSerializeSizeInternal() const override {
+ size_t sSize = Vector::getSerializeSizeInternal();
+ sSize += sizeof(uint64_t) + values.size();
+ return sSize;
+ }
+
+ void serializeInternal(NodeSerializer *serializer) override {
+ Vector::serializeInternal(serializer);
+
+ uint64_t vsz = getNumOfRows() * sizeof(value_type);
+ serializer->write<uint64_t>(vsz);
+ if (vsz > 0) {
+ if (allSelected()) {
+ serializer->writeBytes(getValue(), vsz);
+ } else {
+ auto values = reinterpret_cast<const value_type *>(getValue());
+ for (auto idx : *selected) serializer->write(values[idx]);
+ }
+ }
+ }
+
+ void deserializeInternal(NodeDeserializer *deserializer) override {
+ Vector::deserializeInternal(deserializer);
+
+ uint64_t vsz = deserializer->read<uint64_t>();
+ if (vsz > 0) {
+ const char *dstr = deserializer->readBytes(vsz);
+ setValue(dstr, vsz);
+ }
+
+ assert(isValid());
+ }
+
+ void selectDistinct(SelectList *selected, Scalar *scalar) override {
+ SelectList *sel = getSelected();
+ uint64_t sz = getNumOfRows();
+ const value_type *vals = reinterpret_cast<const value_type *>(getValue());
+
+ auto operation = [&](uint64_t plainIdx) { // NOLINT
+ if (!scalar->isnull) {
+ value_type value = DatumGetValue<value_type>(scalar->value);
+ return (vals[plainIdx] != value);
+ }
+ return false;
+ };
+ const bool *nulls = getNulls();
+ dbcommon::transformDistinct(sz, sel, nulls, selected, scalar->isnull,
+ operation);
+ }
+
+ void selectDistinct(SelectList *selected, Vector *vect) override {
+ SelectList *sel1 = getSelected();
+ SelectList *sel2 = vect->getSelected();
+ uint64_t sz = getNumOfRows();
+ assert(sz == vect->getNumOfRows() && "invalid input");
+
+ const value_type *vals1 = reinterpret_cast<const value_type *>(getValue());
+ const value_type *vals2 =
+ reinterpret_cast<const value_type *>(vect->getValue());
+
+ auto operation = [&](uint64_t plainIdx) { // NOLINT
+ return (vals1[plainIdx] != vals2[plainIdx]);
+ };
+ const bool *nulls1 = getNulls();
+ const bool *nulls2 = vect->getNulls();
+ dbcommon::transformDistinct(sz, sel1, nulls1, nulls2, selected, operation);
+ }
+
+ protected:
+ void resizeInternal(size_t plainSize) override {
+ values.resize(plainSize * sizeof(value_type));
+ }
+
+ struct FixedLenDataInfo {
+ char isNull;
+ char padding[sizeof(value_type) - 1];
+ value_type data;
+ };
+};
+
+class TinyIntVector : public FixedSizeTypeVector<int8_t, TinyIntType> {
+ public:
+ explicit TinyIntVector(bool ownData)
+ : FixedSizeTypeVector<int8_t, TinyIntType>(ownData) {
+ setTypeKind(TINYINTID);
+ }
+ virtual ~TinyIntVector() {}
+};
+
+class SmallIntVector : public FixedSizeTypeVector<int16_t, SmallIntType> {
+ public:
+ explicit SmallIntVector(bool ownData)
+ : FixedSizeTypeVector<int16_t, SmallIntType>(ownData) {
+ setTypeKind(SMALLINTID);
+ }
+ virtual ~SmallIntVector() {}
+};
+
+class IntVector : public FixedSizeTypeVector<int32_t, IntType> {
+ public:
+ explicit IntVector(bool ownData)
+ : FixedSizeTypeVector<int32_t, IntType>(ownData) {
+ setTypeKind(INTID);
+ }
+ virtual ~IntVector() {}
+};
+
+class BigIntVector : public FixedSizeTypeVector<int64_t, BigIntType> {
+ public:
+ explicit BigIntVector(bool ownData)
+ : FixedSizeTypeVector<int64_t, BigIntType>(ownData) {
+ setTypeKind(BIGINTID);
+ }
+ virtual ~BigIntVector() {}
+};
+
+class FloatVector : public FixedSizeTypeVector<float, FloatType> {
+ public:
+ explicit FloatVector(bool ownData)
+ : FixedSizeTypeVector<float, FloatType>(ownData) {
+ setTypeKind(FLOATID);
+ }
+ virtual ~FloatVector() {}
+};
+
+class DoubleVector : public FixedSizeTypeVector<double, DoubleType> {
+ public:
+ explicit DoubleVector(bool ownData)
+ : FixedSizeTypeVector<double, DoubleType>(ownData) {
+ setTypeKind(DOUBLEID);
+ }
+ virtual ~DoubleVector() {}
+};
+
+class BooleanVector : public FixedSizeTypeVector<bool, BooleanType> {
+ public:
+ explicit BooleanVector(bool ownData)
+ : FixedSizeTypeVector<bool, BooleanType>(ownData) {
+ setTypeKind(BOOLEANID);
+ }
+ virtual ~BooleanVector() {}
+};
+
+class DateVector : public FixedSizeTypeVector<int32_t, DateType> {
+ public:
+ explicit DateVector(bool ownData)
+ : FixedSizeTypeVector<int32_t, DateType>(ownData) {
+ setTypeKind(DATEID);
+ }
+ virtual ~DateVector() {}
+};
+
+class TimeVector : public FixedSizeTypeVector<int64_t, TimeType> {
+ public:
+ explicit TimeVector(bool ownData)
+ : FixedSizeTypeVector<int64_t, TimeType>(ownData) {
+ setTypeKind(TIMEID);
+ }
+ virtual ~TimeVector() {}
+};
+
+class MagmaTidVector : public FixedSizeTypeVector<MagmaTid, MagmaTidType> {
+ public:
+ explicit MagmaTidVector(bool ownData)
+ : FixedSizeTypeVector<MagmaTid, MagmaTidType>(ownData) {
+ setTypeKind(MAGMATID);
+ }
+
+ ~MagmaTidVector() override {}
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_FIXED_LENGTH_VECTOR_H_
diff --git a/depends/dbcommon/src/dbcommon/common/vector/interval-vector.cc b/depends/dbcommon/src/dbcommon/common/vector/interval-vector.cc
new file mode 100644
index 0000000..fa539d6
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector/interval-vector.cc
@@ -0,0 +1,553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "dbcommon/common/vector/interval-vector.h"
+
+#include "dbcommon/common/vector-transformer.h"
+
+namespace dbcommon {
+
+const char *IntervalVector::read(uint64_t index, uint64_t *len,
+ bool *null) const {
+ assert(index < getNumOfRows() && "invalid input");
+ assert(nullptr != null && "invalid input");
+ assert(nullptr != len && "invalid input");
+
+ uint64_t i = getPlainIndex(index);
+ *null = isNullPlain(i);
+
+ assert(nextBuffer == 0 || nextBuffer == 1);
+ nextBuffer = 1 - nextBuffer;
+ readBuffer[nextBuffer].timeOffset =
+ *reinterpret_cast<int64_t *>(values.data() + (i * sizeof(int64_t)));
+ readBuffer[nextBuffer].day =
+ *reinterpret_cast<int32_t *>(dayValues.data() + (i * sizeof(int32_t)));
+ readBuffer[nextBuffer].month =
+ *reinterpret_cast<int32_t *>(monthValues.data() + (i * sizeof(int32_t)));
+ *len = sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t);
+ return reinterpret_cast<const char *>(&readBuffer[nextBuffer]);
+}
+
+std::string IntervalVector::readPlain(uint64_t index, bool *null) const {
+ *null = isNullPlain(index);
+ if (*null) return "";
+ return std::move(IntervalType::toString(values.get<int64_t>(index),
+ dayValues.get<int32_t>(index),
+ monthValues.get<int32_t>(index)));
+}
+
+void IntervalVector::readPlainScalar(uint64_t index, Scalar *scalar) const {
+ assert(nextBuffer == 0 || nextBuffer == 1);
+ nextBuffer = 1 - nextBuffer;
+ readBuffer[nextBuffer].timeOffset =
+ *reinterpret_cast<int64_t *>(values.data() + (index * sizeof(int64_t)));
+ readBuffer[nextBuffer].day = *reinterpret_cast<int32_t *>(
+ dayValues.data() + (index * sizeof(int32_t)));
+ readBuffer[nextBuffer].month = *reinterpret_cast<int32_t *>(
+ monthValues.data() + (index * sizeof(int32_t)));
+ scalar->value =
+ CreateDatum(reinterpret_cast<const char *>(&readBuffer[nextBuffer]));
+ scalar->length = sizeof(IntervalVar);
+ scalar->isnull = isNullPlain(index);
+}
+
+void IntervalVector::append(const std::string &strValue, bool null) {
+ assert(getOwnData());
+ IntervalVar value = {0, 0, 0};
+ if (!null) {
+ value = IntervalType::fromString(strValue);
+ }
+ append(reinterpret_cast<char *>(&value), sizeof(IntervalVar), null);
+}
+
+void IntervalVector::append(const char *v, uint64_t valueLen, bool null) {
+ assert(getOwnData() && v != nullptr);
+ assert(valueLen == sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t));
+
+ int64_t timeOffset = *(reinterpret_cast<const int64_t *>(v));
+ int32_t dayInterval = *(reinterpret_cast<const int32_t *>(v + 8));
+ int32_t monthInterval = *(reinterpret_cast<const int32_t *>(v + 12));
+ values.append<int64_t>(timeOffset);
+ dayValues.append<int32_t>(dayInterval);
+ monthValues.append<int32_t>(monthInterval);
+ appendNull(null);
+}
+
+void IntervalVector::append(const Datum &datum, bool null) {
+ dbcommon::IntervalVar *val = DatumGetValue<IntervalVar *>(datum);
+ if (val) {
+ values.append(val->timeOffset);
+ dayValues.append(val->day);
+ monthValues.append(val->month);
+ } else {
+ assert(null == true);
+ values.append(int64_t(0));
+ dayValues.append(int32_t(0));
+ monthValues.append(int32_t(0));
+ }
+ appendNull(null);
+}
+
+void IntervalVector::append(const Scalar *scalar) {
+ dbcommon::IntervalVar *val = DatumGetValue<IntervalVar *>(scalar->value);
+ if (val) {
+ values.append(val->timeOffset);
+ dayValues.append(val->day);
+ monthValues.append(val->month);
+ }
+ appendNull(scalar->isnull);
+}
+
+void IntervalVector::append(Vector *src, uint64_t index) {
+ bool isnull = false;
+ uint64_t len = 0;
+ const char *ret = src->read(index, &len, &isnull);
+ append(ret, len, isnull);
+}
+
+void IntervalVector::append(Vector *vect) {
+ assert(getOwnData());
+ this->append(vect, vect->getSelected());
+}
+
+void IntervalVector::append(Vector *vect, SelectList *sel) {
+ Vector::append(vect, sel);
+ const dbcommon::ByteBuffer *timeOffsetBuffer = vect->getValueBuffer();
+ const dbcommon::ByteBuffer *dayBuffer = vect->getDayBuffer();
+ const dbcommon::ByteBuffer *monthBuffer = vect->getMonthBuffer();
+ if (!sel || sel->size() == vect->getNumOfRowsPlain()) {
+ values.append(timeOffsetBuffer->data(), timeOffsetBuffer->size());
+ dayValues.append(dayBuffer->data(), dayBuffer->size());
+ monthValues.append(monthBuffer->data(), monthBuffer->size());
+ } else {
+ uint64_t newSize = sel->size();
+ for (uint64_t i = 0; i < newSize; i++) {
+ values.append(timeOffsetBuffer->get<int64_t>((*sel)[i]));
+ dayValues.append(dayBuffer->get<int32_t>((*sel)[i]));
+ monthValues.append(monthBuffer->get<int32_t>((*sel)[i]));
+ }
+ }
+}
+
+void IntervalVector::materialize() {
+ values.materialize<int64_t>(selected);
+ dayValues.materialize<int32_t>(selected);
+ monthValues.materialize<int32_t>(selected);
+ if (hasNulls) nulls.materialize(selected);
+ selected = nullptr;
+}
+
+std::unique_ptr<Vector> IntervalVector::cloneSelected(
+ const SelectList *sel) const {
+ assert(isValid());
+
+ std::unique_ptr<Vector> retval = Vector::BuildVector(getTypeKind(), false);
+ IntervalVector *ret = static_cast<IntervalVector *>(retval.get());
+
+ ret->setTypeKind(typeKind);
+ ret->setHasNull(hasNulls);
+ if (hasNulls) ret->setNulls(getNulls(), getNumOfRowsPlain());
+ ret->setValue(values.data(), values.size());
+ ret->setDayValue(dayValues.data(), dayValues.size());
+ ret->setMonthValue(monthValues.data(), monthValues.size());
+
+ ret->setSelected(sel, true);
+ ret->materialize();
+
+ assert(ret->isValid());
+ return std::move(retval);
+}
+
+std::unique_ptr<Vector> IntervalVector::extract(const SelectList *sel,
+ uint64_t start,
+ uint64_t end) const {
+ assert(isValid() && start < end);
+ uint64_t total = getNumOfRowsPlain();
+ assert(end <= total);
+ uint64_t extSz = end - start;
+
+ std::unique_ptr<Vector> retval = Vector::BuildVector(getTypeKind(), false);
+ IntervalVector *ret = static_cast<IntervalVector *>(retval.get());
+
+ ret->setTypeKind(typeKind);
+ ret->setHasNull(hasNulls);
+ if (hasNulls) ret->setNulls(getNulls() + start, extSz);
+
+ uint64_t typLen = sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t);
+ if (values.data()) {
+ ret->setValue(values.data() + start * typLen, extSz * typLen);
+ }
+ if (dayValues.data()) {
+ ret->setDayValue(dayValues.data() + start * typLen, extSz * typLen);
+ }
+ if (monthValues.data()) {
+ ret->setMonthValue(monthValues.data() + start * typLen, extSz * typLen);
+ }
+ ret->setSelected(sel, true);
+
+ assert(ret->isValid());
+ return std::move(retval);
+}
+
+std::unique_ptr<Vector> IntervalVector::replicateItemToVec(
+ uint64_t index, int replicateNum) const {
+ assert(isValid());
+ auto retval = Vector::BuildVector(getTypeKind(), true);
+ auto ret = static_cast<IntervalVector *>(retval.get());
+
+ ret->setTypeKind(typeKind);
+ ret->setHasNull(hasNulls);
+
+ dbcommon::ByteBuffer timebuf(true);
+ dbcommon::ByteBuffer daybuf(true);
+ dbcommon::ByteBuffer monthbuf(true);
+ uint64_t idx = getPlainIndex(index);
+ auto tmValue = values.get<int64_t>(idx);
+ auto dayValue = dayValues.get<int32_t>(idx);
+ auto monthValue = monthValues.get<int32_t>(idx);
+ auto null = isNullPlain(idx);
+
+ if (hasNulls) {
+ for (int i = 0; i < replicateNum; ++i) {
+ timebuf.append(tmValue);
+ daybuf.append(dayValue);
+ monthbuf.append(monthValue);
+ ret->nulls.append(null);
+ }
+ } else {
+ for (int i = 0; i < replicateNum; ++i) {
+ timebuf.append(values.get<int64_t>(idx));
+ daybuf.append(dayValues.get<int32_t>(idx));
+ monthbuf.append(monthValues.get<int32_t>(idx));
+ }
+ }
+ ret->values.swap(timebuf);
+ ret->dayValues.swap(daybuf);
+ ret->monthValues.swap(monthbuf);
+ assert(ret->isValid());
+ return std::move(retval);
+}
+
+void IntervalVector::merge(size_t plainSize, const std::vector<Datum> &data,
+ const std::vector<SelectList *> &sels) {
+ assert(data.size() == sels.size());
+
+ values.resize(plainSize * sizeof(int64_t));
+ dayValues.resize(plainSize * sizeof(int32_t));
+ monthValues.resize(plainSize * sizeof(int32_t));
+ nulls.resize(plainSize);
+ int64_t *__restrict__ tvals = reinterpret_cast<int64_t *>(values.data());
+ int32_t *__restrict__ dvals = reinterpret_cast<int32_t *>(dayValues.data());
+ int32_t *__restrict__ mvals = reinterpret_cast<int32_t *>(monthValues.data());
+ bool *__restrict__ nulls = this->nulls.data();
+
+ for (auto inputIdx = 0; inputIdx < data.size(); inputIdx++) {
+ auto sel = sels[inputIdx];
+ if (sel == nullptr) continue;
+ auto datum = DatumGetValue<Object *>(data[inputIdx]);
+
+ if (auto vec = dynamic_cast<Vector *>(datum)) {
+ assert(dynamic_cast<IntervalVector *>(vec));
+ IntervalVector *ivec = dynamic_cast<IntervalVector *>(vec);
+ auto tvalsIn = reinterpret_cast<const int64_t *>(ivec->getValue());
+ auto dvalsIn = reinterpret_cast<const int32_t *>(ivec->getDayValue());
+ auto mvalsIn = reinterpret_cast<const int32_t *>(ivec->getMonthValue());
+ for (auto idx : *sel) {
+ tvals[idx] = tvalsIn[idx];
+ dvals[idx] = dvalsIn[idx];
+ mvals[idx] = mvalsIn[idx];
+ }
+ if (ivec->hasNullValue()) {
+ auto nullsIn = ivec->getNulls();
+ for (auto idx : *sel) nulls[idx] = nullsIn[idx];
+ } else {
+ for (auto idx : *sel) nulls[idx] = false;
+ }
+ } else {
+ auto scalar = reinterpret_cast<Scalar *>(datum);
+ IntervalVar *val = DatumGetValue<IntervalVar *>(scalar->value);
+ auto null = scalar->isnull;
+ if (null) {
+ for (auto idx : *sel) {
+ nulls[idx] = true;
+ }
+ } else {
+ for (auto idx : *sel) {
+ tvals[idx] = val->timeOffset;
+ dvals[idx] = val->day;
+ mvals[idx] = val->month;
+ nulls[idx] = false;
+ }
+ }
+ }
+ }
+}
+
+void IntervalVector::hash(std::vector<uint64_t> &retval) const {
+ auto base = reinterpret_cast<const int64_t *>(getValue());
+
+ if (selected) {
+ fast_hash(base, *selected, retval);
+ } else {
+ fast_hash(base, this->getNumOfRows(), retval);
+ }
+
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ if (nulls.get(getPlainIndex(i))) {
+ retval[i] = 0;
+ }
+ }
+ }
+}
+
+void IntervalVector::cdbHash(std::vector<uint64_t> &retval) const {
+ auto base = reinterpret_cast<const int64_t *>(getValue());
+ size_t len = sizeof(uint64_t);
+
+ if (selected) {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = nulls.get(idx) ? cdbhashnull(retval[i])
+ : cdbhash(&base[idx], len, retval[i]);
+ }
+ } else {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = cdbhash(&base[idx], len, retval[i]);
+ }
+ }
+ } else {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = nulls.get(i) ? cdbhashnull(retval[i])
+ : cdbhash(&base[i], len, retval[i]);
+ }
+ } else {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = cdbhash(&base[i], len, retval[i]);
+ }
+ }
+ }
+}
+
+void IntervalVector::swap(Vector &vect) {
+ Vector::swap(vect);
+ auto &v = dynamic_cast<IntervalVector &>(vect);
+ values.swap(v.values);
+ dayValues.swap(v.dayValues);
+ monthValues.swap(v.monthValues);
+}
+
+void IntervalVector::checkGroupByKeys(
+ std::vector<const void *> &dataPtrSrc) const {
+ const int64_t *tmdata = reinterpret_cast<const int64_t *>(values.data());
+ const int32_t *daydata = reinterpret_cast<const int32_t *>(dayValues.data());
+ const int32_t *mondata =
+ reinterpret_cast<const int32_t *>(monthValues.data());
+ const uint64_t size = getNumOfRows();
+ const char **dataPtr = reinterpret_cast<const char **>(&dataPtrSrc[0]);
+ if (selected) {
+ auto &sel = *selected;
+
+#pragma clang loop unroll_count(4)
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ uint64_t index = sel[i];
+ char null = hasNulls ? nulls.getChar(index) : 0;
+ const int64_t t = tmdata[index];
+ const int32_t d = daydata[index];
+ const int32_t m = mondata[index];
+
+ const IntervalDataInfo *info =
+ reinterpret_cast<const IntervalDataInfo *>(dataPtr[i]);
+ if (null == info->isNull &&
+ (null || (t == info->timeOffsetData && d == info->dayData &&
+ m == info->monthData)))
+ dataPtr[i] = alignedAddress(dataPtr[i] + sizeof(IntervalDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ } else {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ char null = nulls.getChar(i);
+ const int64_t t = tmdata[i];
+ const int32_t d = daydata[i];
+ const int32_t m = mondata[i];
+
+ const IntervalDataInfo *info =
+ reinterpret_cast<const IntervalDataInfo *>(dataPtr[i]);
+ if (null == info->isNull &&
+ (null || (t == info->timeOffsetData && d == info->dayData &&
+ m == info->monthData)))
+ dataPtr[i] = alignedAddress(dataPtr[i] + sizeof(IntervalDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ } else {
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ char null = false;
+ const int64_t t = tmdata[i];
+ const int32_t d = daydata[i];
+ const int32_t m = mondata[i];
+
+ const IntervalDataInfo *info =
+ reinterpret_cast<const IntervalDataInfo *>(dataPtr[i]);
+ if (null == info->isNull &&
+ (null || (t == info->timeOffsetData && d == info->dayData &&
+ m == info->monthData)))
+ dataPtr[i] = alignedAddress(dataPtr[i] + sizeof(IntervalDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ }
+ }
+}
+
+uint64_t IntervalVector::checkGroupByKey(uint64_t index,
+ const char *dataPtr) const {
+ assert(index < getNumOfRows() && "invalid input");
+ assert(getNumOfRows() > 0);
+ uint64_t i = getPlainIndex(index);
+ const IntervalDataInfo *info =
+ reinterpret_cast<const IntervalDataInfo *>(dataPtr);
+ if (isNullPlain(i) != info->isNull) return 0;
+ if (!info->isNull) {
+ if (*reinterpret_cast<int64_t *>(values.data() + i * (sizeof(int64_t))) !=
+ info->timeOffsetData)
+ return 0;
+ if (*reinterpret_cast<int32_t *>(dayValues.data() +
+ i * (sizeof(int32_t))) != info->dayData)
+ return 0;
+ if (*reinterpret_cast<int32_t *>(monthValues.data() +
+ i * (sizeof(int32_t))) != info->monthData)
+ return 0;
+ }
+ return sizeof(IntervalDataInfo);
+}
+
+uint64_t IntervalVector::getSerializeSizeInternal() const {
+ size_t sSize = Vector::getSerializeSizeInternal();
+ sSize += sizeof(uint64_t);
+ sSize += values.size();
+ sSize += dayValues.size();
+ sSize += monthValues.size();
+ return sSize;
+}
+
+void IntervalVector::serializeInternal(NodeSerializer *serializer) {
+ Vector::serializeInternal(serializer);
+
+ uint64_t vsz = getNumOfRows() * sizeof(int64_t);
+ serializer->write<uint64_t>(vsz * 2);
+ if (vsz > 0) {
+ if (allSelected()) {
+ serializer->writeBytes(getValue(), vsz);
+ serializer->writeBytes(getDayValue(), vsz / 2);
+ serializer->writeBytes(getMonthValue(), vsz / 2);
+ } else {
+ auto values = reinterpret_cast<const int64_t *>(getValue());
+ for (auto i : *selected) {
+ serializer->write(values[i]);
+ }
+ auto dayVal = reinterpret_cast<const int32_t *>(getDayValue());
+ for (auto i : *selected) {
+ serializer->write(dayVal[i]);
+ }
+ auto monthVal = reinterpret_cast<const int32_t *>(getMonthValue());
+ for (auto i : *selected) {
+ serializer->write(monthVal[i]);
+ }
+ }
+ }
+}
+
+void IntervalVector::deserializeInternal(NodeDeserializer *deserializer) {
+ Vector::deserializeInternal(deserializer);
+
+ uint64_t vsz = deserializer->read<int64_t>();
+ if (vsz > 0) {
+ const char *dstr = deserializer->readBytes(vsz);
+ setValue(dstr, vsz / 2);
+ setDayValue(dstr + vsz / 2, vsz / 4);
+ setMonthValue(dstr + vsz / 2 + vsz / 4, vsz / 4);
+ }
+
+ assert(isValid());
+}
+
+void IntervalVector::selectDistinct(SelectList *selected, Scalar *scalar) {
+ SelectList *sel = getSelected();
+ uint64_t sz = getNumOfRows();
+ const int64_t *timeOffsets = reinterpret_cast<const int64_t *>(getValue());
+ const int32_t *days = reinterpret_cast<const int32_t *>(getDayValue());
+ const int32_t *months = reinterpret_cast<const int32_t *>(getMonthValue());
+
+ auto operation = [&](uint64_t plainIdx) { // NOLINT
+ if (!scalar->isnull) {
+ IntervalVar *value = DatumGetValue<IntervalVar *>(scalar->value);
+ return (timeOffsets[plainIdx] != value->timeOffset ||
+ days[plainIdx] != value->day || months[plainIdx] != value->month);
+ }
+ return false;
+ };
+ const bool *nulls = getNulls();
+ dbcommon::transformDistinct(sz, sel, nulls, selected, scalar->isnull,
+ operation);
+}
+
+void IntervalVector::selectDistinct(SelectList *selected, Vector *vect) {
+ SelectList *sel1 = getSelected();
+ SelectList *sel2 = vect->getSelected();
+ uint64_t sz = getNumOfRows();
+ assert(sz == vect->getNumOfRows() && "invalid input");
+
+ const int64_t *timeOffsets1 = reinterpret_cast<const int64_t *>(getValue());
+ const int32_t *days1 = reinterpret_cast<const int32_t *>(getDayValue());
+ const int32_t *months1 = reinterpret_cast<const int32_t *>(getMonthValue());
+ const int64_t *timeOffsets2 =
+ reinterpret_cast<const int64_t *>(vect->getValue());
+ const int32_t *days2 = reinterpret_cast<const int32_t *>(vect->getDayValue());
+ const int32_t *months2 =
+ reinterpret_cast<const int32_t *>(vect->getMonthValue());
+
+ auto operation = [&](uint64_t plainIdx) { // NOLINT
+ return (timeOffsets1[plainIdx] != timeOffsets2[plainIdx] ||
+ days1[plainIdx] != days2[plainIdx] ||
+ months1[plainIdx] != months2[plainIdx]);
+ };
+ const bool *nulls1 = getNulls();
+ const bool *nulls2 = vect->getNulls();
+ dbcommon::transformDistinct(sz, sel1, nulls1, nulls2, selected, operation);
+}
+
+void IntervalVector::resizeInternal(size_t plainSize) {
+ values.resize(plainSize * sizeof(int64_t));
+ dayValues.resize(plainSize * sizeof(int32_t));
+ monthValues.resize(plainSize * sizeof(int32_t));
+}
+
+} // namespace dbcommon
diff --git a/depends/dbcommon/src/dbcommon/common/vector/interval-vector.h b/depends/dbcommon/src/dbcommon/common/vector/interval-vector.h
new file mode 100644
index 0000000..5b664c0
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector/interval-vector.h
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_INTERVAL_VECTOR_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_INTERVAL_VECTOR_H_
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "dbcommon/common/vector.h"
+#include "dbcommon/type/interval.h"
+
+namespace dbcommon {
+
+class IntervalVector : public Vector {
+ public:
+ explicit IntervalVector(bool ownData)
+ : Vector(ownData), dayValues(ownData), monthValues(ownData) {
+ if (ownData) {
+ values.reserve(sizeof(uint64_t) * DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ dayValues.reserve(sizeof(int32_t) * DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ monthValues.reserve(sizeof(int32_t) * DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ }
+ setTypeKind(INTERVALID);
+ }
+
+ ~IntervalVector() override {}
+
+ uint64_t getNumOfRowsPlain() const override {
+ return values.size() / sizeof(int64_t);
+ }
+
+ dbcommon::ByteBuffer *getValueBuffer() override { return &values; }
+
+ dbcommon::ByteBuffer *getDayBuffer() override { return &dayValues; }
+
+ dbcommon::ByteBuffer *getMonthBuffer() override { return &monthValues; }
+
+ const char *getDayValue() const override { return dayValues.data(); }
+
+ const char *getMonthValue() const override { return monthValues.data(); }
+
+ void setValue(const char *value, uint64_t size) override {
+ assert(!getOwnData() && value != nullptr && size >= 0);
+ values.setData(value, size);
+ }
+
+ void setDayValue(const char *value, uint64_t size) override {
+ assert(!getOwnData() && value != nullptr && size >= 0);
+ dayValues.setData(value, size);
+ }
+
+ void setMonthValue(const char *value, uint64_t size) override {
+ assert(!getOwnData() && value != nullptr && size >= 0);
+ monthValues.setData(value, size);
+ }
+
+ const char *read(uint64_t index, uint64_t *len, bool *null) const override;
+
+ std::string readPlain(uint64_t index, bool *null) const override;
+
+ void readPlainScalar(uint64_t index, Scalar *scalar) const override;
+
+ void append(const std::string &strValue, bool null) override;
+
+ void append(const char *v, uint64_t valueLen, bool null) override;
+
+ void append(const Datum &datum, bool null) override;
+
+ void append(const Scalar *scalar) override;
+
+ void append(Vector *src, uint64_t index) override;
+
+ void append(Vector *vect) override;
+
+ void append(Vector *vect, SelectList *sel) override;
+
+ void materialize() override;
+
+ void reset(bool resetBelongingTupleBatch = false) override {
+ Vector::reset(resetBelongingTupleBatch);
+ values.resize(0);
+ dayValues.resize(0);
+ monthValues.resize(0);
+ }
+
+ std::unique_ptr<Vector> cloneSelected(const SelectList *sel) const override;
+
+ std::unique_ptr<Vector> extract(const SelectList *sel, uint64_t start,
+ uint64_t end) const override;
+
+ std::unique_ptr<Vector> replicateItemToVec(uint64_t index,
+ int replicateNum) const override;
+
+ void merge(size_t plainSize, const std::vector<Datum> &data,
+ const std::vector<SelectList *> &sels) override;
+
+ bool isValid() const override {
+ Vector::isValid();
+
+ if (hasNulls) {
+ assert(nulls.size() == values.size() / sizeof(int64_t));
+ assert(nulls.size() == dayValues.size() / sizeof(int32_t));
+ assert(nulls.size() == monthValues.size() / sizeof(int32_t));
+ }
+ return true;
+ }
+
+ void hash(std::vector<uint64_t> &retval) const override;
+
+ void cdbHash(std::vector<uint64_t> &retval) const override;
+
+ void swap(Vector &vect) override;
+
+ void checkGroupByKeys(std::vector<const void *> &dataPtrSrc) const override;
+
+ uint64_t checkGroupByKey(uint64_t index, const char *dataPtr) const override;
+
+ uint64_t getSerializeSizeInternal() const override;
+
+ void serializeInternal(NodeSerializer *serializer) override;
+
+ void deserializeInternal(NodeDeserializer *deserializer) override;
+
+ void selectDistinct(SelectList *selected, Scalar *scalar) override;
+
+ void selectDistinct(SelectList *selected, Vector *vect) override;
+
+ protected:
+ void resizeInternal(size_t plainSize) override;
+
+ dbcommon::ByteBuffer dayValues;
+ dbcommon::ByteBuffer monthValues;
+ mutable IntervalVar readBuffer[2];
+ mutable uint8_t nextBuffer = 0;
+ struct IntervalDataInfo {
+ char isNull;
+ char padding[7];
+ int64_t timeOffsetData;
+ int32_t dayData;
+ int32_t monthData;
+ };
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_INTERVAL_VECTOR_H_
diff --git a/depends/dbcommon/src/dbcommon/common/vector/list-vector.h b/depends/dbcommon/src/dbcommon/common/vector/list-vector.h
new file mode 100644
index 0000000..8514ae4
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector/list-vector.h
@@ -0,0 +1,621 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_LIST_VECTOR_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_LIST_VECTOR_H_
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "dbcommon/common/vector.h"
+
+namespace dbcommon {
+
+class ListVector : public Vector {
+ public:
+ explicit ListVector(bool ownData) : Vector(ownData), offsets(ownData) {}
+
+ virtual ~ListVector() {}
+
+ uint64_t getNumOfRowsPlain() const override {
+ return offsets.size() / sizeof(uint64_t) - 1;
+ }
+
+ void setValue(const char *value, uint64_t size) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not have value array");
+ }
+
+ const char *getValue() const {
+ assert(childs.size() == 1);
+ return childs[0]->getValue();
+ }
+
+ const uint64_t *getOffsets() const {
+ return reinterpret_cast<const uint64_t *>(offsets.data());
+ }
+
+ void setOffsets(const uint64_t *offsetsPtr, uint64_t size) {
+ assert(!getOwnData() && offsetsPtr != nullptr && size > 0);
+ this->offsets.setData(reinterpret_cast<const char *>(offsetsPtr),
+ (size) * sizeof(uint64_t));
+ }
+
+ const char *read(uint64_t index, uint64_t *num, uint64_t *width,
+ bool *null) const {
+ assert(childs.size() == 1);
+ uint64_t currentOffset = const_cast<uint64_t *>(getOffsets())[index];
+ uint64_t nextOffset = const_cast<uint64_t *>(getOffsets())[index + 1];
+ const char *ret = childs[0]->read(currentOffset, width, null);
+ *num = nextOffset - currentOffset;
+ *null = isNullPlain(index);
+ return ret;
+ }
+
+ const char *read(uint64_t index, std::string *nullBitmap, int32_t *dims,
+ bool *null) {
+ assert(childs.size() == 1);
+ uint64_t currentOffset = const_cast<uint64_t *>(getOffsets())[index];
+ uint64_t nextOffset = const_cast<uint64_t *>(getOffsets())[index + 1];
+ dims[0] = nextOffset - currentOffset;
+ auto nulls = std::unique_ptr<char[]>{new char[(dims[0] + 7) / 8]};
+ uint64_t width;
+ nulls[0] = 0;
+ bool isNull;
+ int bitmask = 1;
+ *null = isNullPlain(index);
+ if (*null || dims[0] == 0) return NULL;
+ const char *ret = childs[0]->read(currentOffset, &width, &isNull);
+ nulls[0] = isNull ? 0 : bitmask;
+ bitmask <<= 1;
+ for (uint64_t i = currentOffset + 1; i < nextOffset; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ childs[0]->read(i, &width, &isNull);
+ if (!isNull) nulls[(i - currentOffset) / 8] |= bitmask;
+ bitmask <<= 1;
+ }
+ *nullBitmap = std::string(nulls.get());
+ return ret;
+ }
+
+ const char *read(uint64_t index, uint64_t *len, bool *null) const override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support read");
+ }
+
+ std::string readPlain(uint64_t index, bool *null) const override {
+ assert(childs.size() == 1);
+ std::string ret = "{";
+ uint64_t currentOffset = const_cast<uint64_t *>(getOffsets())[index];
+ uint64_t nextOffset = const_cast<uint64_t *>(getOffsets())[index + 1];
+ for (int i = currentOffset; i < nextOffset; i++) {
+ ret += childs[0]->readPlain(i, null);
+ if (i != nextOffset - 1) {
+ ret += ",";
+ }
+ }
+ *null = isNullPlain(index);
+
+ return ret + '}';
+ }
+
+ void readPlainScalar(uint64_t index, Scalar *scalar) const override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support readPlainScalar");
+ }
+
+ void append(const std::string &strValue, bool null) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support append");
+ }
+
+ virtual void append(const char *v, uint64_t valueLen,
+ unsigned char *nullBitmap, int32_t *dims, bool null,
+ bool forInsert) {}
+
+ void append(const char *v, uint64_t valueLen, bool null) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support append");
+ }
+
+ void append(const Datum &datum, bool null) override {
+ assert(getOwnData());
+
+ std::string buffer;
+
+ if (!null) {
+ buffer = DatumToBinary(datum, typeKind);
+ }
+
+ append(buffer.data(), buffer.size(), null);
+ }
+
+ void append(const Scalar *scalar) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support append(Scalar) yet");
+ }
+
+ void append(Vector *src, uint64_t index) override {
+ assert(childs.size() == 1);
+
+ ListVector *lsrc = dynamic_cast<ListVector *>(src);
+ uint64_t rows = getNumOfRowsPlain();
+ uint64_t currentOffset = const_cast<uint64_t *>(lsrc->getOffsets())[index];
+ uint64_t nextOffset = const_cast<uint64_t *>(lsrc->getOffsets())[index + 1];
+ if (offsets.size() == 0) offsets.append(uint64_t(0));
+ offsets.append(getOffsets()[rows] + nextOffset - currentOffset);
+ for (uint64_t i = currentOffset; i < nextOffset; i++) {
+ childs[0]->append(lsrc->getChildVector(0), i);
+ }
+ appendNull(src->isNullPlain(index));
+ }
+
+ void append(Vector *vect) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support append yet");
+ }
+
+ void plainize() {
+ if (selected) {
+ auto cloned = clone();
+ this->swap(*cloned);
+ }
+ selected = nullptr;
+ }
+
+ void materialize() override {
+ plainize();
+ offsets.materialize();
+
+ for (int i = 0; i < childs.size(); i++) {
+ childs[i]->materialize();
+ }
+ }
+
+ std::unique_ptr<Vector> cloneSelected(const SelectList *sel) const override {
+ assert(isValid());
+
+ std::unique_ptr<ListVector> vec(new ListVector(true));
+ vec->typeKind = typeKind;
+ vec->setHasNull(hasNulls);
+ if (!sel || sel->size() == offsets.size() / sizeof(uint64_t) - 1) {
+ dbcommon::ByteBuffer offsetbuf(true);
+ offsetbuf.append(offsets.data(), offsets.size());
+ vec->offsets.swap(offsetbuf);
+ if (hasNulls) vec->nulls.append(nulls.data(), nulls.size());
+ vec->selectListOwner.reset();
+ vec->selected = nullptr;
+ vec->addChildVector(childs[0]->cloneSelected(nullptr));
+ } else {
+ SelectList csel;
+ uint64_t size = sel->size();
+ assert(size < offsets.size() / sizeof(uint64_t) - 1);
+ dbcommon::ByteBuffer offsetbuf(true);
+
+ offsetbuf.append(offsets.get<uint64_t>(0));
+ if (hasNulls) {
+ for (uint64_t i = 0; i < size; i++) {
+ vec->nulls.append(nulls.get((*sel)[i]));
+ uint64_t currentOffset = offsets.get<uint64_t>((*sel)[i]);
+ uint64_t nextOffset = offsets.get<uint64_t>((*sel)[i] + 1);
+ offsetbuf.append(offsetbuf.get<uint64_t>(i) + nextOffset -
+ currentOffset);
+ for (uint64_t j = currentOffset; j < nextOffset; j++) {
+ csel.push_back(j);
+ }
+ }
+ } else {
+ for (uint64_t i = 0; i < size; i++) {
+ uint64_t currentOffset = offsets.get<uint64_t>((*sel)[i]);
+ uint64_t nextOffset = offsets.get<uint64_t>((*sel)[i] + 1);
+ offsetbuf.append(offsetbuf.get<uint64_t>(i) + nextOffset -
+ currentOffset);
+ for (uint64_t j = currentOffset; j < nextOffset; j++) {
+ csel.push_back(j);
+ }
+ }
+ }
+
+ vec->offsets.swap(offsetbuf);
+ vec->selectListOwner.reset();
+ vec->selected = nullptr;
+ vec->addChildVector(childs[0]->cloneSelected(&csel));
+ }
+ return std::move(vec);
+ }
+
+ std::unique_ptr<Vector> extract(const SelectList *sel, uint64_t start,
+ uint64_t end) const override {
+ assert(isValid() && start < end);
+ uint64_t total = getNumOfRowsPlain();
+ assert(end <= total);
+ uint64_t extSz = end - start;
+
+ std::unique_ptr<ListVector> vec(new ListVector(false));
+ vec->typeKind = typeKind;
+ vec->setHasNull(hasNulls);
+ if (offsets.data()) {
+ uint64_t offLen = sizeof(uint64_t);
+ vec->offsets.setData(offsets.data() + start * offLen, extSz * offLen);
+ }
+
+ if (hasNulls) vec->setNulls(getNulls() + start, extSz);
+ vec->selected = nullptr;
+ vec->addChildVector(childs[0]->extract(sel, start, end));
+
+ return std::move(vec);
+ }
+
+ std::unique_ptr<Vector> replicateItemToVec(uint64_t index,
+ int replicateNum) const override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support replicateItemToVec yet");
+ }
+
+ void hash(std::vector<uint64_t> &retval) const override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support hash yet");
+ }
+
+ void cdbHash(std::vector<uint64_t> &retval) const override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support hash yet");
+ }
+
+ dbcommon::ByteBuffer *getValueBuffer() override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support getValueBuffer yet");
+ }
+
+ bool isValid() const override {
+ assert(childs.size() == 1);
+ uint64_t rows = getNumOfRowsPlain();
+ assert(offsets.size() == sizeof(uint64_t) * (rows + 1));
+ if (hasNulls) assert(nulls.size() == rows);
+ if (!childs[0]->isValid()) {
+ return false;
+ }
+ return true;
+ }
+
+ void addChildVector(std::unique_ptr<Vector> vec) override {
+ childs.push_back(std::move(vec));
+ }
+
+ void swap(Vector &vect) override {
+ Vector::swap(vect);
+ auto &v = reinterpret_cast<ListVector &>(vect);
+ this->offsets.swap(v.offsets);
+ this->childs.swap(v.childs);
+ }
+
+ uint64_t getSerializeSizeInternal() const override {
+ size_t sSize = Vector::getSerializeSizeInternal();
+
+ assert(getChildSize() == 1);
+ uint64_t size = getNumOfRowsPlain();
+ sSize += sizeof(uint64_t);
+ sSize += sizeof(uint64_t) * (size + 1);
+ sSize += childs[0]->getSerializeSize();
+
+ return sSize;
+ }
+
+ void serializeInternal(NodeSerializer *serializer) override {
+ Vector::serializeInternal(serializer);
+
+ assert(getChildSize() == 1);
+ uint64_t size = getNumOfRows();
+ serializer->write<uint64_t>(size);
+
+ if (allSelected()) {
+ serializer->writeBytes(offsets.data(), sizeof(uint64_t) * (size + 1));
+ getChildVector(0)->serialize(serializer);
+ } else {
+ // TODO(chiyang): no clone
+ auto tmp = this->clone();
+ serializer->writeBytes(
+ reinterpret_cast<ListVector *>(tmp.get())->offsets.data(),
+ sizeof(uint64_t) * (size + 1));
+ tmp->getChildVector(0)->serialize(serializer);
+ }
+ }
+
+ void deserializeInternal(NodeDeserializer *deserializer) override {
+ Vector::deserializeInternal(deserializer);
+
+ uint64_t size = deserializer->read<uint64_t>();
+ const char *dstr = deserializer->readBytes(sizeof(uint64_t) * (size + 1));
+ setOffsets(reinterpret_cast<const uint64_t *>(dstr), size + 1);
+
+ std::unique_ptr<Vector> vect = Vector::deserialize(deserializer);
+ this->addChildVector(std::move(vect));
+ }
+
+ void selectDistinct(SelectList *selected, Scalar *scalar) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support selectDistinct yet");
+ }
+
+ void selectDistinct(SelectList *selected, Vector *vect) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "ListVector does not support selectDistinct yet");
+ }
+
+ protected:
+ void resizeInternal(size_t plainSize) override {}
+
+ dbcommon::ByteBuffer offsets; // type: uint64_t
+};
+
+class SmallIntArrayVector : public ListVector {
+ public:
+ explicit SmallIntArrayVector(bool ownData) : ListVector(ownData) {
+ setTypeKind(SMALLINTARRAYID);
+ }
+
+ void append(const char *v, uint64_t valueLen, unsigned char *nullBitmap,
+ int32_t *dims, bool null, bool forInsert) override {
+ assert(childs.size() == 1);
+ uint64_t width = sizeof(int16_t);
+ // Now we only support 1 dimension array
+ int32_t nums = dims ? dims[0] : 0;
+ if (!null) {
+ if (nullBitmap) {
+ int bitmask = 1;
+ if (forInsert) {
+ for (int32_t i = 0, j = 0; i < nums; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ if (nullBitmap[i / 8] & bitmask) {
+ childs[0]->append(v + j * width, width, false);
+ j++;
+ } else {
+ childs[0]->append(v + j * width, width, true);
+ }
+ bitmask <<= 1;
+ }
+ } else {
+ for (int32_t i = 0; i < nums; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ if (nullBitmap[i / 8] & bitmask) {
+ childs[0]->append(v + i * width, width, false);
+ } else {
+ childs[0]->append(v + i * width, width, true);
+ }
+ bitmask <<= 1;
+ }
+ }
+ } else {
+ for (int32_t i = 0; i < nums; i++)
+ childs[0]->append(v + i * width, width, false);
+ }
+ }
+
+ if (offsets.size() == 0) offsets.append(uint64_t(0));
+ uint64_t rows = getNumOfRowsPlain();
+ offsets.append(getOffsets()[rows] + nums);
+ appendNull(null);
+ }
+};
+
+class IntArrayVector : public ListVector {
+ public:
+ explicit IntArrayVector(bool ownData) : ListVector(ownData) {
+ setTypeKind(INTARRAYID);
+ }
+
+ void append(const char *v, uint64_t valueLen, unsigned char *nullBitmap,
+ int32_t *dims, bool null, bool forInsert) override {
+ assert(childs.size() == 1);
+ uint64_t width = sizeof(int32_t);
+ // Now we only support 1 dimension array
+ int32_t nums = dims ? dims[0] : 0;
+ if (!null) {
+ if (nullBitmap) {
+ int bitmask = 1;
+ if (forInsert) {
+ for (int32_t i = 0, j = 0; i < nums; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ if (nullBitmap[i / 8] & bitmask) {
+ childs[0]->append(v + j * width, width, false);
+ j++;
+ } else {
+ childs[0]->append(v + j * width, width, true);
+ }
+ bitmask <<= 1;
+ }
+ } else {
+ for (int32_t i = 0; i < nums; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ if (nullBitmap[i / 8] & bitmask) {
+ childs[0]->append(v + i * width, width, false);
+ } else {
+ childs[0]->append(v + i * width, width, true);
+ }
+ bitmask <<= 1;
+ }
+ }
+ } else {
+ for (int32_t i = 0; i < nums; i++)
+ childs[0]->append(v + i * width, width, false);
+ }
+ }
+
+ if (offsets.size() == 0) offsets.append(uint64_t(0));
+ uint64_t rows = getNumOfRowsPlain();
+ offsets.append(getOffsets()[rows] + nums);
+ appendNull(null);
+ }
+};
+
+class BigIntArrayVector : public ListVector {
+ public:
+ explicit BigIntArrayVector(bool ownData) : ListVector(ownData) {
+ setTypeKind(BIGINTARRAYID);
+ }
+
+ void append(const char *v, uint64_t valueLen, unsigned char *nullBitmap,
+ int32_t *dims, bool null, bool forInsert) override {
+ assert(childs.size() == 1);
+ uint64_t width = sizeof(int64_t);
+ // Now we only support 1 dimension array
+ int32_t nums = dims ? dims[0] : 0;
+ if (!null) {
+ if (nullBitmap) {
+ int bitmask = 1;
+ if (forInsert) {
+ for (int32_t i = 0, j = 0; i < nums; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ if (nullBitmap[i / 8] & bitmask) {
+ childs[0]->append(v + j * width, width, false);
+ j++;
+ } else {
+ childs[0]->append(v + j * width, width, true);
+ }
+ bitmask <<= 1;
+ }
+ } else {
+ for (int32_t i = 0; i < nums; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ if (nullBitmap[i / 8] & bitmask) {
+ childs[0]->append(v + i * width, width, false);
+ } else {
+ childs[0]->append(v + i * width, width, true);
+ }
+ bitmask <<= 1;
+ }
+ }
+ } else {
+ for (int32_t i = 0; i < nums; i++)
+ childs[0]->append(v + i * width, width, false);
+ }
+ }
+
+ if (offsets.size() == 0) offsets.append(uint64_t(0));
+ uint64_t rows = getNumOfRowsPlain();
+ offsets.append(getOffsets()[rows] + nums);
+ appendNull(null);
+ }
+};
+
+class FloatArrayVector : public ListVector {
+ public:
+ explicit FloatArrayVector(bool ownData) : ListVector(ownData) {
+ setTypeKind(FLOATARRAYID);
+ }
+
+ void append(const char *v, uint64_t valueLen, unsigned char *nullBitmap,
+ int32_t *dims, bool null, bool forInsert) override {
+ assert(childs.size() == 1);
+ uint64_t width = sizeof(float);
+ // Now we only support 1 dimension array
+ int32_t nums = dims ? dims[0] : 0;
+ if (!null) {
+ if (nullBitmap) {
+ int bitmask = 1;
+ if (forInsert) {
+ for (int32_t i = 0, j = 0; i < nums; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ if (nullBitmap[i / 8] & bitmask) {
+ childs[0]->append(v + j * width, width, false);
+ j++;
+ } else {
+ childs[0]->append(v + j * width, width, true);
+ }
+ bitmask <<= 1;
+ }
+ } else {
+ for (int32_t i = 0; i < nums; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ if (nullBitmap[i / 8] & bitmask) {
+ childs[0]->append(v + i * width, width, false);
+ } else {
+ childs[0]->append(v + i * width, width, true);
+ }
+ bitmask <<= 1;
+ }
+ }
+ } else {
+ for (int32_t i = 0; i < nums; i++)
+ childs[0]->append(v + i * width, width, false);
+ }
+ }
+
+ if (offsets.size() == 0) offsets.append(uint64_t(0));
+ uint64_t rows = getNumOfRowsPlain();
+ offsets.append(getOffsets()[rows] + nums);
+ appendNull(null);
+ }
+};
+
+class DoubleArrayVector : public ListVector {
+ public:
+ explicit DoubleArrayVector(bool ownData) : ListVector(ownData) {
+ setTypeKind(DOUBLEARRAYID);
+ }
+
+ void append(const char *v, uint64_t valueLen, unsigned char *nullBitmap,
+ int32_t *dims, bool null, bool forInsert) override {
+ assert(childs.size() == 1);
+ uint64_t width = sizeof(double);
+ // Now we only support 1 dimension array
+ int32_t nums = dims ? dims[0] : 0;
+ if (!null) {
+ if (nullBitmap) {
+ int bitmask = 1;
+ if (forInsert) {
+ for (int32_t i = 0, j = 0; i < nums; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ if (nullBitmap[i / 8] & bitmask) {
+ childs[0]->append(v + j * width, width, false);
+ j++;
+ } else {
+ childs[0]->append(v + j * width, width, true);
+ }
+ bitmask <<= 1;
+ }
+ } else {
+ for (int32_t i = 0; i < nums; i++) {
+ if (bitmask == 0x100) bitmask = 1;
+ if (nullBitmap[i / 8] & bitmask) {
+ childs[0]->append(v + i * width, width, false);
+ } else {
+ childs[0]->append(v + i * width, width, true);
+ }
+ bitmask <<= 1;
+ }
+ }
+ } else {
+ for (int32_t i = 0; i < nums; i++)
+ childs[0]->append(v + i * width, width, false);
+ }
+ }
+
+ if (offsets.size() == 0) offsets.append(uint64_t(0));
+ uint64_t rows = getNumOfRowsPlain();
+ offsets.append(getOffsets()[rows] + nums);
+ appendNull(null);
+ }
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_LIST_VECTOR_H_
diff --git a/depends/dbcommon/src/dbcommon/common/vector/struct-vector.h b/depends/dbcommon/src/dbcommon/common/vector/struct-vector.h
new file mode 100644
index 0000000..dbd5d43
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector/struct-vector.h
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_STRUCT_VECTOR_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_STRUCT_VECTOR_H_
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "dbcommon/common/vector.h"
+
+namespace dbcommon {
+class StructVector : public Vector {
+ public:
+ explicit StructVector(bool ownData) : Vector(ownData) {
+ setTypeKind(STRUCTID);
+ setHasNull(false);
+ }
+
+ virtual ~StructVector() {}
+
+ uint64_t getNumOfRowsPlain() const override {
+ if (childs.size() > 0) {
+ return childs[0]->getNumOfRowsPlain();
+ }
+ return 0;
+ }
+
+ void setValue(const char *value, uint64_t size) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not have value array");
+ }
+
+ const char *read(uint64_t index, uint64_t *len, bool *null) const override {
+ if (childs.size() > 0) {
+ return childs[0]->read(index, len, null);
+ }
+ return nullptr;
+ }
+
+ std::string readPlain(uint64_t index, bool *null) const override {
+ std::string ret = "";
+ for (int i = 0; i < childs.size(); i++) {
+ ret += childs[i]->readPlain(index, null);
+ if (i != childs.size() - 1) {
+ ret += ",";
+ }
+ }
+ return ret;
+ }
+
+ void readPlainScalar(uint64_t index, Scalar *scalar) const override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support readPlainScalar");
+ }
+
+ void append(const std::string &strValue, bool null) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support append yet");
+ }
+
+ void append(const char *v, uint64_t valueLen, bool null) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support append yet");
+ }
+
+ void append(const Datum &datum, bool null) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support append yet");
+ }
+
+ void append(const Scalar *scalar) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support append yet");
+ }
+
+ void append(Vector *src, uint64_t index) override {
+ for (int i = 0; i < childs.size(); i++) {
+ childs[i]->append(src->getChildVector(i), index);
+ }
+ }
+
+ void append(Vector *vect) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support append yet");
+ }
+
+ void append(Vector *vect, SelectList *sel) override {
+ Vector::append(vect, sel);
+ assert(childs.empty() || childs.size() == vect->getChildSize());
+ if (childs.empty()) {
+ for (auto i = 0; i < vect->getChildSize(); i++)
+ childs.push_back(
+ Vector::BuildVector(vect->getChildVector(i)->getTypeKind(), true,
+ vect->getChildVector(i)->getTypeModifier()));
+ }
+ for (int i = 0; i < childs.size(); i++) {
+ childs[i]->append(vect->getChildVector(i), sel);
+ }
+ }
+
+ void reset(bool resetBelongingTupleBatch = false) override {
+ for (int i = 0; i < childs.size(); i++) {
+ childs[i]->reset(resetBelongingTupleBatch);
+ }
+ }
+
+ void materialize() override {
+ for (int i = 0; i < childs.size(); i++) {
+ childs[i]->materialize();
+ }
+ }
+
+ std::unique_ptr<Vector> cloneSelected(const SelectList *sel) const override {
+ std::unique_ptr<Vector> vec(new StructVector(false));
+ for (int i = 0; i < childs.size(); i++) {
+ vec->addChildVector(childs[i]->cloneSelected(sel));
+ }
+ return vec;
+ }
+
+ std::unique_ptr<Vector> extract(const SelectList *sel, uint64_t start,
+ uint64_t end) const override {
+ std::unique_ptr<Vector> vec(new StructVector(false));
+ for (int i = 0; i < childs.size(); i++) {
+ vec->addChildVector(childs[i]->extract(sel, start, end));
+ }
+ return vec;
+ }
+
+ std::unique_ptr<Vector> replicateItemToVec(uint64_t index,
+ int replicateNum) const override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support replicateItemToVec yet");
+ }
+
+ void hash(std::vector<uint64_t> &retval) const override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support hash yet");
+ }
+
+ void cdbHash(std::vector<uint64_t> &retval) const override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support hash yet");
+ }
+
+ dbcommon::ByteBuffer *getValueBuffer() override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support getValueBuffer yet");
+ }
+
+ bool isValid() const override {
+ for (int i = 0; i < childs.size(); i++) {
+ if (!childs[i]->isValid()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ uint64_t getSerializeSizeInternal() const override {
+ size_t sSize = Vector::getSerializeSizeInternal();
+
+ sSize += sizeof(uint64_t);
+ uint64_t nChild = childs.size();
+ for (int i = 0; i < nChild; i++) {
+ sSize += childs[i]->getSerializeSize();
+ }
+
+ return sSize;
+ }
+
+ void serializeInternal(NodeSerializer *serializer) override {
+ Vector::serializeInternal(serializer);
+
+ uint64_t nChild = getChildSize();
+ serializer->write<uint64_t>(nChild);
+
+ for (int i = 0; i < nChild; i++) {
+ getChildVector(i)->serialize(serializer);
+ }
+ }
+
+ void deserializeInternal(NodeDeserializer *deserializer) override {
+ Vector::deserializeInternal(deserializer);
+
+ uint64_t nChild = deserializer->read<uint64_t>();
+
+ for (int i = 0; i < nChild; i++) {
+ std::unique_ptr<Vector> vect = Vector::deserialize(deserializer);
+ this->addChildVector(std::move(vect));
+ }
+ }
+
+ void selectDistinct(SelectList *selected, Scalar *scalar) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support selectDistinct yet");
+ }
+
+ void selectDistinct(SelectList *selected, Vector *vect) override {
+ LOG_ERROR(ERRCODE_FEATURE_NOT_SUPPORTED,
+ "StructVector does not support selectDistinct yet");
+ }
+
+ protected:
+ void resizeInternal(size_t plainSize) override {}
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_STRUCT_VECTOR_H_
diff --git a/depends/dbcommon/src/dbcommon/common/vector/timestamp-vector.h b/depends/dbcommon/src/dbcommon/common/vector/timestamp-vector.h
new file mode 100644
index 0000000..e1528f5
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector/timestamp-vector.h
@@ -0,0 +1,698 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_TIMESTAMP_VECTOR_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_TIMESTAMP_VECTOR_H_
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "dbcommon/common/vector-transformer.h"
+#include "dbcommon/common/vector.h"
+
+namespace dbcommon {
+
+class TimestampVector : public Vector {
+ public:
+ explicit TimestampVector(bool ownData)
+ : Vector(ownData), nanoseconds(ownData) {
+ if (ownData) {
+ values.reserve(sizeof(int64_t) * DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ nanoseconds.reserve(sizeof(int64_t) * DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ }
+ setTypeKind(TIMESTAMPID);
+ }
+
+ ~TimestampVector() override {}
+
+ const char *getNanoseconds() const override { return nanoseconds.data(); }
+
+ uint64_t getNumOfRowsPlain() const override {
+ assert(values.size() == nanoseconds.size());
+ return values.size() / sizeof(int64_t);
+ }
+
+ void setValue(const char *value, uint64_t size) override {
+ assert(!getOwnData() && value != nullptr && size >= 0);
+ values.setData(value, size);
+ }
+
+ void setNanoseconds(const char *value, uint64_t size) override {
+ assert(!getOwnData() && value != nullptr && size >= 0);
+ nanoseconds.setData(value, size);
+ }
+
+ void materialize() override {
+ values.materialize<int64_t>(selected);
+ nanoseconds.materialize<int64_t>(selected);
+ if (hasNulls) nulls.materialize(selected);
+ selected = nullptr;
+ }
+
+ std::unique_ptr<Vector> cloneSelected(const SelectList *sel) const override {
+ assert(isValid());
+
+ std::unique_ptr<Vector> retval = Vector::BuildVector(getTypeKind(), true);
+ TimestampVector *rt = static_cast<TimestampVector *>(retval.get());
+
+ rt->setTypeKind(typeKind);
+ rt->setHasNull(hasNulls);
+ assert(values.size() == nanoseconds.size());
+ if (!sel || sel->size() == values.size() / sizeof(int64_t)) {
+ dbcommon::ByteBuffer valuebuf(true);
+ dbcommon::ByteBuffer nanosecondsbuf(true);
+ valuebuf.append(values.data(), values.size());
+ nanosecondsbuf.append(nanoseconds.data(), nanoseconds.size());
+ rt->values.swap(valuebuf);
+ rt->nanoseconds.swap(nanosecondsbuf);
+ if (hasNulls) rt->nulls.append(nulls.data(), nulls.size());
+ rt->selectListOwner.reset();
+ rt->selected = nullptr;
+ } else {
+ uint64_t size = sel->size();
+ dbcommon::ByteBuffer valbuf(true);
+ dbcommon::ByteBuffer nanosecondsbuf(true);
+
+ if (hasNulls) {
+ for (uint64_t i = 0; i < size; ++i) {
+ valbuf.append(values.get<int64_t>((*sel)[i]));
+ nanosecondsbuf.append(nanoseconds.get<int64_t>((*sel)[i]));
+ rt->nulls.append(nulls.get((*sel)[i]));
+ }
+ } else {
+ for (uint64_t i = 0; i < size; ++i) {
+ valbuf.append(values.get<int64_t>((*sel)[i]));
+ nanosecondsbuf.append(nanoseconds.get<int64_t>((*sel)[i]));
+ }
+ }
+
+ rt->values.swap(valbuf);
+ rt->nanoseconds.swap(nanosecondsbuf);
+ rt->selectListOwner.reset();
+ rt->selected = nullptr;
+ }
+
+ assert(rt->isValid());
+ return std::move(retval);
+ }
+
+ std::unique_ptr<Vector> extract(const SelectList *sel, uint64_t start,
+ uint64_t end) const override {
+ assert(isValid() && start < end);
+ uint64_t total = getNumOfRowsPlain();
+ assert(end <= total);
+ uint64_t extSz = end - start;
+
+ std::unique_ptr<Vector> retval = Vector::BuildVector(getTypeKind(), false);
+ TimestampVector *rt = static_cast<TimestampVector *>(retval.get());
+
+ rt->setTypeKind(typeKind);
+ rt->setHasNull(hasNulls);
+ if (hasNulls) rt->setNulls(getNulls() + start, extSz);
+
+ uint64_t typLen = sizeof(int64_t);
+ if (values.data())
+ rt->setValue(values.data() + start * typLen, extSz * typLen);
+ if (nanoseconds.data())
+ rt->setNanoseconds(nanoseconds.data() + start * typLen, extSz * typLen);
+ rt->setSelected(sel, true);
+
+ assert(rt->isValid());
+ return std::move(retval);
+ }
+
+ std::unique_ptr<Vector> replicateItemToVec(uint64_t index,
+ int replicateNum) const override {
+ assert(isValid());
+
+ auto retval = Vector::BuildVector(getTypeKind(), true);
+ auto rt = static_cast<TimestampVector *>(retval.get());
+
+ rt->setTypeKind(typeKind);
+ rt->setHasNull(hasNulls);
+
+ dbcommon::ByteBuffer valbuf(true);
+ dbcommon::ByteBuffer nanosecondsbuf(true);
+ uint64_t idx = getPlainIndex(index);
+ auto value = values.get<int64_t>(idx);
+ auto nanosecond = nanoseconds.get<int64_t>(idx);
+ auto null = isNullPlain(idx);
+
+ if (hasNulls) {
+ for (int i = 0; i < replicateNum; ++i) {
+ valbuf.append(value);
+ nanosecondsbuf.append(nanosecond);
+ rt->nulls.append(null);
+ }
+ } else {
+ for (int i = 0; i < replicateNum; ++i) {
+ valbuf.append(values.get<int64_t>(idx));
+ nanosecondsbuf.append(nanoseconds.get<int64_t>(idx));
+ }
+ }
+ rt->values.swap(valbuf);
+ rt->nanoseconds.swap(nanosecondsbuf);
+ assert(rt->isValid());
+
+ return std::move(retval);
+ }
+
+ void merge(size_t plainSize, const std::vector<Datum> &data,
+ const std::vector<SelectList *> &sels) override {
+ assert(data.size() == sels.size());
+
+ values.resize(plainSize * sizeof(int64_t));
+ nanoseconds.resize(plainSize * sizeof(int64_t));
+ nulls.resize(plainSize);
+ int64_t *__restrict__ secs = reinterpret_cast<int64_t *>(values.data());
+ int64_t *__restrict__ nanos =
+ reinterpret_cast<int64_t *>(nanoseconds.data());
+ bool *__restrict__ nulls = this->nulls.data();
+
+ for (auto inputIdx = 0; inputIdx < data.size(); inputIdx++) {
+ auto sel = sels[inputIdx];
+ if (sel == nullptr) continue;
+ auto datum = DatumGetValue<Object *>(data[inputIdx]);
+
+ if (auto vec = dynamic_cast<Vector *>(datum)) {
+ auto secsIn = reinterpret_cast<const int64_t *>(vec->getValue());
+ auto nanosIn = reinterpret_cast<const int64_t *>(vec->getNanoseconds());
+ for (auto idx : *sel) secs[idx] = secsIn[idx];
+ for (auto idx : *sel) nanos[idx] = nanosIn[idx];
+ if (vec->hasNullValue()) {
+ auto nullsIn = vec->getNulls();
+ for (auto idx : *sel) nulls[idx] = nullsIn[idx];
+ } else {
+ for (auto idx : *sel) nulls[idx] = false;
+ }
+ } else {
+ assert(dynamic_cast<Scalar *>(datum));
+ auto scalar = reinterpret_cast<Scalar *>(datum);
+ Timestamp *ts = DatumGetValue<Timestamp *>(scalar->value);
+ auto sec = ts->second;
+ auto nano = ts->nanosecond;
+ auto null = scalar->isnull;
+ if (null) {
+ for (auto idx : *sel) {
+ nulls[idx] = true;
+ }
+ } else {
+ for (auto idx : *sel) {
+ secs[idx] = sec;
+ nanos[idx] = nano;
+ nulls[idx] = false;
+ }
+ }
+ }
+ } // End of loop each input data
+ }
+
+ // XXX: Not able keep multiple iterator at the same time
+ const char *read(uint64_t index, uint64_t *len, bool *null) const override {
+ uint64_t i = getPlainIndex(index);
+ *null = isNullPlain(i);
+ nextBuffer = 1 - nextBuffer;
+ readBuffer[nextBuffer].second =
+ *reinterpret_cast<int64_t *>(values.data() + (i * (sizeof(int64_t))));
+ readBuffer[nextBuffer].nanosecond = *reinterpret_cast<int64_t *>(
+ nanoseconds.data() + (i * (sizeof(int64_t))));
+ *len = sizeof(readBuffer[nextBuffer]);
+ return reinterpret_cast<const char *>(&readBuffer[nextBuffer]);
+ }
+
+ const Timestamp *read(uint64_t index, bool *null) const {
+ uint64_t i = getPlainIndex(index);
+ *null = isNullPlain(i);
+ nextBuffer = 1 - nextBuffer;
+ readBuffer[nextBuffer].second =
+ *reinterpret_cast<int64_t *>(values.data() + (i * (sizeof(int64_t))));
+ readBuffer[nextBuffer].nanosecond = *reinterpret_cast<int64_t *>(
+ nanoseconds.data() + (i * (sizeof(int64_t))));
+ return &readBuffer[nextBuffer];
+ }
+
+ std::string readPlain(uint64_t index, bool *null) const override {
+ *null = isNullPlain(index);
+ if (*null) return "NULL";
+ return std::move(TimestampType::toString(values.get<int64_t>(index),
+ nanoseconds.get<int64_t>(index)));
+ }
+
+ void readPlainScalar(uint64_t index, Scalar *scalar) const override {
+ nextBuffer = 1 - nextBuffer;
+ readBuffer[nextBuffer].second = *reinterpret_cast<int64_t *>(
+ values.data() + (index * (sizeof(int64_t))));
+ readBuffer[nextBuffer].nanosecond = *reinterpret_cast<int64_t *>(
+ nanoseconds.data() + (index * (sizeof(int64_t))));
+ scalar->value =
+ CreateDatum(reinterpret_cast<const char *>(&readBuffer[nextBuffer]));
+ scalar->isnull = isNullPlain(index);
+ scalar->length = sizeof(readBuffer[0]);
+ }
+
+ void append(const char *v, uint64_t valueLen, bool null) override {
+ assert(getOwnData() && v != nullptr);
+
+ if (!null) {
+ assert(valueLen == sizeof(int64_t) + sizeof(int64_t));
+ const Timestamp *tsv = reinterpret_cast<const Timestamp *>(v);
+ values.append<int64_t>(tsv->second);
+ nanoseconds.append<int64_t>(tsv->nanosecond);
+ } else {
+ values.append<int64_t>(0);
+ nanoseconds.append<int64_t>(0);
+ }
+
+ appendNull(null);
+ }
+
+ void append(Vector *src, uint64_t index) override {
+ bool isnull = false;
+ uint64_t len = 0;
+ const char *ret = src->read(index, &len, &isnull);
+ append(ret, len, isnull);
+ }
+
+ void append(const std::string &strValue, bool null) override {
+ assert(getOwnData());
+
+ Timestamp value;
+
+ if (!null) {
+ value = TimestampType::fromString(strValue);
+ }
+
+ append(reinterpret_cast<char *>(&value), sizeof(int64_t) + sizeof(int64_t),
+ null);
+ }
+
+ void append(const Datum &datum, bool null) override {
+ Timestamp *v = DatumGetValue<Timestamp *>(datum);
+ if (v) {
+ values.append(v->second);
+ nanoseconds.append(v->nanosecond);
+ } else {
+ assert(null == true);
+ values.append(int64_t(0));
+ nanoseconds.append(int64_t(0));
+ }
+
+ appendNull(null);
+ }
+
+ void append(const Scalar *scalar) override {
+ Timestamp *v = DatumGetValue<Timestamp *>(scalar->value);
+ if (v) {
+ values.append(v->second);
+ nanoseconds.append(v->nanosecond);
+ } else {
+ assert(scalar->isnull == true);
+ values.append(int64_t(0));
+ nanoseconds.append(int64_t(0));
+ }
+
+ appendNull(scalar->isnull);
+ }
+
+ void append(const Datum &valDatum, const Datum &nanoDatum,
+ bool null) override {
+ if (!null) {
+ values.append(*reinterpret_cast<const int64_t *>(&valDatum));
+ nanoseconds.append(*reinterpret_cast<const int64_t *>(&nanoDatum));
+ } else {
+ values.append(int64_t(0));
+ nanoseconds.append(int64_t(0));
+ }
+ appendNull(null);
+ }
+
+ void append(Vector *vect, SelectList *sel) override {
+ Vector::append(vect, sel);
+
+ const dbcommon::ByteBuffer *newValues = vect->getValueBuffer();
+ const dbcommon::ByteBuffer *newNanoseconds = vect->getNanosecondsBuffer();
+
+ if (!sel || sel->size() == vect->getNumOfRowsPlain()) {
+ values.append(newValues->data(), newValues->size());
+ nanoseconds.append(newNanoseconds->data(), newNanoseconds->size());
+ } else {
+ uint64_t newSize = sel->size();
+ for (uint64_t i = 0; i < newSize; i++) {
+ values.append(newValues->get<int64_t>((*sel)[i]));
+ nanoseconds.append(newNanoseconds->get<int64_t>((*sel)[i]));
+ }
+ }
+ }
+
+ void reset(bool resetBelongingTupleBatch = false) override {
+ Vector::reset(resetBelongingTupleBatch);
+ values.resize(0);
+ nanoseconds.resize(0);
+ }
+
+ void append(Vector *vect) override {
+ assert(getOwnData());
+
+ this->append(vect, vect->getSelected());
+ }
+
+ bool isValid() const override {
+ Vector::isValid();
+
+ if (hasNulls) {
+ assert(nulls.size() == values.size() / sizeof(int64_t));
+ assert(nulls.size() == nanoseconds.size() / sizeof(int64_t));
+ }
+ return true;
+ }
+
+ void hash(std::vector<uint64_t> &retval) const override {
+ auto sec = reinterpret_cast<const int64_t *>(values.data());
+ auto nanosec = reinterpret_cast<const int64_t *>(nanoseconds.data());
+ auto size = getNumOfRows();
+ auto sel = selected;
+ retval.resize(size);
+ if (allSelected()) {
+ for (size_t i = 0; i < size; ++i) {
+ // this comes from Hash128to64 in cityhash
+ const uint64_t kMul = 0x9ddfea08eb382d69ULL;
+ uint64_t a = (sec[i] ^ nanosec[i]) * kMul;
+ a ^= (a >> 47);
+ uint64_t b = (nanosec[i] ^ a) * kMul;
+ b ^= (b >> 47);
+ b *= kMul;
+ retval[i] = b;
+ }
+ } else {
+ for (size_t idx = 0; idx < size; ++idx) {
+ // this comes from Hash128to64 in cityhash
+ auto i = (*sel)[idx];
+ const uint64_t kMul = 0x9ddfea08eb382d69ULL;
+ uint64_t a = (sec[i] ^ nanosec[i]) * kMul;
+ a ^= (a >> 47);
+ uint64_t b = (nanosec[i] ^ a) * kMul;
+ b ^= (b >> 47);
+ b *= kMul;
+ retval[idx] = b;
+ }
+ }
+
+ if (hasNulls) {
+ auto nulls = getNulls();
+ if (allSelected()) {
+ for (uint64_t i = 0; i < size; ++i)
+ if (nulls[i]) retval[i] = 0;
+ } else {
+ for (uint64_t i = 0; i < size; ++i)
+ if (nulls[(*sel)[i]]) retval[i] = 0;
+ }
+ }
+ }
+
+ void cdbHash(std::vector<uint64_t> &retval) const override {
+ auto base = reinterpret_cast<const int64_t *>(getValue());
+ size_t len = sizeof(int64_t);
+
+ if (selected) {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = nulls.get(idx) ? cdbhashnull(retval[i])
+ : cdbhash(&base[idx], len, retval[i]);
+ }
+ } else {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = cdbhash(&base[idx], len, retval[i]);
+ }
+ }
+ } else {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = nulls.get(i) ? cdbhashnull(retval[i])
+ : cdbhash(&base[i], len, retval[i]);
+ }
+ } else {
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = cdbhash(&base[i], len, retval[i]);
+ }
+ }
+ }
+ }
+
+ dbcommon::ByteBuffer *getValueBuffer() override { return &values; }
+
+ dbcommon::ByteBuffer *getNanosecondsBuffer() override { return &nanoseconds; }
+
+ void swap(Vector &vect) override {
+ Vector::swap(vect);
+ auto &v = dynamic_cast<TimestampVector &>(vect);
+ this->values.swap(v.values);
+ this->nanoseconds.swap(v.nanoseconds);
+ }
+
+ uint64_t checkGroupByKey(uint64_t index, const char *dataPtr) const override {
+ assert(index < getNumOfRows() && "invalid input");
+ assert(getNumOfRows() > 0);
+ uint64_t i = getPlainIndex(index);
+ const TimestampDataInfo *info =
+ reinterpret_cast<const TimestampDataInfo *>(dataPtr);
+ if (isNullPlain(i) != info->isNull) return 0;
+ if (!info->isNull) {
+ if (*reinterpret_cast<int64_t *>(values.data() + i * (sizeof(int64_t))) !=
+ info->data.second)
+ return 0;
+ if (*reinterpret_cast<int64_t *>(nanoseconds.data() +
+ i * (sizeof(int64_t))) !=
+ info->data.nanosecond)
+ return 0;
+ }
+ return sizeof(TimestampDataInfo);
+ }
+
+ void checkGroupByKeys(std::vector<const void *> &dataPtrSrc) const override {
+ const int64_t *data =
+ reinterpret_cast<const int64_t *>(TimestampVector::getValue());
+ const int64_t *nanos =
+ reinterpret_cast<const int64_t *>(TimestampVector::getNanoseconds());
+
+ const uint64_t size = TimestampVector::getNumOfRows();
+ const char **dataPtr = reinterpret_cast<const char **>(&dataPtrSrc[0]);
+ if (selected) {
+ auto &sel = *selected;
+
+#pragma clang loop unroll_count(4)
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ uint64_t index = sel[i];
+ char null = hasNulls ? nulls.getChar(index) : 0;
+ const int64_t d = data[index];
+ const int64_t n = nanos[index];
+
+ const TimestampDataInfo *info =
+ reinterpret_cast<const TimestampDataInfo *>(dataPtr[i]);
+ if (null == info->isNull &&
+ (null || (d == info->data.second && n == info->data.nanosecond)))
+ dataPtr[i] = alignedAddress(dataPtr[i] + sizeof(TimestampDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ } else {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ char null = nulls.getChar(i);
+ const int64_t d = data[i];
+ const int64_t n = nanos[i];
+
+ const TimestampDataInfo *info =
+ reinterpret_cast<const TimestampDataInfo *>(dataPtr[i]);
+ if (null == info->isNull && (null || (d == info->data.second &&
+ n == info->data.nanosecond)))
+ dataPtr[i] =
+ alignedAddress(dataPtr[i] + sizeof(TimestampDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ } else {
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ char null = false;
+ const int64_t d = data[i];
+ const int64_t n = nanos[i];
+
+ const TimestampDataInfo *info =
+ reinterpret_cast<const TimestampDataInfo *>(dataPtr[i]);
+ if (null == info->isNull && (null || (d == info->data.second &&
+ n == info->data.nanosecond)))
+ dataPtr[i] =
+ alignedAddress(dataPtr[i] + sizeof(TimestampDataInfo));
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ }
+ }
+ }
+
+ uint64_t getSerializeSizeInternal() const override {
+ size_t sSize = Vector::getSerializeSizeInternal();
+ sSize += sizeof(uint64_t) + values.size();
+ sSize += sizeof(uint64_t) + nanoseconds.size();
+ return sSize;
+ }
+
+ void serializeInternal(NodeSerializer *serializer) override {
+ Vector::serializeInternal(serializer);
+
+ if (allSelected()) {
+ uint64_t vsz = getNumOfRowsPlain() * 16;
+ serializer->write<uint64_t>(vsz);
+ if (vsz > 0) {
+ serializer->writeBytes(getValue(), vsz / 2);
+ serializer->writeBytes(getNanoseconds(), vsz / 2);
+ }
+ } else {
+ uint64_t vsz = selected->size() * 16;
+ serializer->write<uint64_t>(vsz);
+ if (vsz > 0) {
+ auto seconds = reinterpret_cast<const uint64_t *>(getValue());
+ for (auto i : *selected) {
+ serializer->write(seconds[i]);
+ }
+ auto nanoseconds = reinterpret_cast<const uint64_t *>(getNanoseconds());
+ for (auto i : *selected) {
+ serializer->write(nanoseconds[i]);
+ }
+ }
+ }
+ }
+
+ void deserializeInternal(NodeDeserializer *deserializer) override {
+ Vector::deserializeInternal(deserializer);
+
+ uint64_t vsz = deserializer->read<uint64_t>();
+ if (vsz > 0) {
+ const char *dstr = deserializer->readBytes(vsz);
+ setValue(dstr, vsz / 2);
+ setNanoseconds(dstr + vsz / 2, vsz / 2);
+ }
+
+ assert(isValid());
+ }
+
+ void selectDistinct(SelectList *selected, Scalar *scalar) override {
+ SelectList *sel = getSelected();
+ uint64_t sz = getNumOfRows();
+ const int64_t *vals = reinterpret_cast<const int64_t *>(getValue());
+ const int64_t *nanos = reinterpret_cast<const int64_t *>(getNanoseconds());
+
+ auto operation = [&](uint64_t plainIdx) { // NOLINT
+ if (!scalar->isnull) {
+ Timestamp *value = DatumGetValue<Timestamp *>(scalar->value);
+ return (vals[plainIdx] != value->second ||
+ nanos[plainIdx] != value->nanosecond);
+ }
+ return false;
+ };
+ const bool *nulls = getNulls();
+ dbcommon::transformDistinct(sz, sel, nulls, selected, scalar->isnull,
+ operation);
+ }
+
+ void selectDistinct(SelectList *selected, Vector *vect) override {
+ SelectList *sel1 = getSelected();
+ SelectList *sel2 = vect->getSelected();
+ uint64_t sz = getNumOfRows();
+ assert(sz == vect->getNumOfRows() && "invalid input");
+
+ const int64_t *vals1 = reinterpret_cast<const int64_t *>(values.data());
+ const int64_t *nanos1 = reinterpret_cast<const int64_t *>(getNanoseconds());
+ const int64_t *vals2 = reinterpret_cast<const int64_t *>(vect->getValue());
+ const int64_t *nanos2 =
+ reinterpret_cast<const int64_t *>(vect->getNanoseconds());
+
+ auto operation = [&](uint64_t plainIdx) { // NOLINT
+ return (vals1[plainIdx] != vals2[plainIdx] ||
+ nanos1[plainIdx] != nanos2[plainIdx]);
+ };
+ const bool *nulls1 = getNulls();
+ const bool *nulls2 = vect->getNulls();
+ dbcommon::transformDistinct(sz, sel1, nulls1, nulls2, selected, operation);
+ }
+
+ protected:
+ void resizeInternal(size_t plainSize) override {
+ values.resize(plainSize * sizeof(int64_t));
+ nanoseconds.resize(plainSize * sizeof(int64_t));
+ }
+
+ dbcommon::ByteBuffer nanoseconds;
+ mutable Timestamp readBuffer[2];
+ mutable uint8_t nextBuffer = 0;
+ struct TimestampDataInfo {
+ char isNull;
+ char padding[7];
+ Timestamp data;
+ };
+}; // namespace dbcommon
+
+class TimestamptzVector : public TimestampVector {
+ public:
+ explicit TimestamptzVector(bool ownData) : TimestampVector(ownData) {
+ setTypeKind(TIMESTAMPTZID);
+ }
+
+ ~TimestamptzVector() override {}
+
+ void setTimezone(int64_t timezone) { timezoneOffset_ = timezone; }
+
+ int64_t getTimezone() { return timezoneOffset_; }
+
+ std::string readPlain(uint64_t index, bool *null) const override {
+ *null = isNullPlain(index);
+ if (*null) return "NULL";
+ return std::move(TimestamptzType::toString(
+ values.get<int64_t>(index), nanoseconds.get<int64_t>(index)));
+ }
+
+ void append(const std::string &strValue, bool null) override {
+ assert(getOwnData());
+
+ Timestamp value;
+
+ if (!null) {
+ value = TimestamptzType::fromString(strValue);
+ }
+
+ TimestampVector::append(reinterpret_cast<char *>(&value),
+ sizeof(int64_t) + sizeof(int64_t), null);
+ }
+
+ protected:
+ int64_t timezoneOffset_;
+};
+
+} // namespace dbcommon
+
+#endif // DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_TIMESTAMP_VECTOR_H_
diff --git a/depends/dbcommon/src/dbcommon/common/vector/variable-length-vector.cc b/depends/dbcommon/src/dbcommon/common/vector/variable-length-vector.cc
new file mode 100644
index 0000000..de910aa
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector/variable-length-vector.cc
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "dbcommon/common/vector/variable-length-vector.h"
+
+#include "dbcommon/common/vector-transformer.h"
+#include "dbcommon/utils/string-util.h"
+
+namespace dbcommon {
+
+std::string BlankPaddedCharVector::readPlain(uint64_t index, bool *null) const {
+ auto srcLen = getElementWidthPlain(index);
+ auto srcStr = getValPtrPlain(index);
+ *null = isNullPlain(index);
+ if (*null) return std::string(getMaxLenModifier(), ' ');
+ auto ret = newBlankPaddedChar(srcStr, bpCharTrueLen(srcStr, srcLen),
+ getMaxLenModifier());
+ return std::move(ret);
+}
+void BlankPaddedCharVector::trim() {
+ if (isTrimmed) return;
+
+ VariableSizeTypeVectorRawData data(this);
+ auto bpcharTrueLen = [&](uint64_t plainIdx) {
+ const char *val = data.valptrs[plainIdx];
+ uint64_t len = data.lengths[plainIdx];
+#pragma clang loop unroll_count(4)
+ while (len != 0 && val[len - 1] == ' ') len--;
+ data.lengths[plainIdx] = len;
+ };
+ dbcommon::transformVector(getNumOfRowsPlain(), getSelected(), getNulls(),
+ bpcharTrueLen);
+
+ isTrimmed = true;
+}
+
+void BinaryVector::append(const std::string &srcStr, bool null) {
+ auto srcLen = srcStr.size();
+ std::vector<char> binary(srcLen);
+ uint32_t dstLen = 0;
+ for (auto i = 0; i < srcLen;) {
+ if (srcStr[i] != '\\') {
+ binary[dstLen++] = srcStr[i];
+ i += 1;
+ } else if (srcStr[i + 1] == '\\') {
+ binary[dstLen++] = '\\';
+ i += 2;
+ } else {
+ binary[dstLen++] = (srcStr[i + 1] - '0') * 64 +
+ (srcStr[i + 2] - '0') * 8 + (srcStr[i + 3] - '0');
+ i += 4;
+ }
+ }
+ BytesVector::append(&binary[0], dstLen, null);
+}
+
+std::string BinaryVector::readPlain(uint64_t index, bool *null) const {
+ auto srcLen = getElementWidthPlain(index);
+ auto srcBin = getValPtrPlain(index);
+ *null = isNullPlain(index);
+ if (*null)
+ return "";
+ else
+ return StringUtil::toOct(srcBin, srcLen);
+}
+
+} // namespace dbcommon
diff --git a/depends/dbcommon/src/dbcommon/common/vector/variable-length-vector.h b/depends/dbcommon/src/dbcommon/common/vector/variable-length-vector.h
new file mode 100644
index 0000000..e23b39e
--- /dev/null
+++ b/depends/dbcommon/src/dbcommon/common/vector/variable-length-vector.h
@@ -0,0 +1,1173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_VARIABLE_LENGTH_VECTOR_H_
+#define DBCOMMON_SRC_DBCOMMON_COMMON_VECTOR_VARIABLE_LENGTH_VECTOR_H_
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "dbcommon/common/vector-transformer.h"
+#include "dbcommon/common/vector.h"
+
+namespace dbcommon {
+
+template <typename ValueType, typename ADT>
+class VariableSizeTypeVector : public Vector {
+ public:
+ typedef ValueType value_type;
+ typedef ADT abstract_type;
+
+ explicit VariableSizeTypeVector(bool ownData)
+ : Vector(ownData), lengths(ownData), valPtrs(ownData) {
+ if (ownData) {
+ values.reserve(DEFAULT_RESERVED_SIZE_OF_STRING *
+ DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ lengths.reserve(8 * DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ valPtrs.reserve(8 * DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ }
+ }
+
+ void convert(const dbcommon::SupportedEncodingSet inputEncoding,
+ const dbcommon::SupportedEncodingSet outputEncoding) override {
+ assert(inputEncoding != outputEncoding);
+ dbcommon::MbConverter converter(inputEncoding, outputEncoding);
+ std::vector<std::string> tmp0;
+ std::vector<bool> tmp1;
+ size_t rowNum = this->getNumOfRows();
+ bool isNull;
+ // if the vector contains selectlist, the vector behaves similar to
+ // plainize()
+ if (this->allSelected()) {
+ for (size_t i = 0; i < rowNum; i++) {
+ std::string str = this->readPlain(i, &isNull);
+ str = converter.convert(str);
+ tmp0.push_back(str);
+ tmp1.push_back(isNull);
+ }
+ } else {
+ for (size_t i = 0; i < rowNum; i++) {
+ size_t index = (*selected)[i];
+ std::string str = this->readPlain(index, &isNull);
+ str = converter.convert(str);
+ tmp0.push_back(str);
+ tmp1.push_back(isNull);
+ }
+ }
+ this->reset();
+ size_t newRowNum = tmp0.size();
+ for (size_t i = 0; i < newRowNum; i++) {
+ this->append(tmp0[i], tmp1[i]);
+ }
+ }
+
+ virtual ~VariableSizeTypeVector() {}
+
+ uint64_t getNumOfRowsPlain() const override {
+ return lengths.size() / sizeof(uint64_t);
+ }
+
+ void setValue(const char *value, uint64_t size) override {
+ assert(!getOwnData() && value != nullptr && size >= 0);
+ values.setData(value, size);
+ }
+
+ void materialize() override {
+ if (isDirectEncoding && getValue() && !selected) {
+ values.materialize();
+ } else {
+ ByteBuffer valBuf(true);
+ computeValues(getValPtrs(), getLengths(), hasNulls ? getNulls() : nullptr,
+ getNumOfRowsPlain(), selected, &valBuf);
+ values.swap(valBuf);
+ }
+
+ lengths.materialize<uint64_t>(selected);
+
+ if (hasNulls) nulls.materialize(selected);
+
+ selected = nullptr;
+
+ computeValPtrs();
+ }
+
+ std::unique_ptr<Vector> cloneSelected(const SelectList *sel) const override {
+ auto retval = Vector::BuildVector(typeKind, false, typeModifier);
+ auto ret =
+ reinterpret_cast<VariableSizeTypeVector<value_type, abstract_type> *>(
+ retval.get());
+
+ ret->setTypeKind(typeKind);
+ ret->setHasNull(hasNulls);
+ ret->setTypeModifier(typeModifier);
+
+ if (hasNulls) ret->setNulls(getNulls(), getNumOfRowsPlain());
+ if (values.data()) ret->setValue(values.data(), values.size());
+ ret->setLengths(getLengths(), getNumOfRowsPlain());
+ if (valPtrs.data()) ret->setValPtrs(getValPtrs(), getNumOfRowsPlain());
+
+ ret->setSelected(sel, true);
+ ret->isDirectEncoding = this->isDirectEncoding;
+ ret->materialize();
+
+ ret->isDirectEncoding = true;
+
+ assert(ret->isValid());
+ return retval;
+ }
+
+ std::unique_ptr<Vector> extract(const SelectList *sel, uint64_t start,
+ uint64_t end) const override {
+ assert(isValid() && start < end);
+ uint64_t total = getNumOfRowsPlain();
+ assert(end <= total);
+ uint64_t extSz = end - start;
+
+ auto retval = Vector::BuildVector(typeKind, false, typeModifier);
+ auto ret =
+ reinterpret_cast<VariableSizeTypeVector<value_type, abstract_type> *>(
+ retval.get());
+
+ ret->setTypeKind(typeKind);
+ ret->setHasNull(hasNulls);
+ ret->setTypeModifier(typeModifier);
+
+ if (hasNulls) ret->setNulls(getNulls() + start, extSz);
+
+ const char *pStart = getValPtrPlain(start);
+ const char *pEnd = (end != total)
+ ? getValPtrPlain(end)
+ : getValPtrPlain(end - 1) + getLengths()[end - 1];
+ uint64_t valSz = static_cast<uint64_t>(pEnd - pStart);
+ if (values.data()) {
+ ret->setValue(pStart, valSz);
+ }
+
+ ret->setLengths(getLengths() + start, extSz);
+ ret->setValPtrs(getValPtrs() + start, extSz);
+
+ ret->setSelected(sel, true);
+ ret->isDirectEncoding = true;
+
+ assert(ret->isValid());
+ return std::move(retval);
+ }
+
+ std::unique_ptr<Vector> replicateItemToVec(uint64_t index,
+ int replicateNum) const override {
+ assert(isValid());
+
+ std::unique_ptr<Vector> retval(
+ new VariableSizeTypeVector<value_type, abstract_type>(true));
+ auto rt = static_cast<VariableSizeTypeVector<value_type, abstract_type> *>(
+ retval.get());
+
+ rt->setTypeKind(typeKind);
+ rt->setHasNull(hasNulls);
+ rt->setTypeModifier(typeModifier);
+
+ dbcommon::ByteBuffer lengthBuf(true);
+ dbcommon::ByteBuffer valueBuf(true);
+
+ uint64_t idx = getPlainIndex(index);
+ bool isNull = isNullPlain(idx);
+ uint64_t len = getElementWidthPlain(idx);
+ const char *curVal = getValPtrPlain(idx);
+
+ if (hasNulls) {
+ if (isNull) {
+ for (uint64_t i = 0; i < replicateNum; ++i) {
+ lengthBuf.append<uint64_t>(0);
+ rt->nulls.append(isNull);
+ }
+ } else {
+ for (uint64_t i = 0; i < replicateNum; ++i) {
+ lengthBuf.append<uint64_t>(len);
+ valueBuf.append(curVal, len);
+ rt->nulls.append(isNull);
+ }
+ }
+ } else {
+ for (uint64_t i = 0; i < replicateNum; ++i) {
+ lengthBuf.append<uint64_t>(len);
+ valueBuf.append(curVal, len);
+ }
+ }
+
+ rt->lengths.swap(lengthBuf);
+ rt->values.swap(valueBuf);
+ rt->computeValPtrs();
+
+ assert(rt->isValid());
+
+ return std::move(retval);
+ }
+
+ void merge(size_t plainSize, const std::vector<Datum> &data,
+ const std::vector<SelectList *> &sels) override {
+ assert(data.size() == sels.size());
+
+ valPtrs.resize(plainSize * sizeof(value_type));
+ lengths.resize(plainSize * sizeof(uint64_t));
+ nulls.resize(plainSize);
+ const char **__restrict__ valptrs =
+ reinterpret_cast<const char **>(valPtrs.data());
+ uint64_t *__restrict__ lens = reinterpret_cast<uint64_t *>(lengths.data());
+ bool *__restrict__ nulls = this->nulls.data();
+ std::fill_n(nulls, plainSize, true); // skip non-selected data
+
+ for (auto inputIdx = 0; inputIdx < data.size(); inputIdx++) {
+ auto sel = sels[inputIdx];
+ if (sel == nullptr) continue;
+ auto datum = DatumGetValue<Object *>(data[inputIdx]);
+
+ if (auto vec = dynamic_cast<Vector *>(datum)) {
+ auto valPtrsIn = vec->getValPtrs();
+ auto lensIn = vec->getLengths();
+ for (auto idx : *sel) {
+ valptrs[idx] = valPtrsIn[idx];
+ lens[idx] = lensIn[idx];
+ }
+ if (vec->hasNullValue()) {
+ auto nullsIn = vec->getNulls();
+ for (auto idx : *sel) nulls[idx] = nullsIn[idx];
+ } else {
+ for (auto idx : *sel) nulls[idx] = false;
+ }
+ } else {
+ auto scalar = reinterpret_cast<Scalar *>(datum);
+ auto null = scalar->isnull;
+ if (null) {
+ for (auto idx : *sel) {
+ nulls[idx] = true;
+ }
+ } else {
+ auto valptr = DatumGetValue<const char *>(scalar->value);
+ uint64_t len = scalar->length;
+ for (auto idx : *sel) {
+ valptrs[idx] = valptr;
+ lens[idx] = len;
+ nulls[idx] = false;
+ }
+ }
+ }
+ } // End of loop each input data
+
+ values.reset(false);
+ isDirectEncoding = false;
+ materialize();
+ }
+
+ const uint64_t *getLengths() const override {
+ return reinterpret_cast<const uint64_t *>(lengths.data());
+ }
+
+ void setLengths(const uint64_t *lengthsPtr, uint64_t size) override {
+ assert(!getOwnData() && lengthsPtr != nullptr && size >= 0);
+
+ this->lengths.setData(reinterpret_cast<const char *>(lengthsPtr),
+ (size) * sizeof(uint64_t));
+ }
+
+ const char *getValPtrPlain(uint64_t index) const {
+ assert(getNumOfRows() > 0 && index < getNumOfRowsPlain());
+ return valPtrs.get<const char *>(index);
+ }
+
+ uint64_t getElementWidthPlain(uint64_t index) const {
+ assert(index < getNumOfRowsPlain());
+ return lengths.get<uint64_t>(index);
+ }
+
+ const char **getValPtrs() const override {
+ return reinterpret_cast<const char **>(valPtrs.data());
+ }
+
+ void setValPtrs(const char **valPtrs, uint64_t size) override {
+ assert(!getOwnData() && valPtrs != nullptr && size >= 0);
+
+ this->valPtrs.setData(reinterpret_cast<const char *>(valPtrs),
+ (size) * sizeof(char *));
+ }
+
+ void computeValPtrs() override {
+ if (!this->valPtrs.getOwnData()) {
+ this->valPtrs.reset(true);
+ }
+ valPtrs.resize(lengths.size());
+ auto retrievedSize = this->computeValPtrs(
+ values.data(), getLengths(), hasNulls ? getNulls() : nullptr,
+ getNumOfRowsPlain(), (const char **)valPtrs.data(), this->selected);
+ assert(values.size() == retrievedSize ||
+ (typeKind == TypeKind::CHARID && values.size() >= retrievedSize));
+ }
+
+ void checkGroupByKeys(std::vector<const void *> &dataPtrSrc) const override {
+ const char *vals = VariableSizeTypeVector::getValue();
+ const uint64_t *lens = VariableSizeTypeVector::getLengths();
+ const char **valPtrs = VariableSizeTypeVector::getValPtrs();
+
+ const uint64_t size = VariableSizeTypeVector::getNumOfRows();
+ const char **dataPtr = reinterpret_cast<const char **>(&dataPtrSrc[0]);
+
+ if (selected) {
+ SelectList &sel = *selected;
+
+ if (hasNulls) {
+ for (uint64_t i = 0; i < size; ++i) {
+ uint64_t index = sel[i];
+
+ if (dataPtr[i] != nullptr) {
+ // the order of the statement in this block significantly affect the
+ // assembly organization, do not change it easily
+ char null = nulls.getChar(index);
+ uint64_t len = lens[index];
+ const char *d = valPtrs[index];
+
+ const StringDataInfo *info =
+ reinterpret_cast<const StringDataInfo *>(dataPtr[i]);
+
+ bool check = ((null == 0x0) ? (len == info->len &&
+ simpleStrcmp(d, info->data, len))
+ : true) &
+ (null == info->isNull);
+ if (check)
+ dataPtr[i] = alignedAddress(info->data + info->len);
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ } else {
+ for (uint64_t i = 0; i < size; ++i) {
+ uint64_t index = sel[i];
+
+ if (dataPtr[i] != nullptr) {
+ // the order of the statement in this block significantly affect the
+ // assembly organization, do not change it easily
+ uint64_t len = lens[index];
+ const char *d = valPtrs[index];
+
+ const StringDataInfo *info =
+ reinterpret_cast<const StringDataInfo *>(dataPtr[i]);
+
+ bool check = (len == info->len) & (0x0 == info->isNull);
+ if (check && simpleStrcmp(d, info->data, len))
+ dataPtr[i] = alignedAddress(info->data + info->len);
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ }
+ } else { // without select list
+ if (isDirectEncoding && getOwnData()) {
+ if (hasNulls) {
+ for (uint64_t i = 0; i < size; ++i) {
+ const char *d = valPtrs[i];
+ if (dataPtr[i] != nullptr) {
+ char null = nulls.getChar(i);
+ uint64_t len = lens[i];
+
+ const StringDataInfo *info =
+ reinterpret_cast<const StringDataInfo *>(dataPtr[i]);
+
+ bool check = ((null == 0x0) ? (len == info->len &&
+ simpleStrcmp(d, info->data, len))
+ : true) &
+ (null == info->isNull);
+ if (check)
+ dataPtr[i] = alignedAddress(info->data + info->len);
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ } else { // not Null
+ const char *d = values.data();
+ for (uint64_t i = 0; i < size; ++i) {
+ uint64_t len = lens[i];
+ if (dataPtr[i] != nullptr) {
+ const StringDataInfo *info =
+ reinterpret_cast<const StringDataInfo *>(dataPtr[i]);
+
+ bool check = (len == info->len) & (0x0 == info->isNull);
+ if (check && simpleStrcmp(d, info->data, len))
+ dataPtr[i] = alignedAddress(info->data + info->len);
+ else
+ dataPtr[i] = nullptr;
+ }
+ d += len;
+ }
+ }
+ } else { // dictionary encoding
+ for (uint64_t i = 0; i < size; ++i) {
+ if (dataPtr[i] != nullptr) {
+ char null = hasNulls ? nulls.getChar(i) : 0;
+ uint64_t len = lens[i];
+ const char *d = valPtrs[i];
+
+ const StringDataInfo *info =
+ reinterpret_cast<const StringDataInfo *>(dataPtr[i]);
+
+ bool check = ((null == 0x0) ? (len == info->len &&
+ simpleStrcmp(d, info->data, len))
+ : true) &
+ (null == info->isNull);
+ if (check)
+ dataPtr[i] = alignedAddress(info->data + info->len);
+ else
+ dataPtr[i] = nullptr;
+ }
+ }
+ }
+ }
+ }
+ uint64_t checkGroupByKey(uint64_t index, const char *dataPtr) const override {
+ assert(index < getNumOfRows() && "invalid input");
+ assert(getNumOfRows() > 0);
+
+ uint64_t i = getPlainIndex(index);
+
+ const StringDataInfo *info =
+ reinterpret_cast<const StringDataInfo *>(dataPtr);
+ if (isNullPlain(i) != info->isNull) return 0;
+ if (!info->isNull) {
+ if (info->len != VariableSizeTypeVector::getElementWidthPlain(i))
+ return 0;
+ const char *dataPtr0 = VariableSizeTypeVector::getValPtrPlain(i);
+ if (!simpleStrcmp(dataPtr0, info->data, info->len)) return 0;
+ return 9 + info->len;
+ }
+ return 9; // both are null values
+ }
+
+ const char *read(uint64_t index, uint64_t *len, bool *null) const override {
+ assert(index < getNumOfRows() && "invalid input");
+ assert(getNumOfRows() > 0);
+
+ assert(nullptr != null && "invalid input");
+ assert(nullptr != len && "invalid input");
+
+ uint64_t i = getPlainIndex(index);
+
+ *null = isNullPlain(i);
+
+ *len = getElementWidthPlain(i);
+
+ return getValPtrPlain(i);
+ }
+
+ std::string readPlain(uint64_t index, bool *null) const override {
+ *null = isNullPlain(index);
+ if (*null) {
+ return "NULL";
+ } else {
+ uint64_t len = getElementWidthPlain(index);
+ return std::move(
+ ADT::toString(getValPtrPlain(index), getElementWidthPlain(index)));
+ }
+ }
+
+ void readPlainScalar(uint64_t index, Scalar *scalar) const override {
+ scalar->value = CreateDatum(getValPtrPlain(index));
+ scalar->length = getElementWidthPlain(index);
+ scalar->isnull = isNullPlain(index);
+ }
+
+ void append(const char *v, uint64_t valueLen, bool null) override {
+ assert(this->getOwnData());
+
+ char *oldBegin = values.data();
+
+ if (!null && valueLen > 0) {
+ values.append(v, valueLen);
+ } else {
+ valueLen = 0;
+ }
+
+ lengths.append<uint64_t>(valueLen);
+ appendNull(null);
+
+ if (values.data() == oldBegin) { // no realloc happens
+ valPtrs.append<char *>(values.tail() - valueLen);
+ } else {
+ this->computeValPtrs();
+ }
+ }
+
+ void append(Vector *src, uint64_t index) override {
+ assert(this->getOwnData());
+
+ bool isnull = false;
+ uint64_t len = 0;
+ const char *ret = src->read(index, &len, &isnull);
+ append(ret, len, isnull);
+ }
+
+ void append(const std::string &strValue, bool null) override {
+ assert(getOwnData());
+ assert(!null || strValue.size() == 0);
+
+ uint64_t len = 0;
+ const value_type *value = NULL;
+ value = abstract_type::fromString(strValue, &len);
+ append(reinterpret_cast<const char *>(value), len, null);
+ }
+
+ // TODO(lei) this can be optimized since input datum
+ // contains a pointer, do not need to materialize to
+ // buffer
+ void append(const Datum &datum, bool null) override {
+ assert(getOwnData());
+
+ std::string buffer;
+
+ if (!null) {
+ buffer = DatumToBinary(datum, typeKind);
+ }
+
+ append(buffer.data(), buffer.size(), null);
+ }
+
+ void append(const Scalar *scalar) override {
+ append(DatumGetValue<const char *>(scalar->value), scalar->length,
+ scalar->isnull);
+ }
+
+ void append(Vector *vect, SelectList *sel) override {
+ assert(this->getOwnData() && selected == nullptr);
+
+ Vector::append(vect, sel);
+
+ char *oldBegin = values.data();
+ // need append the whole vector
+ uint64_t curOffset = values.size();
+ if (!sel || sel->size() == vect->getNumOfRowsPlain()) {
+ if (vect->getOwnData() &&
+ reinterpret_cast<VariableSizeTypeVector<value_type, abstract_type> *>(
+ vect)
+ ->isDirectEncoding) {
+ values.append(vect->getValueBuffer()->data(),
+ vect->getValueBuffer()->size());
+ lengths.append(reinterpret_cast<const char *>(vect->getLengths()),
+ vect->getNumOfRowsPlain() * sizeof(uint64_t));
+ } else {
+ uint64_t newSize = vect->getNumOfRowsPlain();
+ auto valPtrs = vect->getValPtrs();
+ auto lens = vect->getLengths();
+
+ lengths.append(reinterpret_cast<const char *>(lens),
+ newSize * sizeof(lens[0]));
+ if (vect->hasNullValue()) {
+ auto nulls = vect->getNulls();
+ for (uint64_t i = 0; i < newSize; i++)
+ if (!nulls[i]) values.append(valPtrs[i], lens[i]);
+ } else {
+ for (uint64_t i = 0; i < newSize; i++)
+ values.append(valPtrs[i], lens[i]);
+ }
+ }
+ } else {
+ auto valPtrs = vect->getValPtrs();
+ auto lens = vect->getLengths();
+
+ if (vect->hasNullValue()) {
+ auto nulls = vect->getNulls();
+ for (auto i : *sel) {
+ uint64_t len = lens[i];
+ lengths.append<uint64_t>(len);
+ if (!nulls[i]) values.append(valPtrs[i], len);
+ }
+ } else {
+ for (auto i : *sel) {
+ uint64_t len = lens[i];
+ lengths.append<uint64_t>(len);
+ values.append(valPtrs[i], len);
+ }
+ }
+ }
+
+ this->computeValPtrs();
+ }
+
+ void append(Vector *vect) override {
+ assert(getOwnData());
+
+ append(vect, vect->getSelected());
+ }
+
+ void reset(bool resetBelongingTupleBatch = false) override {
+ Vector::reset(resetBelongingTupleBatch);
+ values.resize(0);
+ lengths.resize(0);
+ valPtrs.resize(0);
+ }
+
+ bool isValid() const override {
+ Vector::isValid();
+
+ uint64_t rows = getNumOfRowsPlain();
+ assert(lengths.size() == sizeof(uint64_t) * rows);
+ assert(valPtrs.size() == sizeof(char *) * rows);
+
+ if (rows == 0) {
+ assert(values.size() == 0);
+ } else {
+ if (this->getOwnData() && values.size() > 0) {
+ uint64_t len = 0;
+ for (uint64_t i = 0; i < rows; i++) {
+ if ((!hasNulls || !nulls.get(i)) && lengths.get<uint64_t>(i) > 0) {
+ assert(valPtrs.get<char *>(i) >= values.data());
+ assert(valPtrs.get<char *>(i) < values.data() + values.size());
+ len += lengths.get<uint64_t>(i);
+ }
+ }
+ assert(typeKind == dbcommon::TypeKind::CHARID || values.size() == len);
+ }
+ }
+
+ for (int i = 0; i < rows; i++) {
+ assert(this->getElementWidthPlain(i) == lengths.get<uint64_t>(i));
+ }
+
+ if (isDirectEncoding && getOwnData() && getSelected() == nullptr &&
+ getNumOfRowsPlain() > 0) {
+ const char *data = values.data();
+ assert(data == getValPtrs()[0]);
+ for (auto i = 1; i < getNumOfRowsPlain(); i++)
+ if (!isNull(i - 1)) {
+ data += getLengths()[i - 1];
+ // fixme(xsheng): impact magma datatype testcases
+ // assert(data == getValPtrs()[i]);
+ }
+ }
+
+ return true;
+ }
+
+ void hash(std::vector<uint64_t> &retval) const override {
+ const uint64_t *lens = reinterpret_cast<const uint64_t *>(lengths.data());
+
+ if (selected) {
+ if (hasNulls) {
+ const char **valPs = reinterpret_cast<const char **>(valPtrs.data());
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = nulls.get(idx) ? 0 : raw_fast_hash(valPs[idx], lens[idx]);
+ }
+ } else {
+ const char **valPs = reinterpret_cast<const char **>(valPtrs.data());
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = raw_fast_hash(valPs[idx], lens[idx]);
+ }
+ }
+ } else {
+ if (isDirectEncoding && !hasNulls && getOwnData()) {
+ const char **valPs = reinterpret_cast<const char **>(valPtrs.data());
+ const char *valP = valPs[0];
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = raw_fast_hash(valP, lens[i]);
+ valP += lens[i];
+ }
+ } else {
+ if (hasNulls) {
+ const char **valPs = reinterpret_cast<const char **>(valPtrs.data());
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = nulls.get(i) ? 0 : raw_fast_hash(valPs[i], lens[i]);
+ }
+ } else {
+ const char **valPs = reinterpret_cast<const char **>(valPtrs.data());
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = raw_fast_hash(valPs[i], lens[i]);
+ }
+ }
+ }
+ }
+ }
+
+ void cdbHash(std::vector<uint64_t> &retval) const override {
+ const uint64_t *lens = reinterpret_cast<const uint64_t *>(lengths.data());
+
+ if (selected) {
+ if (hasNulls) {
+ const char **valPs = reinterpret_cast<const char **>(valPtrs.data());
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = nulls.get(idx)
+ ? cdbhashnull(retval[i])
+ : cdbhash(valPs[idx], lens[idx], retval[i]);
+ }
+ } else {
+ const char **valPs = reinterpret_cast<const char **>(valPtrs.data());
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ uint64_t idx = (*selected)[i];
+ retval[i] = cdbhash(valPs[idx], lens[idx], retval[i]);
+ }
+ }
+ } else {
+ if (isDirectEncoding && !hasNulls && getOwnData()) {
+ const char **valPs = reinterpret_cast<const char **>(valPtrs.data());
+ const char *valP = valPs[0];
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = cdbhash(valP, lens[i], retval[i]);
+ valP += lens[i];
+ }
+ } else {
+ if (hasNulls) {
+ const char **valPs = reinterpret_cast<const char **>(valPtrs.data());
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = nulls.get(i) ? cdbhashnull(retval[i])
+ : cdbhash(valPs[i], lens[i], retval[i]);
+ }
+ } else {
+ const char **valPs = reinterpret_cast<const char **>(valPtrs.data());
+ for (uint64_t i = 0; i < retval.size(); ++i) {
+ retval[i] = cdbhash(valPs[i], lens[i], retval[i]);
+ }
+ }
+ }
+ }
+ }
+
+ dbcommon::ByteBuffer *getValueBuffer() override { return &values; }
+ dbcommon::ByteBuffer *getLengthsBuffer() { return &lengths; }
+
+ void swap(Vector &vect) override {
+ Vector::swap(vect);
+ auto &v =
+ dynamic_cast<VariableSizeTypeVector<value_type, abstract_type> &>(vect);
+ this->values.swap(v.values);
+ this->lengths.swap(v.lengths);
+ this->valPtrs.swap(v.valPtrs);
+ }
+
+ uint64_t getSerializeSizeInternal() const override {
+ uint64_t sSize = Vector::getSerializeSizeInternal();
+ sSize += sizeof(uint64_t) + values.size();
+ sSize += sizeof(uint64_t) + lengths.size();
+ return sSize;
+ }
+
+ void serializeInternal(NodeSerializer *serializer) override {
+ // null data
+ Vector::serializeInternal(serializer);
+
+ // value data
+ uint64_t vsz = 0;
+ auto lens = getLengths();
+ auto valPtrs = getValPtrs();
+ if (hasNulls) { // has nulls
+ const bool *nulls = getNullBuffer()->getBools();
+ if (allSelected()) { // no select list
+ uint64_t size = getNumOfRowsPlain();
+ for (auto idx = 0; idx < size; ++idx) {
+ if (!nulls[idx]) vsz += lens[idx];
+ }
+ serializer->write<uint64_t>(vsz);
+ for (uint64_t idx = 0; idx < size; ++idx) {
+ if (!nulls[idx]) serializer->writeBytes(valPtrs[idx], lens[idx]);
+ }
+ } else { // has select list
+ for (auto idx : *selected) {
+ if (!nulls[idx]) vsz += lens[idx];
+ }
+ serializer->write<uint64_t>(vsz);
+ for (auto idx : *selected) {
+ if (!nulls[idx]) serializer->writeBytes(valPtrs[idx], lens[idx]);
+ }
+ }
+ } else { // no nulls
+ if (allSelected()) { // no select list
+ uint64_t size = getNumOfRowsPlain();
+ for (auto idx = 0; idx < size; ++idx) vsz += lens[idx];
+ serializer->write<uint64_t>(vsz);
+ for (uint64_t idx = 0; idx < size; ++idx) {
+ serializer->writeBytes(valPtrs[idx], lens[idx]);
+ }
+ } else { // has select list
+ for (auto idx : *selected) vsz += lens[idx];
+ serializer->write<uint64_t>(vsz);
+ for (auto idx : *selected) {
+ serializer->writeBytes(valPtrs[idx], lens[idx]);
+ }
+ }
+ }
+
+ // length data
+ if (allSelected()) {
+ uint64_t lsz = lengths.size();
+ serializer->write<uint64_t>(lsz);
+ serializer->writeBytes(lengths.data(), lsz);
+ } else {
+ uint64_t lsz = selected->size() * sizeof(uint64_t);
+ auto lens = getLengths();
+ serializer->write<uint64_t>(lsz);
+ for (auto idx : *selected) serializer->write(lens[idx]);
+ }
+ }
+
+ void deserializeInternal(NodeDeserializer *deserializer) override {
+ Vector::deserializeInternal(deserializer);
+
+ uint64_t vsz = deserializer->read<uint64_t>();
+ const char *dstr = deserializer->readBytes(vsz);
+ setValue(dstr, vsz);
+
+ uint64_t lsz = deserializer->read<uint64_t>();
+ if (lsz > 0) {
+ const char *ostr = deserializer->readBytes(lsz);
+ setLengths((const uint64_t *)ostr, lsz / sizeof(uint64_t));
+ }
+
+ this->isDirectEncoding = true;
+ computeValPtrs();
+
+ assert(isValid());
+ }
+
+ void selectDistinct(SelectList *selected, Scalar *scalar) override {
+ SelectList *sel = getSelected();
+ uint64_t sz = getNumOfRows();
+ const char **valptrs = this->VariableSizeTypeVector::getValPtrs();
+ const uint64_t *lens = this->getLengths();
+
+ auto operation = [&](uint64_t plainIdx) { // NOLINT
+ if (!scalar->isnull) {
+ char *value = DatumGetValue<char *>(scalar->value);
+ auto len = scalar->length;
+ return (lens[plainIdx] != len ||
+ !simpleStrcmp(valptrs[plainIdx], value, lens[plainIdx]));
+ }
+ return false;
+ };
+ const bool *nulls = getNulls();
+ dbcommon::transformDistinct(sz, sel, nulls, selected, scalar->isnull,
+ operation);
+ }
+
+ void selectDistinct(SelectList *selected, Vector *vect) override {
+ SelectList *sel1 = getSelected();
+ SelectList *sel2 = vect->getSelected();
+ uint64_t sz = getNumOfRows();
+ assert(sz == vect->getNumOfRows() && "invalid input");
+
+ const char **valPtrs1 = this->VariableSizeTypeVector::getValPtrs();
+ const uint64_t *lengths1 = this->getLengths();
+ const char **valPtrs2 = vect->getValPtrs();
+ const uint64_t *lengths2 = vect->getLengths();
+
+ auto operation = [&](uint64_t plainIdx) { // NOLINT
+ return (lengths1[plainIdx] != lengths2[plainIdx] ||
+ !simpleStrcmp(valPtrs1[plainIdx], valPtrs2[plainIdx],
+ lengths1[plainIdx]));
+ };
+ const bool *nulls1 = getNulls();
+ const bool *nulls2 = vect->getNulls();
+ dbcommon::transformDistinct(sz, sel1, nulls1, nulls2, selected, operation);
+ }
+
+ void setDirectEncoding(bool isDirectEncoding) override {
+ this->isDirectEncoding = isDirectEncoding;
+ }
+ bool getIsDirectEncoding() { return isDirectEncoding; }
+
+ void setCloneInMaterialize(bool cloneInMaterialize) {
+ this->cloneInMaterialize = cloneInMaterialize;
+ }
+ bool getCloneInMaterialize() { return cloneInMaterialize; }
+
+ protected:
+ void resizeInternal(size_t plainSize) override {
+ values.resize(0);
+ valPtrs.resize(plainSize * sizeof(uint64_t));
+ lengths.resize(plainSize * sizeof(uint64_t));
+ ::memset(lengths.data(), 0, plainSize * sizeof(uint64_t));
+ }
+
+ static force_inline bool simpleStrcmp(const char *s1, const char *s2,
+ uint64_t len) {
+ uint64_t idx = 0;
+ while (idx + 8 <= len) {
+ if (*reinterpret_cast<const uint64_t *>(s1 + idx) !=
+ *reinterpret_cast<const uint64_t *>(s2 + idx))
+ return false;
+ idx += 8;
+ }
+ if (idx + 4 <= len) {
+ if (*reinterpret_cast<const uint32_t *>(s1 + idx) !=
+ *reinterpret_cast<const uint32_t *>(s2 + idx))
+ return false;
+ idx += 4;
+ }
+ if (idx + 2 <= len) {
+ if (*reinterpret_cast<const uint16_t *>(s1 + idx) !=
+ *reinterpret_cast<const uint16_t *>(s2 + idx))
+ return false;
+ idx += 2;
+ }
+ if (idx < len) {
+ if (*reinterpret_cast<const uint8_t *>(s1 + idx) !=
+ *reinterpret_cast<const uint8_t *>(s2 + idx))
+ return false;
+ }
+ return true;
+ }
+
+ // Compute the value pointer array according to input length array.
+ // Each value locate continuously inside the data source.
+ // @param[in] vals Source of values
+ // @param[in] lens Length array
+ // @param[in] size Size of input values
+ // @param[out] valPtrs Output value pointer array
+ // @return Size of retrieved bytes from vals
+ static uint64_t computeValPtrs(const char *vals, const uint64_t *lens,
+ const bool *nulls, uint64_t size,
+ const char **valPtrs,
+ const SelectList *sel = nullptr) {
+ const char *ptr = vals;
+
+ if (sel) {
+ if (nulls) {
+ for (auto i : *sel) {
+ valPtrs[i] = ptr;
+ ptr += nulls[i] ? 0 : lens[i];
+ }
+ } else {
+ for (auto i : *sel) {
+ valPtrs[i] = ptr;
+ ptr += lens[i];
+ }
+ }
+ } else {
+ if (nulls) {
+ for (uint64_t i = 0; i < size; i++) {
+ valPtrs[i] = ptr;
+ ptr += nulls[i] ? 0 : lens[i];
+ }
+ } else {
+ for (uint64_t i = 0; i < size; i++) {
+ valPtrs[i] = ptr;
+ ptr += lens[i];
+ }
+ }
+ }
+
+ return ptr - vals;
+ }
+
+ // Compute the value array
+ // @param[in] valPtrs
+ // @param[in] lens
+ // @param[in] nulls
+ // @param[in] size
+ // @param[in] sel
+ // @param[out] valBuf
+ static void computeValues(const char **valPtrs, const uint64_t *lens,
+ const bool *nulls, uint64_t size,
+ const SelectList *sel,
+ dbcommon::ByteBuffer *valBuf) {
+ assert(valBuf->getOwnData());
+
+ if (sel) {
+ if (nulls) {
+ for (auto plainIdx : *sel) {
+ if (!nulls[plainIdx])
+ valBuf->append(valPtrs[plainIdx], lens[plainIdx]);
+ }
+ } else {
+ for (auto plainIdx : *sel) {
+ valBuf->append(valPtrs[plainIdx], lens[plainIdx]);
+ }
+ }
+ } else {
+ if (nulls) {
+ for (auto plainIdx = 0; plainIdx < size; plainIdx++) {
+ if (!nulls[plainIdx])
+ valBuf->append(valPtrs[plainIdx], lens[plainIdx]);
+ }
+ } else {
+ for (auto plainIdx = 0; plainIdx < size; plainIdx++) {
+ valBuf->append(valPtrs[plainIdx], lens[plainIdx]);
+ }
+ }
+ }
+ }
+
+ bool isDirectEncoding = false; // false for dict encoding
+ bool cloneInMaterialize =
+ false; // even with no selected it also clone when true
+ dbcommon::ByteBuffer lengths; // type: uint64_t
+ dbcommon::ByteBuffer valPtrs; // type: char *
+ struct StringDataInfo {
+ uint64_t len;
+ char isNull;
+ char data[7]; // serve as a virtual data pointer
+ };
+};
+
+class BytesVector : public VariableSizeTypeVector<char, BytesType> {
+ public:
+ typedef VariableSizeTypeVector<char, BytesType> base_type;
+
+ std::string readPlain(uint64_t index, bool *null) const override {
+ *null = isNullPlain(index);
+ if (*null) return "NULL";
+ return std::move(StringType::toString(getValPtrPlain(index),
+ getElementWidthPlain(index)));
+ }
+
+ const char *read(uint64_t index, uint64_t *len, bool *null) const override {
+ return base_type::read(index, len, null);
+ }
+
+ uint64_t getMaxLenModifier() const {
+ return TypeModifierUtil::getMaxLen(typeModifier);
+ }
+
+ protected:
+ explicit BytesVector(bool ownData)
+ : VariableSizeTypeVector<char, BytesType>(ownData) {}
+};
+
+class BlankPaddedCharVector : public BytesVector {
+ public:
+ explicit BlankPaddedCharVector(bool ownData, int64_t typeMod)
+ : BytesVector(ownData) {
+ setTypeKind(CHARID);
+ typeModifier = typeMod;
+ if (ownData)
+ values.reserve(TypeModifierUtil::getMaxLen(typeModifier) *
+ DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ }
+ std::string readPlain(uint64_t index, bool *null) const override;
+ void trim() override;
+ void setMaxLenModifier(uint64_t maxLenModifier) {
+ typeModifier =
+ TypeModifierUtil::getTypeModifierFromMaxLength(maxLenModifier);
+ }
+
+ private:
+ bool isTrimmed = false;
+};
+
+class VaryingCharVector : public BytesVector {
+ public:
+ explicit VaryingCharVector(bool ownData, int64_t typeMod)
+ : BytesVector(ownData) {
+ setTypeKind(VARCHARID);
+ typeModifier = typeMod;
+ if (ownData)
+ values.reserve(TypeModifierUtil::getMaxLen(typeModifier) *
+ DEFAULT_NUMBER_TUPLES_PER_BATCH);
+ }
+};
+
+class StringVector : public BytesVector {
+ public:
+ explicit StringVector(bool ownData) : BytesVector(ownData) {
+ setTypeKind(STRINGID);
+ }
+};
+
+class BinaryVector : public BytesVector {
+ public:
+ explicit BinaryVector(bool ownData) : BytesVector(ownData) {
+ setTypeKind(BINARYID);
+ }
+ void append(const std::string &strValue, bool null) override;
... 52146 lines suppressed ...