You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ja...@apache.org on 2024/06/17 08:41:15 UTC
(tsfile) 10/20: refine code dir and add exmaples, benchmark etc.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit bd99e1619c47a743ea5b2222732dd23d07be4d4e
Author: colin <sh...@163.com>
AuthorDate: Thu May 23 15:45:04 2024 +0800
refine code dir and add exmaples, benchmark etc.
---
.github/workflows/unit-test.yml | 12 +-
cpp/.clang-format | 226 ++
cpp/.gitignore | 7 +
cpp/CMakeLists.txt | 129 +-
cpp/{tsfile => }/README-zh.md | 0
cpp/README.md | 79 +
cpp/bench_mark/CMakeLists.txt | 33 +
cpp/bench_mark/bench_mark_src/CMakeLists.txt | 57 +
.../bench_mark_src/bench_conf.h} | 19 +-
cpp/bench_mark/bench_mark_src/bench_mark.cc | 161 +
.../bench_mark_src/bench_mark.h} | 10 +-
cpp/bench_mark/build.sh | 50 +
cpp/build-utils/source-bundle-descriptor.xml | 36 -
cpp/build.sh | 142 +
cpp/cmake/build_function.cmake | 34 +
.../test/common_test.cpp => cmake/cmake_config.h} | 11 +-
cpp/cmake/config.h.in | 1 +
cpp/common/CMakeLists.txt | 48 -
cpp/examples/CMakeLists.txt | 68 +-
cpp/examples/build.sh | 32 +
cpp/examples/cpp_examples/CMakeLists.txt | 22 +
.../cpp_examples/cpp_examples.h} | 26 +-
cpp/examples/cpp_examples/demo_read.cpp | 101 +
cpp/examples/cpp_examples/demo_write.cpp | 55 +
.../include/common.hpp => examples/examples.cc} | 21 +-
cpp/examples/src/demo_read.cpp | 92 -
cpp/pom.xml | 94 +-
cpp/src/CMakeLists.txt | 55 +
cpp/src/common/CMakeLists.txt | 35 +
cpp/src/common/allocator/CMakeLists.txt | 26 +
cpp/src/common/allocator/alloc_base.h | 131 +
cpp/src/common/allocator/byte_stream.h | 1109 +++++++
cpp/src/common/allocator/mem_alloc.cc | 253 ++
cpp/src/common/allocator/my_string.h | 144 +
cpp/src/common/allocator/object_pool.h | 110 +
cpp/src/common/allocator/page_arena.cc | 65 +
cpp/src/common/allocator/page_arena.h | 101 +
cpp/src/common/allocator/stl_allocator.h | 90 +
.../src/common/allocator/util_define.h | 84 +-
cpp/src/common/config/CMakeLists.txt | 20 +
cpp/src/common/config/config.h | 70 +
cpp/src/common/container/CMakeLists.txt | 22 +
cpp/src/common/container/array.h | 305 ++
cpp/src/common/container/bit_map.cc | 45 +
cpp/src/common/container/bit_map.h | 83 +
cpp/src/common/container/blocking_queue.cc | 57 +
.../common/container/blocking_queue.h} | 34 +-
cpp/src/common/container/byte_buffer.h | 127 +
cpp/src/common/container/hash_func.h | 106 +
cpp/src/common/container/hash_node.h | 73 +
cpp/src/common/container/hash_segm.h | 53 +
cpp/src/common/container/hash_table.h | 792 +++++
cpp/src/common/container/list.h | 149 +
cpp/src/common/container/murmur_hash3.cc | 138 +
cpp/src/common/container/murmur_hash3.h | 75 +
cpp/src/common/container/simple_vector.h | 58 +
cpp/src/common/container/slice.h | 148 +
cpp/src/common/container/sorted_array.h | 329 ++
cpp/src/common/datatype/CMakeLists.txt | 18 +
cpp/src/common/datatype/value.h | 223 ++
cpp/src/common/db_common.h | 183 ++
cpp/src/common/global.cc | 189 ++
cpp/src/common/global.h | 48 +
.../src/example.cpp => src/common/logger/elog.h} | 30 +-
cpp/src/common/mutex/CMakeLists.txt | 20 +
cpp/src/common/mutex/mutex.h | 74 +
cpp/src/common/path.h | 50 +
cpp/src/common/record.h | 138 +
cpp/src/common/row_record.h | 193 ++
cpp/src/common/schema.h | 73 +
cpp/src/common/seq_tvlist.h | 172 ++
.../src/tsfile => src/common}/seq_tvlist.inc | 25 +-
cpp/src/common/statistic.h | 839 ++++++
cpp/src/common/tablet.cc | 164 +
cpp/src/common/tablet.h | 100 +
cpp/src/common/tablet_iterator.h | 78 +
cpp/src/common/tsblock/CMakeLists.txt | 22 +
cpp/src/common/tsblock/tsblock.cc | 310 ++
cpp/src/common/tsblock/tsblock.h | 272 ++
cpp/src/common/tsblock/tuple_desc.cc | 88 +
cpp/src/common/tsblock/tuple_desc.h | 118 +
.../common/tsblock/vector/fixed_length_vector.h | 82 +
.../common/tsblock/vector/variable_length_vector.h | 87 +
cpp/src/common/tsblock/vector/vector.h | 94 +
cpp/src/common/tsfile_common.cc | 259 ++
cpp/src/common/tsfile_common.h | 823 +++++
cpp/src/common/tsfile_mgr.cc | 139 +
cpp/src/common/tsfile_mgr.h | 95 +
cpp/src/compress/CMakeLists.txt | 23 +
.../example.hpp => src/compress/compressor.c} | 10 +-
cpp/src/compress/compressor.h | 51 +
cpp/src/compress/compressor_factory.h | 71 +
cpp/src/compress/gzip_compressor.cc | 292 ++
cpp/src/compress/gzip_compressor.h | 153 +
cpp/src/compress/lz4.c | 3162 ++++++++++++++++++++
cpp/src/compress/lz4.h | 1000 +++++++
cpp/src/compress/lz4_compressor.cc | 141 +
cpp/src/compress/lz4_compressor.h | 66 +
cpp/src/compress/uncompressed_compressor.h | 54 +
cpp/src/encoding/CMakeLists.txt | 32 +
cpp/src/encoding/bitpack_decoder.h | 175 ++
cpp/src/encoding/bitpack_encoder.h | 173 ++
cpp/src/encoding/decoder.h | 41 +
cpp/src/encoding/decoder_factory.h | 97 +
cpp/src/encoding/dictionary_decoder.h | 78 +
cpp/src/encoding/dictionary_encoder.h | 101 +
cpp/src/encoding/encode_utils.h | 146 +
cpp/src/encoding/encoder.h | 44 +
cpp/src/encoding/encoder_factory.h | 113 +
cpp/src/encoding/gorilla_decoder.h | 411 +++
cpp/src/encoding/gorilla_encoder.h | 468 +++
cpp/src/encoding/intpacker.h | 155 +
cpp/src/encoding/plain_decoder.h | 52 +
cpp/src/encoding/plain_encoder.h | 63 +
cpp/src/encoding/ts2diff_decoder.h | 311 ++
cpp/src/encoding/ts2diff_encoder.h | 335 +++
cpp/src/encoding/zigzag_decoder.h | 205 ++
cpp/src/encoding/zigzag_encoder.h | 172 ++
cpp/src/file/CMakeLists.txt | 24 +
cpp/src/file/open_file.cc | 110 +
cpp/src/file/open_file.h | 137 +
cpp/src/file/read_file.cc | 126 +
cpp/src/file/read_file.h | 66 +
cpp/src/file/tsfile_io_reader.cc | 535 ++++
cpp/src/file/tsfile_io_reader.h | 169 ++
cpp/src/file/tsfile_io_writer.cc | 757 +++++
cpp/src/file/tsfile_io_writer.h | 191 ++
cpp/src/file/write_file.cc | 128 +
cpp/src/file/write_file.h | 61 +
cpp/src/reader/CMakeLists.txt | 23 +
cpp/src/reader/bloom_filter.cc | 268 ++
cpp/src/reader/bloom_filter.h | 128 +
cpp/src/reader/chunk_reader.cc | 454 +++
cpp/src/reader/chunk_reader.h | 137 +
cpp/src/reader/expression.cc | 207 ++
cpp/src/reader/expression.h | 133 +
cpp/src/reader/filter/and_filter.h | 87 +
cpp/src/reader/filter/between.h | 135 +
cpp/src/reader/filter/binary_filter.h | 45 +
cpp/src/reader/filter/eq.h | 71 +
cpp/src/reader/filter/filter.h | 60 +
cpp/src/reader/filter/filter_type.h | 51 +
cpp/src/reader/filter/gt.h | 69 +
cpp/src/reader/filter/gt_eq.h | 70 +
cpp/src/reader/filter/in.h | 55 +
cpp/src/reader/filter/lt.h | 69 +
cpp/src/reader/filter/lt_eq.h | 70 +
cpp/src/reader/filter/not_eq.h | 71 +
cpp/src/reader/filter/object.h | 84 +
cpp/src/reader/filter/or_filter.h | 89 +
cpp/src/reader/filter/time_filter.cc | 46 +
cpp/src/reader/filter/time_filter.h | 53 +
cpp/src/reader/filter/time_operator.cc | 274 ++
cpp/src/reader/filter/time_operator.h | 213 ++
cpp/src/reader/filter/unary_filter.h | 66 +
cpp/src/reader/qds_with_timegenerator.cc | 384 +++
cpp/src/reader/qds_with_timegenerator.h | 136 +
cpp/src/reader/qds_without_timegenerator.cc | 159 +
cpp/src/reader/qds_without_timegenerator.h | 65 +
.../tsfile.hpp => src/reader/query_data_set.h} | 26 +-
cpp/src/reader/query_executor.h | 63 +
cpp/src/reader/scan_iterator.cc | 257 ++
cpp/src/reader/scan_iterator.h | 189 ++
cpp/src/reader/tsfile_executor.cc | 114 +
cpp/src/reader/tsfile_executor.h | 57 +
cpp/src/reader/tsfile_reader.cc | 67 +
cpp/src/reader/tsfile_reader.h | 55 +
cpp/src/reader/tsfile_series_scan_iterator.cc | 111 +
cpp/src/reader/tsfile_series_scan_iterator.h | 98 +
cpp/src/utils/CMakeLists.txt | 19 +
cpp/src/utils/db_utils.h | 417 +++
.../src/common => src/utils}/errno_define.h | 34 +-
cpp/src/utils/injection.h | 68 +
cpp/src/utils/storage_utils.h | 86 +
cpp/src/utils/util_define.h | 145 +
cpp/src/writer/CMakeLists.txt | 24 +
cpp/src/writer/chunk_writer.cc | 154 +
cpp/src/writer/chunk_writer.h | 136 +
cpp/src/writer/page_writer.cc | 210 ++
cpp/src/writer/page_writer.h | 178 ++
cpp/src/writer/tsfile_writer.cc | 429 +++
cpp/src/writer/tsfile_writer.h | 121 +
cpp/test_all.sh | 18 +
cpp/tsfile/CMakeLists.txt | 49 -
cpp/tsfile/README.md | 34 -
cpp/tsfile/src/CMakeLists.txt | 4 -
cpp/tsfile/src/common/CMakeLists.txt | 20 -
cpp/tsfile/src/common/allocator/CMakeLists.txt.bak | 16 -
cpp/tsfile/src/common/allocator/alloc_base.h | 128 -
cpp/tsfile/src/common/allocator/byte_stream.h | 1111 -------
cpp/tsfile/src/common/allocator/mem_alloc.cc | 236 --
cpp/tsfile/src/common/allocator/my_string.h | 143 -
cpp/tsfile/src/common/allocator/object_pool.h | 104 -
cpp/tsfile/src/common/allocator/page_arena.cc | 52 -
cpp/tsfile/src/common/allocator/page_arena.h | 90 -
cpp/tsfile/src/common/allocator/stl_allocator.h | 68 -
cpp/tsfile/src/common/config/CMakeLists.txt | 2 -
cpp/tsfile/src/common/config/config.h | 52 -
cpp/tsfile/src/common/container/CMakeLists.txt | 12 -
cpp/tsfile/src/common/container/array.h | 312 --
cpp/tsfile/src/common/container/bit_map.cc | 32 -
cpp/tsfile/src/common/container/bit_map.h | 71 -
cpp/tsfile/src/common/container/blocking_queue.cc | 47 -
cpp/tsfile/src/common/container/blocking_queue.h | 32 -
cpp/tsfile/src/common/container/byte_buffer.h | 114 -
cpp/tsfile/src/common/container/hash_func.h | 102 -
cpp/tsfile/src/common/container/hash_node.h | 90 -
cpp/tsfile/src/common/container/hash_segm.h | 41 -
cpp/tsfile/src/common/container/hash_table.h | 685 -----
cpp/tsfile/src/common/container/list.h | 147 -
cpp/tsfile/src/common/container/murmur_hash3.cc | 126 -
cpp/tsfile/src/common/container/murmur_hash3.h | 67 -
cpp/tsfile/src/common/container/simple_vector.h | 48 -
cpp/tsfile/src/common/container/slice.h | 142 -
cpp/tsfile/src/common/container/sorted_array.h | 333 ---
cpp/tsfile/src/common/datatype/CMakeLists.txt | 13 -
cpp/tsfile/src/common/datatype/value.h | 216 --
cpp/tsfile/src/common/db_common.h | 168 --
cpp/tsfile/src/common/db_utils.h | 457 ---
cpp/tsfile/src/common/global.cc | 201 --
cpp/tsfile/src/common/global.h | 28 -
cpp/tsfile/src/common/injection.h | 53 -
cpp/tsfile/src/common/mutex/CMakeLists.txt | 2 -
cpp/tsfile/src/common/mutex/mutex.h | 64 -
cpp/tsfile/src/common/tsblock/CMakeLists.txt | 11 -
cpp/tsfile/src/common/tsblock/tsblock.cc | 293 --
cpp/tsfile/src/common/tsblock/tsblock.h | 291 --
cpp/tsfile/src/common/tsblock/tuple_desc.cc | 76 -
cpp/tsfile/src/common/tsblock/tuple_desc.h | 116 -
.../common/tsblock/vector/fixed_length_vector.h | 76 -
.../common/tsblock/vector/variable_length_vector.h | 77 -
cpp/tsfile/src/common/tsblock/vector/vector.h | 75 -
cpp/tsfile/src/common/util_define.h | 133 -
cpp/tsfile/src/tsfile/CMakeLists.txt | 118 -
cpp/tsfile/src/tsfile/Makefile | 1281 --------
cpp/tsfile/src/tsfile/bloom_filter.cc | 254 --
cpp/tsfile/src/tsfile/bloom_filter.h | 120 -
cpp/tsfile/src/tsfile/chunk_reader | 0
cpp/tsfile/src/tsfile/chunk_reader.cc | 425 ---
cpp/tsfile/src/tsfile/chunk_reader.h | 128 -
cpp/tsfile/src/tsfile/chunk_writer.cc | 140 -
cpp/tsfile/src/tsfile/chunk_writer.h | 116 -
cpp/tsfile/src/tsfile/compress/CMakeLists.txt | 14 -
cpp/tsfile/src/tsfile/compress/Makefile | 285 --
cpp/tsfile/src/tsfile/compress/compressor.c | 0
cpp/tsfile/src/tsfile/compress/compressor.h | 42 -
.../src/tsfile/compress/compressor_factory.h | 62 -
cpp/tsfile/src/tsfile/compress/gzip_compressor.cc | 271 --
cpp/tsfile/src/tsfile/compress/gzip_compressor.h | 137 -
cpp/tsfile/src/tsfile/compress/lz4.c | 2751 -----------------
cpp/tsfile/src/tsfile/compress/lz4.h | 861 ------
cpp/tsfile/src/tsfile/compress/lz4_compressor.cc | 132 -
cpp/tsfile/src/tsfile/compress/lz4_compressor.h | 59 -
.../src/tsfile/compress/uncompressed_compressor.h | 56 -
.../tsfile/data_dir/smoke_test.write_record.dat | Bin 1273 -> 0 bytes
cpp/tsfile/src/tsfile/encode/CMakeLists.txt | 14 -
cpp/tsfile/src/tsfile/encode/Makefile | 189 --
cpp/tsfile/src/tsfile/encode/bitpack_decoder.h | 164 -
cpp/tsfile/src/tsfile/encode/bitpack_encoder.h | 170 --
cpp/tsfile/src/tsfile/encode/decoder.h | 30 -
cpp/tsfile/src/tsfile/encode/decoder_factory.h | 89 -
cpp/tsfile/src/tsfile/encode/dictionary_decoder.h | 69 -
cpp/tsfile/src/tsfile/encode/dictionary_encoder.h | 96 -
cpp/tsfile/src/tsfile/encode/encode_utils.h | 138 -
cpp/tsfile/src/tsfile/encode/encoder.h | 33 -
cpp/tsfile/src/tsfile/encode/encoder_factory.h | 105 -
cpp/tsfile/src/tsfile/encode/gorilla_decoder.h | 311 --
cpp/tsfile/src/tsfile/encode/gorilla_encoder.h | 388 ---
cpp/tsfile/src/tsfile/encode/intpacker.h | 144 -
cpp/tsfile/src/tsfile/encode/plain_decoder.h | 45 -
cpp/tsfile/src/tsfile/encode/plain_encoder.h | 56 -
cpp/tsfile/src/tsfile/encode/ts2diff_decoder.h | 214 --
cpp/tsfile/src/tsfile/encode/ts2diff_encoder.h | 273 --
cpp/tsfile/src/tsfile/encode/zigzag_decoder.h | 206 --
cpp/tsfile/src/tsfile/encode/zigzag_encoder.h | 172 --
cpp/tsfile/src/tsfile/expression.cc | 180 --
cpp/tsfile/src/tsfile/expression.h | 132 -
cpp/tsfile/src/tsfile/filter/and_filter.h | 79 -
cpp/tsfile/src/tsfile/filter/between.h | 107 -
cpp/tsfile/src/tsfile/filter/binary_filter.h | 31 -
cpp/tsfile/src/tsfile/filter/eq.h | 63 -
cpp/tsfile/src/tsfile/filter/filter.h | 32 -
cpp/tsfile/src/tsfile/filter/filter_type.h | 42 -
cpp/tsfile/src/tsfile/filter/gt.h | 59 -
cpp/tsfile/src/tsfile/filter/gt_eq.h | 60 -
cpp/tsfile/src/tsfile/filter/in.h | 42 -
cpp/tsfile/src/tsfile/filter/lt.h | 59 -
cpp/tsfile/src/tsfile/filter/lt_eq.h | 60 -
cpp/tsfile/src/tsfile/filter/not_eq.h | 60 -
cpp/tsfile/src/tsfile/filter/object.h | 78 -
cpp/tsfile/src/tsfile/filter/or_filter.h | 81 -
cpp/tsfile/src/tsfile/filter/time_filter.cc | 30 -
cpp/tsfile/src/tsfile/filter/time_filter.h | 39 -
cpp/tsfile/src/tsfile/filter/time_operator.cc | 295 --
cpp/tsfile/src/tsfile/filter/time_operator.h | 207 --
cpp/tsfile/src/tsfile/filter/unary_filter.h | 42 -
cpp/tsfile/src/tsfile/open_file.cc | 99 -
cpp/tsfile/src/tsfile/open_file.h | 123 -
cpp/tsfile/src/tsfile/page_writer.cc | 191 --
cpp/tsfile/src/tsfile/page_writer.h | 156 -
cpp/tsfile/src/tsfile/path.h | 37 -
cpp/tsfile/src/tsfile/qds_with_timegenerator.cc | 377 ---
cpp/tsfile/src/tsfile/qds_with_timegenerator.h | 127 -
cpp/tsfile/src/tsfile/qds_without_timegenerator.cc | 149 -
cpp/tsfile/src/tsfile/qds_without_timegenerator.h | 52 -
cpp/tsfile/src/tsfile/query_data_set.h | 23 -
cpp/tsfile/src/tsfile/query_executor.h | 50 -
cpp/tsfile/src/tsfile/read_file.cc | 106 -
cpp/tsfile/src/tsfile/read_file.h | 51 -
cpp/tsfile/src/tsfile/record.h | 144 -
cpp/tsfile/src/tsfile/row_record.h | 194 --
cpp/tsfile/src/tsfile/scan_iterator.cc | 238 --
cpp/tsfile/src/tsfile/scan_iterator.h | 190 --
cpp/tsfile/src/tsfile/schema.h | 63 -
cpp/tsfile/src/tsfile/seq_tvlist.h | 176 --
cpp/tsfile/src/tsfile/statistic.h | 792 -----
cpp/tsfile/src/tsfile/storage_utils.h | 80 -
cpp/tsfile/src/tsfile/tablet.cc | 154 -
cpp/tsfile/src/tsfile/tablet.h | 82 -
cpp/tsfile/src/tsfile/tablet_iterator.h | 59 -
cpp/tsfile/src/tsfile/tsfile_common.cc | 230 --
cpp/tsfile/src/tsfile/tsfile_common.h | 812 -----
cpp/tsfile/src/tsfile/tsfile_executor.cc | 107 -
cpp/tsfile/src/tsfile/tsfile_executor.h | 42 -
cpp/tsfile/src/tsfile/tsfile_io_reader.cc | 495 ---
cpp/tsfile/src/tsfile/tsfile_io_reader.h | 158 -
cpp/tsfile/src/tsfile/tsfile_io_writer.cc | 716 -----
cpp/tsfile/src/tsfile/tsfile_io_writer.h | 190 --
cpp/tsfile/src/tsfile/tsfile_mgr.cc | 130 -
cpp/tsfile/src/tsfile/tsfile_mgr.h | 89 -
cpp/tsfile/src/tsfile/tsfile_reader.cc | 53 -
cpp/tsfile/src/tsfile/tsfile_reader.h | 45 -
.../src/tsfile/tsfile_series_scan_iterator.cc | 102 -
.../src/tsfile/tsfile_series_scan_iterator.h | 93 -
cpp/tsfile/src/tsfile/tsfile_writer.cc | 438 ---
cpp/tsfile/src/tsfile/tsfile_writer.h | 120 -
cpp/tsfile/src/tsfile/write_file.cc | 116 -
cpp/tsfile/src/tsfile/write_file.h | 49 -
pom.xml | 6 +
339 files changed, 28311 insertions(+), 26995 deletions(-)
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index d9dc947b..1d3e9ff8 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -81,8 +81,18 @@ jobs:
} else {
core.setOutput('platform_suffix', ``)
}
+
+ - name: Install python packages
+ if: ${{ matrix.os == 'ubuntu-latest'}}
+ run: pip3 install cython pandas numpy
+
# Run the actual maven build including all unit- and integration-tests.
- name: Build and test with Maven (All others)
shell: bash
- run: ./mvnw${{ steps.platform_suffix.outputs.platform_suffix }} -P with-java,with-cpp clean verify
+ run: |
+ if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
+ ./mvnw${{ steps.platform_suffix.outputs.platform_suffix }} -P with-java clean verify
+ else
+ ./mvnw${{ steps.platform_suffix.outputs.platform_suffix }} -P with-java,with-cpp clean verify
+ fi
\ No newline at end of file
diff --git a/cpp/.clang-format b/cpp/.clang-format
new file mode 100644
index 00000000..26651a2c
--- /dev/null
+++ b/cpp/.clang-format
@@ -0,0 +1,226 @@
+---
+Language: Cpp
+# BasedOnStyle: Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignArrayOfStructures: None
+AlignConsecutiveMacros: None
+AlignConsecutiveAssignments: None
+AlignConsecutiveBitFields: None
+AlignConsecutiveDeclarations: None
+AlignEscapedNewlines: Left
+AlignOperands: Align
+AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortEnumsOnASingleLine: true
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortLambdasOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: WithoutElse
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: Yes
+AttributeMacros:
+ - __capability
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+ AfterCaseLabel: false
+ AfterClass: false
+ AfterControlStatement: Never
+ AfterEnum: false
+ AfterFunction: false
+ AfterNamespace: false
+ AfterObjCDeclaration: false
+ AfterStruct: false
+ AfterUnion: false
+ AfterExternBlock: false
+ BeforeCatch: false
+ BeforeElse: false
+ BeforeLambdaBody: false
+ BeforeWhile: false
+ IndentBraces: false
+ SplitEmptyFunction: true
+ SplitEmptyRecord: true
+ SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeConceptDeclarations: true
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit: 80
+CommentPragmas: '^ IWYU pragma:'
+QualifierAlignment: Leave
+CompactNamespaces: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DeriveLineEnding: true
+DerivePointerAlignment: true
+DisableFormat: false
+EmptyLineAfterAccessModifier: Never
+EmptyLineBeforeAccessModifier: LogicalBlock
+ExperimentalAutoDetectBinPacking: false
+PackConstructorInitializers: NextLine
+BasedOnStyle: ''
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+AllowAllConstructorInitializersOnNextLine: true
+FixNamespaceComments: true
+ForEachMacros:
+ - foreach
+ - Q_FOREACH
+ - BOOST_FOREACH
+IfMacros:
+ - KJ_IF_MAYBE
+IncludeBlocks: Regroup
+IncludeCategories:
+ - Regex: '^<ext/.*\.h>'
+ Priority: 2
+ SortPriority: 0
+ CaseSensitive: false
+ - Regex: '^<.*\.h>'
+ Priority: 1
+ SortPriority: 0
+ CaseSensitive: false
+ - Regex: '^<.*'
+ Priority: 2
+ SortPriority: 0
+ CaseSensitive: false
+ - Regex: '.*'
+ Priority: 3
+ SortPriority: 0
+ CaseSensitive: false
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IncludeIsMainSourceRegex: ''
+IndentAccessModifiers: false
+IndentCaseLabels: true
+IndentCaseBlocks: false
+IndentGotoLabels: true
+IndentPPDirectives: None
+IndentExternBlock: AfterExternBlock
+IndentRequires: false
+IndentWidth: 4
+IndentWrappedFunctionNames: false
+InsertTrailingCommas: None
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+LambdaBodyIndentation: Signature
+MacroBlockBegin: ''
+MacroBlockEnd: ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Never
+ObjCBlockIndentWidth: 2
+ObjCBreakBeforeNestedBlockParam: true
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakOpenParenthesis: 0
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PenaltyIndentedWhitespace: 0
+PointerAlignment: Left
+PPIndentWidth: -1
+RawStringFormats:
+ - Language: Cpp
+ Delimiters:
+ - cc
+ - CC
+ - cpp
+ - Cpp
+ - CPP
+ - 'c++'
+ - 'C++'
+ CanonicalDelimiter: ''
+ BasedOnStyle: google
+ - Language: TextProto
+ Delimiters:
+ - pb
+ - PB
+ - proto
+ - PROTO
+ EnclosingFunctions:
+ - EqualsProto
+ - EquivToProto
+ - PARSE_PARTIAL_TEXT_PROTO
+ - PARSE_TEST_PROTO
+ - PARSE_TEXT_PROTO
+ - ParseTextOrDie
+ - ParseTextProtoOrDie
+ - ParseTestProto
+ - ParsePartialTestProto
+ CanonicalDelimiter: pb
+ BasedOnStyle: google
+ReferenceAlignment: Pointer
+ReflowComments: true
+RemoveBracesLLVM: false
+SeparateDefinitionBlocks: Leave
+ShortNamespaceLines: 1
+SortIncludes: CaseSensitive
+SortJavaStaticImport: Before
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCaseColon: false
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeParensOptions:
+ AfterControlStatements: true
+ AfterForeachMacros: true
+ AfterFunctionDefinitionName: false
+ AfterFunctionDeclarationName: false
+ AfterIfMacros: true
+ AfterOverloadedOperator: false
+ BeforeNonEmptyParentheses: false
+SpaceAroundPointerQualifiers: Default
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles: Never
+SpacesInConditionalStatement: false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInLineCommentPrefix:
+ Minimum: 1
+ Maximum: -1
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpaceBeforeSquareBrackets: false
+BitFieldColonSpacing: Both
+Standard: Auto
+StatementAttributeLikeMacros:
+ - Q_EMIT
+StatementMacros:
+ - Q_UNUSED
+ - QT_REQUIRE_VERSION
+TabWidth: 8
+UseCRLF: false
+UseTab: Never
+WhitespaceSensitiveMacros:
+ - STRINGIZE
+ - PP_STRINGIZE
+ - BOOST_PP_STRINGIZE
+ - NS_SWIFT_NAME
+ - CF_SWIFT_NAME
+...
+
diff --git a/cpp/.gitignore b/cpp/.gitignore
new file mode 100644
index 00000000..fe3949b9
--- /dev/null
+++ b/cpp/.gitignore
@@ -0,0 +1,7 @@
+build/
+.idea/
+.cproject
+.project
+.settings/
+.vscode/
+cmake-build-debug/
\ No newline at end of file
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 6576ef6f..b26323b4 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -1,84 +1,75 @@
#[[
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
- https://www.apache.org/licenses/LICENSE-2.0
+ https://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
]]
+cmake_minimum_required(VERSION 3.11)
+project(TsFile_CPP)
-cmake_minimum_required(VERSION 3.20)
+set(TsFile_CPP_VERSION_MAJOR 0)
+set(TsFile_CPP_VERSION_MINOR 1)
-# Set the name of this project
-project(TsFile)
+set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -Werror")
+message("cmake using: USE_CPP11=${USE_CPP11}")
-# Set the C language level to C11
-set(CMAKE_C_STANDARD 11)
-# Save the root directory
-set(TSFILE_ROOT_DIR ${CMAKE_SOURCE_DIR})
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-# Our link to the maven lifecycle
-set(BUILD_PHASE test-compile CACHE STRING "Phase of the Maven build we are executing cmake")
+if(DEFINED ENV{CXX})
+ set(CMAKE_CXX_COMPILER $ENV{CXX})
+ message("cmake using: CXX=${CMAKE_CXX_COMPILER}")
+endif()
-# Access the Unity version the maven build is providing us with.
-set(UNITY_VERSION 2.5.2 CACHE STRING "Version of the used Unity test framework")
+if(DEFINED ENV{CC})
+ set(CMAKE_C_COMPILER $ENV{CC})
+ message("cmake using: CC=${CMAKE_C_COMPILER}")
+endif()
-# MinGW GCC versions above 4.7 on Windows include inline definitions of these functions
-# If you run into problems, please read this article: https://stackoverflow.com/questions/27853225/is-there-a-way-to-include-stdio-h-but-ignore-some-of-the-functions-therein
-# These are the constants needed to activate all patches.
-add_compile_definitions(NO_INLINE_VFSCANF=1)
-add_compile_definitions(NO_INLINE_VSSCANF=1)
-add_compile_definitions(NO_INLINE_VSCANF=1)
-add_compile_definitions(NO_INLINE_VSNPRINTF=1)
-add_compile_definitions(NO_INLINE_SNPRINTF=1)
-add_compile_definitions(NO_INLINE_VFWSCANF=1)
-add_compile_definitions(NO_INLINE_VSWSCANF=1)
-add_compile_definitions(NO_INLINE_VWSCANF=1)
-add_compile_definitions(NO_INLINE_SNWPRINTF=1)
-add_compile_definitions(NO_INLINE_VSNWPRINTF=1)
-add_compile_definitions(NO_INLINE_VSWPRINTF=1)
-add_compile_definitions(NO_INLINE_SWPRINTF=1)
+message("cmake using: DEBUG_SE=${DEBUG_SE}")
+if (${DEBUG_SE})
+ add_definitions(-DDEBUG_SE=1)
+ message("add_definitions -DDEBUG_SE=1")
+endif()
-# Depending on the phase of the build we are currently running, initialize
-# The test system.
-if (BUILD_PHASE STREQUAL compile)
- # Nothing really to do here ... just need it to check the known values.
-elseif (BUILD_PHASE STREQUAL test-compile)
- # Initialize the test subsystem as well as the Unity library sources
- include(CTest)
+message("build type: ${BUILD_TYPE}")
+if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g")
+ set(CMAKE_CXX_FLAGS_DEBUG" ${CMAKE_CXX_FLAGS} -O0 -g")
+elseif (CMAKE_BUILD_TYPE STREQUAL "Release")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -g") # disable -g
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS}")
+elseif (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -g")
+ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS}")
+elseif (CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffunction-sections -fdata-sections -Os")
+ set(LDFLAGS "-Wl,--gc-sections")
+ set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS}")
+endif()
+message("CMAKE DEBUG: CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}")
- # Add the Unity sources to the build
- add_subdirectory(target/dependency/Unity-${UNITY_VERSION})
-else ()
- # Output an error
- message(FATAL_ERROR "Given BUILD_PHASE unknown. Known values are 'compile' and 'test-compile'")
-endif ()
+set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
+set(PROJECT_INCLUDE_DIR ${PROJECT_INCLUDE_DIR} ${PROJECT_SOURCE_DIR}/src)
-set(DEBUG_OUTPUT ON CACHE BOOL "Enable outputting of debug information")
-if (DEBUG_OUTPUT)
- # Trace all CMAKE Variables
- get_cmake_property(_variableNames VARIABLES)
- list(SORT _variableNames)
- foreach (_variableName ${_variableNames})
- message(STATUS "${_variableName}=${${_variableName}}")
- endforeach ()
-endif ()
+set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin)
+include_directories(${PROJECT_INCLUDE_DIR})
-#[[
- Build all the modules of TSFILE
-]]
-# Core stuff
-add_subdirectory(common)
-add_subdirectory(examples)
-add_subdirectory(tsfile)
+add_subdirectory(src)
+
+configure_file(
+ "${PROJECT_SOURCE_DIR}/cmake/config.h.in"
+ "${PROJECT_BINARY_DIR}/cmake_config.h"
+)
diff --git a/cpp/tsfile/README-zh.md b/cpp/README-zh.md
similarity index 100%
rename from cpp/tsfile/README-zh.md
rename to cpp/README-zh.md
diff --git a/cpp/README.md b/cpp/README.md
new file mode 100644
index 00000000..2d7be530
--- /dev/null
+++ b/cpp/README.md
@@ -0,0 +1,79 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+# TsFile C++ Document
+
+<pre>
+___________ ___________.__.__
+\__ ___/____\_ _____/|__| | ____
+ | | / ___/| __) | | | _/ __ \
+ | | \___ \ | \ | | |_\ ___/
+ |____|/____ >\___ / |__|____/\___ > version 1.0.0
+ \/ \/ \/
+</pre>
+
+
+## Introduction
+
+
+This directory contains the C++ implementation of TsFile. The C++ version currently supports the query and write functions of TsFile, including time filtering queries.
+
+The source code can be found in the `./src` directory. C/C++ examples are located in the `./examples` directory, and a benchmark for TsFile_cpp can be found in the `./bench_mark` directory. Additionally, a C function wrapper is available in the `./src/cwrapper` directory, which the Python tool relies on.
+
+## How to make contributions
+
+We use `clang-format` to ensure that our C++ code adheres to a consistent set of rules defined in `./clang-format`. This is similar to the Google style.
+
+**Feature List**:
+
+- [ ] Add unit tests for the reader, writer, compression, etc.
+- [ ] Add unit tests for the C wrapper.
+- [ ] Support multiple data flushes.
+- [ ] Support aligned timeseries.
+- [ ] Support table description in tsfile.
+- [ ] Retrieve all table schemas/names.
+- [ ] Implement automatic flush.
+- [ ] Support out-of-order data writing.
+- [ ] Support TsFile V4. Note: TsFile CPP does not implement support for the table model, therefore there are differences in file output compared to the Java version.
+
+**Bug List**:
+
+- [ ] Flushing without writing after registering a timeseries will cause a core dump.
+- [ ] Misalignment in memory may lead to a bus error.
+
+We welcome any bug reports. You can open an issue with a title starting with [CPP] to describe the bug, like: https://github.com/apache/tsfile/issues/94
+
+## Build
+
+### Requirements
+
+```bash
+sudo apt-get update
+sudo apt-get install -y cmake make g++ clang-format
+```
+
+To build tsfile, you can run: `bash build.sh`. If you have Maven tools, you can run: `mvn package -P with-cpp clean verify`. Then, you can find the shared object at `./build`.
+
+Before you submit your code to GitHub, please ensure that the `mvn` compilation is correct.
+
+## Use TsFile
+
+You can find examples on how to read and write data in `demo_read.cpp` and `demo_write.cpp` located under `./examples/cpp_examples`. There are also examples under `./examples/c_examples`on how to use a C-style API to read and write data in a C environment. You can run `bash build.sh` under `./examples` to generate an executable output under `./examples/build`.
\ No newline at end of file
diff --git a/cpp/bench_mark/CMakeLists.txt b/cpp/bench_mark/CMakeLists.txt
new file mode 100644
index 00000000..6db63999
--- /dev/null
+++ b/cpp/bench_mark/CMakeLists.txt
@@ -0,0 +1,33 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
+message("Running in bench_mark directory")
+if(DEFINED ENV{CXX})
+ set(CMAKE_CXX_COMPILER $ENV{CXX})
+endif()
+
+set(CMAKE_CXX_FLAGS "$ENV{CXX_FLAGS} -Wall -Werror")
+
+if (${USE_CPP11})
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+ set(CMAKE_CXX_STANDARD 11)
+else()
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++03")
+endif()
+
+add_subdirectory(bench_mark_src)
\ No newline at end of file
diff --git a/cpp/bench_mark/bench_mark_src/CMakeLists.txt b/cpp/bench_mark/bench_mark_src/CMakeLists.txt
new file mode 100644
index 00000000..dbad71f9
--- /dev/null
+++ b/cpp/bench_mark/bench_mark_src/CMakeLists.txt
@@ -0,0 +1,57 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
+cmake_minimum_required(VERSION 3.1)
+project(libtsfile_bench_mark_project)
+message("Running in bench_mark/bench_mark_src directory")
+if(DEFINED ENV{CXX})
+ set(CMAKE_CXX_COMPILER $ENV{CXX})
+endif()
+
+set(SDK_BENCH_MARK_DIR ${PROJECT_SOURCE_DIR}/)
+message("PROJECT DIR: ${SDK_BENCH_MARK_DIR}")
+set(SDK_INCLUDE_DIR_DEBUG ${SKD_BENCHH_MARK_DIR}../../build/Debug/bin/libtsfile_sdk/include)
+set(SDK_INCLUDE_DIR_RELEASE ${SKD_BENCHH_MARK_DIR}../../build/Release/bin/libtsfile_sdk/include)
+set(SDK_LIB_DIR_DEBUG ${SKD_BENCHH_MARK_DIR}../../build/Debug/bin/libtsfile_sdk/lib)
+set(SDK_LIB_DIR_RELEASE ${SKD_BENCHH_MARK_DIR}../../build/Release/bin/libtsfile_sdk/lib)
+
+if (USE_SDK_DEBUG)
+ SET(SKD_INCLUDE_DIR ${SDK_INCLUDE_DIR_DEBUG})
+ SET(SDK_LIB_DIR ${SDK_LIB_DIR_DEBUG})
+ SET(CMAKE_CXX_FLAGS "-g -O0")
+else()
+ SET(SKD_INCLUDE_DIR ${SDK_INCLUDE_DIR_RELEASE})
+ SET(SDK_LIB_DIR ${SDK_LIB_DIR_RELEASE})
+ SET(CMAKE_CXX_FLAGS "-O3")
+endif()
+
+include_directories(${SKD_INCLUDE_DIR})
+set(MAKE_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/../../src)
+include_directories(${MAKE_INCLUDE})
+message("MAKE_INCLUDE: ${MAKE_INCLUDE}")
+message("SDK_INCLUDE_DIR: ${SKD_INCLUDE_DIR}")
+message("SDK_LIB_DIR: ${SDK_LIB_DIR}")
+
+link_directories(${SDK_LIB_DIR})
+find_library(my_tsfile_lib NAMES tsfile PATHS ${SDK_LIB_DIR} NO_DEFAULT_PATH REQUIRED)
+add_executable(bench_mark_src bench_mark.cc)
+target_link_libraries(bench_mark_src ${my_tsfile_lib})
+
+
+
+
diff --git a/cpp/common/src/common.cpp b/cpp/bench_mark/bench_mark_src/bench_conf.h
similarity index 71%
rename from cpp/common/src/common.cpp
rename to cpp/bench_mark/bench_mark_src/bench_conf.h
index 1d87741f..9d961d72 100644
--- a/cpp/common/src/common.cpp
+++ b/cpp/bench_mark/bench_mark_src/bench_conf.h
@@ -4,10 +4,10 @@
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
+ * License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * https://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -16,12 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-#include "common.hpp"
+#include <vector>
-Common::Common() {
- // Not really doing anything here ...
-}
-
-int Common::doSomething(int a, int b) const {
- return a+b;
-}
\ No newline at end of file
+namespace bench {
+int LOOP_NUM = 100000;
+int THREAD_NUM = 1;
+int TIMESERIES_NUM = 50;
+std::vector<int> TYPE_LIST = {0, 0, 1, 0, 1};
+} // namespace bench
\ No newline at end of file
diff --git a/cpp/bench_mark/bench_mark_src/bench_mark.cc b/cpp/bench_mark/bench_mark_src/bench_mark.cc
new file mode 100644
index 00000000..09c9eb98
--- /dev/null
+++ b/cpp/bench_mark/bench_mark_src/bench_mark.cc
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <chrono>
+#include <cmath>
+#include <iostream>
+#include <numeric>
+#include <string>
+
+#include "bench_conf.h"
+#include "common/db_common.h"
+#include "common/global.h"
+#include "common/path.h"
+#include "writer/tsfile_writer.h"
+
+std::vector<int> register_timeseries(storage::TsFileWriter& writer,
+ int timeseries_num,
+ std::vector<int> type_list) {
+ auto start = std::chrono::high_resolution_clock::now();
+ int sum = std::accumulate(type_list.begin(), type_list.end(), 0);
+ std::vector<float> ratio_list;
+ for (int i = 0; i < type_list.size(); i++) {
+ ratio_list.push_back((float)type_list[i] / sum);
+ }
+ std::vector<int> type_num;
+ for (int i = 0; i < common::TSDataType::TEXT - 1; i++) {
+ type_num.push_back((int)std::ceil(timeseries_num * ratio_list[i]));
+ }
+ type_num.push_back(timeseries_num -
+ std::accumulate(type_num.begin(), type_num.end(), 0));
+ writer.open("/tmp/tsfile_test.tsfile", O_CREAT | O_RDWR, 0644);
+ int ind = 0;
+ int ret = 0;
+ int type = 0;
+ for (auto num : type_num) {
+ for (int i = 0; i < num; i++) {
+ std::string device_name = "root.db001.dev" + std::to_string(ind);
+ std::string measurement_name = "m" + std::to_string(ind);
+ ret = writer.register_timeseries(
+ device_name, measurement_name, (common::TSDataType)type,
+ common::TSEncoding::PLAIN,
+ common::CompressionType::UNCOMPRESSED);
+ ind++;
+ }
+ std::cout << "register finished for TsDataType"
+ << common::s_data_type_names[type]
+ << " timeseries num: " << num << std::endl;
+ type++;
+ }
+ auto end = std::chrono::high_resolution_clock::now();
+ std::chrono::duration<double> elapsed = end - start;
+ std::cout << "register " << timeseries_num << "timeseries in file"
+ << "./test_data/tsfile_test.tsfile" << std::endl;
+ std::cout << "register timeseries cost time: " << elapsed.count() << "s"
+ << std::endl;
+ return type_num;
+}
+
+void test_writer_benchmark(storage::TsFileWriter& writer, int loop_num,
+ std::vector<int> type_num) {
+ std::cout << "start writing data" << std::endl;
+ auto start = std::chrono::high_resolution_clock::now();
+ int type = 0;
+ for (int i = 0; i < loop_num; i++) {
+ int ind = 0;
+ for (auto num : type_num) {
+ for (int j = 0; j < num; j++) {
+ std::string device_name =
+ "root.db001.dev" + std::to_string(ind);
+ std::string measurement_name = "m" + std::to_string(ind);
+ long long currentTimeStamp = i;
+ storage::TsRecord record(currentTimeStamp, device_name, 1);
+ switch (type) {
+ case common::INT32: {
+ storage::DataPoint point(measurement_name, 10000 + i);
+ record.points_.push_back(point);
+ break;
+ }
+ case common::INT64: {
+ storage::DataPoint point(measurement_name,
+ int64_t(10000 + i));
+ record.points_.push_back(point);
+ break;
+ }
+ case common::BOOLEAN: {
+ storage::DataPoint point(measurement_name, i / 2 == 0);
+ record.points_.push_back(point);
+ break;
+ }
+ case common::FLOAT: {
+ storage::DataPoint point(measurement_name, (float)i);
+ record.points_.push_back(point);
+ break;
+ }
+ case common::DOUBLE: {
+ storage::DataPoint point(measurement_name, (double)i);
+ record.points_.push_back(point);
+ break;
+ }
+ }
+ int ret = writer.write_record(record);
+ ASSERT(ret == 0);
+ ind++;
+ }
+ type++;
+ }
+ }
+
+ auto end = std::chrono::high_resolution_clock::now();
+ std::chrono::duration<double> elapsed = end - start;
+ int timeseries_num = std::accumulate(type_num.begin(), type_num.end(), 0);
+ std::cout << "writer loop: " << loop_num
+ << " timeseries num: " << timeseries_num << " records in file"
+ << "./test_data/tsfile_test.tsfile" << std::endl;
+ std::cout << "total num of points: " << loop_num * timeseries_num
+ << std::endl;
+ std::cout << "writer data cost time: " << elapsed.count() << "s"
+ << std::endl;
+ std::cout << "writer data speed:"
+ << loop_num * timeseries_num / elapsed.count() << " points/s"
+ << std::endl;
+ writer.flush();
+ writer.close();
+ auto end_flush = std::chrono::high_resolution_clock::now();
+ std::chrono::duration<double> elapsed_flush = end_flush - end;
+ std::cout << "flush data cost time: " << elapsed_flush.count() << "s"
+ << std::endl;
+}
+
+int main() {
+ std::cout << "LibTsFile benchmark" << std::endl;
+ std::cout << "LOOP_NUM:" << bench::LOOP_NUM << std::endl;
+ std::cout << "THREAD_NUM:" << bench::THREAD_NUM << std::endl;
+ std::cout << "TIMESERIES_NUM:" << bench::TIMESERIES_NUM << std::endl;
+ std::cout << "TYPE_LIST: " << bench::TYPE_LIST[0] << ":"
+ << bench::TYPE_LIST[1] << ":" << bench::TYPE_LIST[2] << ":"
+ << bench::TYPE_LIST[3] << ":" << bench::TYPE_LIST[4] << ":"
+ << bench::TYPE_LIST[5] << std::endl;
+ std::cout << "init tsfile config value" << std::endl;
+ common::init_config_value();
+ storage::TsFileWriter writer;
+ auto type_num =
+ register_timeseries(writer, bench::TIMESERIES_NUM, bench::TYPE_LIST);
+ test_writer_benchmark(writer, bench::LOOP_NUM, type_num);
+ return 0;
+}
diff --git a/cpp/tsfile/test/tsfile_test.cpp b/cpp/bench_mark/bench_mark_src/bench_mark.h
similarity index 79%
rename from cpp/tsfile/test/tsfile_test.cpp
rename to cpp/bench_mark/bench_mark_src/bench_mark.h
index 78f99cc2..e3bbd95d 100644
--- a/cpp/tsfile/test/tsfile_test.cpp
+++ b/cpp/bench_mark/bench_mark_src/bench_mark.h
@@ -4,10 +4,10 @@
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
+ * License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * https://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -16,9 +16,3 @@
* specific language governing permissions and limitations
* under the License.
*/
-#include <cstdio>
-
-int main() {
- printf("Test");
- return 0;
-}
\ No newline at end of file
diff --git a/cpp/bench_mark/build.sh b/cpp/bench_mark/build.sh
new file mode 100644
index 00000000..e0e4b0cb
--- /dev/null
+++ b/cpp/bench_mark/build.sh
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+build_type=Debug
+env_for_cyber=0
+use_cpp11=1
+
+if [[ ${env_for_cyber} -eq 1 ]]
+then
+ export PATH=$PATH:~/dev/gcc-linaro-5.5.0-2017.10-x86_64_arm-linux-gnueabi/bin
+ export CROSS_COMPILE=arm-linux-gnueabi-
+ export ARCH=arm
+ export CC=${CROSS_COMPILE}gcc
+ export CXX=${CROSS_COMPILE}g++
+ echo "set up gcc for cyber"
+fi
+
+
+if [ ${build_type} = "Debug" ]
+then
+ mkdir -p build/Debug
+ cd build/Debug
+ use_sdk_debug=1
+else
+ mkdir -p build/Release
+ cd build/Release
+ use_sdk_debug=0
+fi
+
+echo "use_sdk_debug=${use_sdk_debug}"
+cmake ../../ \
+ -DUSE_SDK_DEBUG=$use_sdk_debug \
+ -DUSE_CPP11=$use_cpp11
+make
diff --git a/cpp/build-utils/source-bundle-descriptor.xml b/cpp/build-utils/source-bundle-descriptor.xml
deleted file mode 100644
index 0657843f..00000000
--- a/cpp/build-utils/source-bundle-descriptor.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0 http://maven.apache.org/xsd/assembly-2.2.0.xsd">
- <id>sources</id>
- <formats>
- <format>zip</format>
- </formats>
- <fileSets>
- <fileSet>
- <directory>${basedir}</directory>
- <includes>
- <include>common/**</include>
- <include>examples/**</include>
- <include>tsfile/**</include>
- <include>CMakeLists.txt</include>
- </includes>
- </fileSet>
- </fileSets>
-</assembly>
diff --git a/cpp/build.sh b/cpp/build.sh
new file mode 100644
index 00000000..8d907cde
--- /dev/null
+++ b/cpp/build.sh
@@ -0,0 +1,142 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# build_type=MinSizeRel
+build_type=Release
+build_test=0
+build_bench=0
+use_cpp11=1
+enable_cov=0
+debug_se=0
+run_cov_only=0
+env_for_cyber=0
+
+shell_dir=$(cd "$(dirname "$0")";pwd)
+
+function print_config()
+{
+ echo "build_type=$build_type"
+ echo "build_test=$build_test"
+ echo "use_cpp11=$use_cpp11"
+ echo "enable_cov=$enable_cov"
+ echo "enable_asan=$enable_asan"
+}
+
+function run_test_for_cov()
+{
+ # sh ${shell_dir}/scripts/regression_unittest.sh
+ sh ${shell_dir}/test/libtsfile_test/run_sdk_tests.sh
+}
+
+parse_options()
+{
+ while test $# -gt 0
+ do
+ case "$1" in
+ clean)
+ do_clean=1;;
+ run_cov)
+ run_cov_only=1;;
+ -t=*)
+ build_type=`get_key_value "$1"`;;
+ -t)
+ shift
+ build_type=`get_key_value "$1"`;;
+ -a=*)
+ enable_asan=`get_key_value "$1"`;;
+ -a)
+ shift
+ enable_asan=`get_key_value "$1"`;;
+ -c=*)
+ enable_cov=`get_key_value "$1"`;;
+ -c)
+ shift
+ enable_cov=`get_key_value "$1"`;;
+ #-h | --help)
+ # usage
+ # exit 0;;
+ #*)
+ # echo "Unknown option '$1'"
+ # exit 1;;
+ esac
+ shift
+ done
+}
+
+parse_options $*
+print_config
+
+if [[ ${run_cov_only} -eq 1 ]]
+then
+ do_cov
+ # exit after run coverage test.
+ exit
+fi
+
+if [ ${build_test} -eq 1 ]
+then
+ compile_gtest
+ echo "building status: build gtest success."
+fi
+
+echo "build using: build_type=$build_type"
+if [ ${build_type} == "Debug" ]
+then
+ mkdir -p build/Debug
+ cd build/Debug
+elif [ ${build_type} == "Release" ]
+then
+ mkdir -p build/Release
+ cd build/Release
+elif [ ${build_type} == "RelWithDebInfo" ]
+then
+ mkdir -p build/relwithdebinfo
+ cd build/relwithdebinfo
+elif [ ${build_type} == "MinSizeRel" ]
+then
+ mkdir -p build/minsizerel
+ cd build/minsizerel
+else
+ echo ""
+ echo "unknow build type: ${build_type}, valid build types(case intensive): Debug, Release, RelWithDebInfo, MinSizeRel"
+ echo ""
+ exit 1
+fi
+
+cmake ../../ \
+ -DGTEST=$gtest_project_dir \
+ -DZLIB=$zlib_project_dir/install \
+ -DLZ4LIB=$lz4lib_project_dir \
+ -DBUILD_TEST=$build_test \
+ -DCMAKE_BUILD_TYPE=$build_type \
+ -DUSE_CPP11=$use_cpp11 \
+ -DENABLE_COV=$enable_cov \
+ -DENABLE_ASAN=$enable_asan \
+ -DDEBUG_SE=$debug_se \
+ -DBUILD_TSFILE_ONLY=$build_tsfile_only
+
+VERBOSE=1 make
+VERBOSE=1 make install
+
+
+
+
+
+
+
diff --git a/cpp/cmake/build_function.cmake b/cpp/cmake/build_function.cmake
new file mode 100644
index 00000000..a1161690
--- /dev/null
+++ b/cpp/cmake/build_function.cmake
@@ -0,0 +1,34 @@
+
+function(unittest_binary_link target_test target_src_files)
+ set(LOGGER_SRC_DIR ${PROJECT_SOURCE_DIR}/src/common/logger)
+ include_directories(${LOGGER_SRC_DIR})
+ aux_source_directory(${LOGGER_SRC_DIR} logger_src_files)
+ set(target_files ${target_src_files} ${logger_src_files})
+
+ add_executable(${target_test} ${target_files})
+
+ # add_executable(${target_test} ${target_src_files})
+ target_link_libraries(${target_test} gtest)
+ target_link_libraries(${target_test} z)
+ target_link_libraries(${target_test} lz4)
+ set_target_properties(${target_test} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${UNITTEST_BIN_DIR})
+ set_target_properties(${target_test} PROPERTIES COMPILE_FLAGS "-pthread" LINK_FLAGS "-pthread")
+endfunction(unittest_binary_link)
+
+function(add_statis_objtarget target_name target_src_list target_inc_list target_defile_list target_compile_list target_link_list)
+ message("add_statis_objtarget(target_name=${target_name}, target_src_list=${${target_src_list}})")
+ add_library(${target_name} OBJECT ${${target_src_list}})
+ # # redefine_file_macro(${target_name})
+ # if (target_inc_list)
+ # target_include_directories(${target_name} PUBLIC ${${target_inc_list}} ${PROJECT_INCLUDE_DIR})
+ # endif()
+ if (target_defile_list)
+ target_compile_definitions(${target_name} PRIVATE ${target_defile_list})
+ endif()
+ if (target_compile_list)
+ target_compile_options(${target_name} PRIVATE ${target_compile_list})
+ endif()
+ if (target_link_list)
+ target_link_options(${target_name} PRIVATE ${target_link_list})
+ endif()
+endfunction(add_statis_objtarget)
diff --git a/cpp/common/test/common_test.cpp b/cpp/cmake/cmake_config.h
similarity index 79%
rename from cpp/common/test/common_test.cpp
rename to cpp/cmake/cmake_config.h
index 78f99cc2..d92824c6 100644
--- a/cpp/common/test/common_test.cpp
+++ b/cpp/cmake/cmake_config.h
@@ -4,10 +4,10 @@
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
+ * License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * https://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -16,9 +16,4 @@
* specific language governing permissions and limitations
* under the License.
*/
-#include <cstdio>
-
-int main() {
- printf("Test");
- return 0;
-}
\ No newline at end of file
+/* #undef ENABLE_TEST */
diff --git a/cpp/cmake/config.h.in b/cpp/cmake/config.h.in
new file mode 100644
index 00000000..3e07b4e6
--- /dev/null
+++ b/cpp/cmake/config.h.in
@@ -0,0 +1 @@
+#cmakedefine ENABLE_TEST
diff --git a/cpp/common/CMakeLists.txt b/cpp/common/CMakeLists.txt
deleted file mode 100644
index 1771cd9b..00000000
--- a/cpp/common/CMakeLists.txt
+++ /dev/null
@@ -1,48 +0,0 @@
-#[[
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- https://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-]]
-
-add_library(tsfile-common
- src/common.cpp
- )
-
-target_include_directories(
- tsfile-common
- PUBLIC
- "include")
-
-IF (NOT WIN32)
- target_link_libraries(tsfile-common
- m
- )
-ELSE()
- target_link_libraries(tsfile-common
- )
-ENDIF()
-
-if (BUILD_PHASE STREQUAL test-compile)
- add_executable(tsfile-common-test
- test/common_test.cpp
- )
- target_link_libraries(tsfile-common-test
- tsfile-common
- unity
-
- )
- add_test(NAME tsfile-common-test COMMAND tsfile-common-test)
-endif ()
\ No newline at end of file
diff --git a/cpp/examples/CMakeLists.txt b/cpp/examples/CMakeLists.txt
index fe4ca822..c99ac0f4 100644
--- a/cpp/examples/CMakeLists.txt
+++ b/cpp/examples/CMakeLists.txt
@@ -1,50 +1,34 @@
#[[
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
- https://www.apache.org/licenses/LICENSE-2.0
+ https://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
]]
+cmake_minimum_required(VERSION 3.10)
+project(examples)
+message("Running in exampes directory")
-add_library(tsfile-examples
- src/example.cpp
- )
+set(SDK_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/../src/)
+message("SDK_INCLUDE_DIR: ${SDK_INCLUDE_DIR}")
+set(SDK_LIB_DIR_RELEASE ${PROJECT_SOURCE_DIR}/../build/Release/lib)
+message("SDK_LIB_DIR_RELEASE: ${SDK_LIB_DIR_RELEASE}")
-target_include_directories(
- tsfile-examples
- PUBLIC
- "include"
- "../common/include"
- "../tsfile/include")
+include_directories(${SDK_INCLUDE_DIR})
+find_library(my_tsfile_lib NAMES tsfile PATHS ${SDK_LIB_DIR_RELEASE} NO_DEFAULT_PATH REQUIRED)
-IF (NOT WIN32)
- target_link_libraries(tsfile-examples
- m
- )
-ELSE()
- target_link_libraries(tsfile-examples
- )
-ENDIF()
+add_subdirectory(cpp_examples)
-#if (BUILD_PHASE STREQUAL test-compile)
-# add_executable(tsfile-examples-test
-# test/system_test.c
-# )
-# target_link_libraries(tsfile-examples-test
-# tsfile-examples
-# unity
-#
-# )
-# add_test(NAME tsfile-examples-test COMMAND tsfile-examples-test)
-#endif ()
\ No newline at end of file
+add_executable(examples examples.cc)
+target_link_libraries(examples ${my_tsfile_lib} cpp_examples_obj)
\ No newline at end of file
diff --git a/cpp/examples/build.sh b/cpp/examples/build.sh
new file mode 100644
index 00000000..bf4a3c45
--- /dev/null
+++ b/cpp/examples/build.sh
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+build_type=Debug
+use_cpp11=1
+if [ ${build_type} = "Debug" ]
+then
+ mkdir -p build/Debug
+ cd build/Debug
+else
+ mkdir -p build/Release
+ cd build/Release
+fi
+
+
+cmake ../../ -DUSE_CPP11=$use_cpp11
+make
\ No newline at end of file
diff --git a/cpp/examples/cpp_examples/CMakeLists.txt b/cpp/examples/cpp_examples/CMakeLists.txt
new file mode 100644
index 00000000..a2ac8d43
--- /dev/null
+++ b/cpp/examples/cpp_examples/CMakeLists.txt
@@ -0,0 +1,22 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
+
+message("Running in examples/cpp_examples directory")
+aux_source_directory(. cpp_SRC_LIST)
+add_library(cpp_examples_obj OBJECT ${cpp_SRC_LIST})
diff --git a/cpp/common/include/common.hpp b/cpp/examples/cpp_examples/cpp_examples.h
similarity index 59%
copy from cpp/common/include/common.hpp
copy to cpp/examples/cpp_examples/cpp_examples.h
index 6eb9830c..f8202537 100644
--- a/cpp/common/include/common.hpp
+++ b/cpp/examples/cpp_examples/cpp_examples.h
@@ -4,10 +4,10 @@
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
+ * License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * https://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -16,15 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-#ifndef TSFILE_COMMON_H
-#define TSFILE_COMMON_H
-class Common
-{
-public:
- Common();
+#include "common/path.h"
+#include "reader/filter/filter.h"
+#include "reader/expression.h"
+#include "reader/tsfile_reader.h"
+#include "reader/qds_with_timegenerator.h"
+#include "reader/qds_without_timegenerator.h"
+#include "common/row_record.h"
+#include "writer/tsfile_writer.h"
+#include "common/record.h"
+#include "common/db_common.h"
- int doSomething(int a, int b) const;
-};
-
-#endif
\ No newline at end of file
+int demo_read();
+int demo_write();
diff --git a/cpp/examples/cpp_examples/demo_read.cpp b/cpp/examples/cpp_examples/demo_read.cpp
new file mode 100644
index 00000000..ce87dc88
--- /dev/null
+++ b/cpp/examples/cpp_examples/demo_read.cpp
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "cpp_examples.h"
+
+#include <iostream>
+#include <vector>
+#include <string>
+
+std::string field_to_string(storage::Field *value)
+{
+ if (value->type_ == common::TEXT) {
+ return std::string(value->value_.sval_);
+ } else {
+ std::stringstream ss;
+ switch (value->type_) {
+ case common::BOOLEAN:
+ ss << (value->value_.bval_ ? "true" : "false");
+ break;
+ case common::INT32:
+ ss << value->value_.ival_;
+ break;
+ case common::INT64:
+ ss << value->value_.lval_;
+ break;
+ case common::FLOAT:
+ ss << value->value_.fval_;
+ break;
+ case common::DOUBLE:
+ ss << value->value_.dval_;
+ break;
+ case common::NULL_TYPE:
+ ss << "NULL";
+ break;
+ default:
+ ASSERT(false);
+ break;
+ }
+ return ss.str();
+ }
+}
+
+int demo_read()
+{
+ std::cout<<"begin to read tsfile from demo_ts.tsfile" << std::endl;
+ std::string device_name = "root.db001.dev001";
+ std::string measurement_name = "m001";
+ storage::Path p1(device_name, measurement_name);
+ std::vector<storage::Path> select_list;
+ select_list.push_back(p1);
+ storage::QueryExpression *query_expr =
+ storage::QueryExpression::create(select_list, nullptr);
+
+ common::init_config_value();
+ storage::TsFileReader reader;
+ int ret = reader.open("demo_ts.tsfile");
+
+ std::cout << "begin to query expr" << std::endl;
+ ASSERT(ret == 0);
+ storage::QueryDataSet *qds = nullptr;
+ ret = reader.query(query_expr, qds);
+
+
+ storage::RowRecord *record;
+ std::cout << "begin to dump data from tsfile ---" << std::endl;
+ int row_cout = 0;
+ do {
+ record = qds->get_next();
+ if (record) {
+ std::cout<< "dump QDS : " << record->get_timestamp() << ",";
+ if (record) {
+ int size = record->get_fields()->size();
+ for (int i = 0; i < size; ++i) {
+ std::cout << field_to_string(record->get_field(i)) << ",";
+ }
+ std::cout<< std::endl;
+ row_cout++;
+ }
+ } else {
+ break;
+ }
+ } while (true);
+
+ return (0);
+}
+
diff --git a/cpp/examples/cpp_examples/demo_write.cpp b/cpp/examples/cpp_examples/demo_write.cpp
new file mode 100644
index 00000000..bb394149
--- /dev/null
+++ b/cpp/examples/cpp_examples/demo_write.cpp
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "cpp_examples.h"
+
+#include <iostream>
+#include <string>
+#include <time.h>
+
+int demo_write()
+{
+ storage::TsFileWriter tsfile_writer;
+ std::string device_name = "root.db001.dev001";
+ std::string measurement_name = "m001";
+ storage::libtsfile_init();
+ tsfile_writer.open("demo_ts.tsfile",O_CREAT | O_RDWR, 0644);
+ int ret = tsfile_writer.register_timeseries(device_name, measurement_name, common::INT32, common::PLAIN, common::UNCOMPRESSED);
+ std::cout<<"register finish" << device_name<< std::endl;
+
+ int row_count = 100;
+ for (int i = 1; i < row_count; ++i) {
+ storage::DataPoint point(measurement_name, 10000 + i);
+ storage::TsRecord record(i, device_name,1);
+ record.points_.push_back(point);
+ ret = tsfile_writer.write_record(record);
+ std::cout<< "write point:" << measurement_name << " " << 10000 + i << std::endl;
+ ASSERT(ret == 0);
+ }
+
+ tsfile_writer.flush();
+ std::cout<<"finish flush"<<std::endl;
+ tsfile_writer.close();
+ std::cout<<"tsfile closed."<<std::endl;
+ storage::libtsfile_destroy();
+ std::cout<< "tsfile to destory."<<std::endl;
+ std::cout<<"finish writing" << std::endl;
+ std::cout<<"will close our files"<<std::endl;
+ return 0;
+}
diff --git a/cpp/common/include/common.hpp b/cpp/examples/examples.cc
similarity index 73%
copy from cpp/common/include/common.hpp
copy to cpp/examples/examples.cc
index 6eb9830c..e800f03e 100644
--- a/cpp/common/include/common.hpp
+++ b/cpp/examples/examples.cc
@@ -4,10 +4,10 @@
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
+ * License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * https://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -16,15 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-#ifndef TSFILE_COMMON_H
-#define TSFILE_COMMON_H
-class Common
-{
-public:
- Common();
+#include "cpp_examples/cpp_examples.h"
- int doSomething(int a, int b) const;
-};
-
-#endif
\ No newline at end of file
+int main() {
+ // C++ examples
+ demo_write();
+ demo_read();
+ return 0;
+}
\ No newline at end of file
diff --git a/cpp/examples/src/demo_read.cpp b/cpp/examples/src/demo_read.cpp
deleted file mode 100644
index a80ff801..00000000
--- a/cpp/examples/src/demo_read.cpp
+++ /dev/null
@@ -1,92 +0,0 @@
-#include <iostream>
-#include <vector>
-#include <string>
-#include "tsfile/path.h"
-#include "tsfile/filter/filter.h"
-#include "tsfile/expression.h"
-#include "tsfile/tsfile_reader.h"
-#include "tsfile/qds_with_timegenerator.h"
-#include "tsfile/qds_without_timegenerator.h"
-#include "tsfile/row_record.h"
-
-using namespace std;
-
-std::string field_to_string(timecho::storage::Field *value)
-{
- if (value->type_ == timecho::common::TEXT) {
- return std::string(value->value_.sval_);
- } else {
- std::stringstream ss;
- switch (value->type_) {
- case timecho::common::BOOLEAN:
- ss << (value->value_.bval_ ? "true" : "false");
- break;
- case timecho::common::INT32:
- ss << value->value_.ival_;
- break;
- case timecho::common::INT64:
- ss << value->value_.lval_;
- break;
- case timecho::common::FLOAT:
- ss << value->value_.fval_;
- break;
- case timecho::common::DOUBLE:
- ss << value->value_.dval_;
- break;
- case timecho::common::NULL_TYPE:
- ss << "NULL";
- break;
- default:
- ASSERT(false);
- break;
- }
- return ss.str();
- }
-}
-
-int main (int argc, char **argv)
-{
- std::cout<<"begin to read tsfile from /tmp/t1" << std::endl;
- std::string device_name = "root.db001.dev001";
- std::string measurement_name = "m001";
- timecho::storage::Path p1(device_name, measurement_name);
- std::vector<timecho::storage::Path> select_list;
- select_list.push_back(p1);
- timecho::storage::QueryExpression *query_expr =
- timecho::storage::QueryExpression::create(select_list, nullptr);
-
- timecho::common::init_config_value();
- timecho::storage::TsFileReader reader;
- int ret = reader.open("/tmp/t1");
- std::cout <<"open file return " << ret<< std::endl;
-
- std::cout << "begin to query expr" << std::endl;
- ASSERT(ret == 0);
- timecho::storage::QueryDataSet *qds = nullptr;
- ret = reader.query(query_expr, qds);
-
-
- timecho::storage::RowRecord *record;
- std::cout << "begin to dump data from tsfile ---" << std::endl;
- int row_cout = 0;
- do {
- record = qds->get_next();
- if (record) {
- std::cout<< "dump QDS : " << record->get_timestamp() << ",";
- if (record) {
- int size = record->get_fields()->size();
- for (int i = 0; i < size; ++i) {
- std::cout << field_to_string(record->get_field(i)) << ",";
- }
- std::cout<< std::endl;
- row_cout++;
- }
- } else {
- break;
- }
- } while (true);
- std::cout << "dump tsfile finish" << std::endl;
- std::cout << "total row count = " << row_cout<<std::endl;
-
- return (0);
-}
diff --git a/cpp/pom.xml b/cpp/pom.xml
index 10369c75..fb40d751 100644
--- a/cpp/pom.xml
+++ b/cpp/pom.xml
@@ -62,100 +62,46 @@
Do the actual build.
-->
<plugin>
- <groupId>com.googlecode.cmake-maven-project</groupId>
- <artifactId>cmake-maven-plugin</artifactId>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.6.0</version>
<executions>
- <!-- Generate the configuration for the test compilation -->
+ <!-- run build script -->
<execution>
- <id>cmake-generate-test-compile</id>
- <phase>generate-test-sources</phase>
+ <id>execute-compile-shell-script</id>
+ <phase>compile</phase>
<goals>
- <goal>generate</goal>
+ <goal>exec</goal>
</goals>
<configuration>
- <options>
- <option>-DUNITY_VERSION:STRING=${unity.version}</option>
- <option>-DBUILD_PHASE=test-compile</option>
- </options>
+ <executable>bash</executable>
+ <arguments>
+ <argument>build.sh</argument>
+ </arguments>
</configuration>
</execution>
- <!-- Compile the test code -->
+ <!-- run test_all.sh script -->
<execution>
- <id>cmake-execute-test-compile</id>
- <phase>test-compile</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- <!-- Execute the tests -->
- <execution>
- <id>cmake-run-tests</id>
+ <id>execute-test-shell-script</id>
<phase>test</phase>
<goals>
- <goal>test</goal>
+ <goal>exec</goal>
</goals>
<configuration>
- <buildDirectory>${project.build.directory}/build</buildDirectory>
+ <executable>bash</executable>
+ <arguments>
+ <argument>test_all.sh</argument>
+ </arguments>
</configuration>
</execution>
</executions>
- <configuration>
- <!--
- Actually the path to the CMakeList.txt file which then again
- tells to tool where to find the sources.
- -->
- <sourcePath>${project.basedir}</sourcePath>
- <!--
- Path to where the build configuration is generated
- (This directory is then used in the compile step to actually perform the build)
- -->
- <targetPath>${project.build.directory}/build</targetPath>
- <!--
- Name of the generator the compile step will be executing.
- -->
- <generator>${cmake.generator}</generator>
- <!-- The directory where the "generate" step generated the build configuration -->
- <projectDirectory>${project.build.directory}/build</projectDirectory>
- </configuration>
</plugin>
- <!-- Create a source bundle, so we can use this in the extras build -->
+ <!-- Overwrite test config-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>package-sources</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <descriptors>
- <descriptor>build-utils/source-bundle-descriptor.xml</descriptor>
- </descriptors>
- </configuration>
- </execution>
- </executions>
+ <artifactId>maven-surefire-plugin</artifactId>
</plugin>
</plugins>
- <!--
- Developers working in the cpp part of tsfile will probably use an IDE like CLion to edit just this
- Part of the project. These IDEs generate some work-directories we need to exclude.
- -->
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <excludes combine.children="append">
- <exclude>cmake-*/**</exclude>
- <exclude>.cmake/**</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
</build>
<profiles>
<!-- When running on jenkins, download the sonar build-wrapper, so we can do a code analysis -->
diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt
new file mode 100644
index 00000000..0675eb00
--- /dev/null
+++ b/cpp/src/CMakeLists.txt
@@ -0,0 +1,55 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
+message("Running in src diectory")
+include(${PROJECT_SOURCE_DIR}/cmake/build_function.cmake)
+
+add_subdirectory(common)
+add_subdirectory(compress)
+add_subdirectory(encoding)
+add_subdirectory(file)
+add_subdirectory(reader)
+add_subdirectory(utils)
+add_subdirectory(writer)
+
+add_library(tsfile SHARED)
+target_link_libraries(tsfile common_obj compress_obj file_obj read_obj write_obj )
+
+set(LIBTSFILE_PROJECT_VERSION 1.0)
+set(LIBTSFILE_SO_VERSION 1.0)
+set_target_properties(tsfile PROPERTIES VERSION ${LIBTSFILE_PROJECT_VERSION})
+set_target_properties(tsfile PROPERTIES SOVERSION ${LIBTSFILE_SO_VERSION})
+
+set(LIBTSFILE_SDK_DIR ${LIBRARY_OUTPUT_PATH})
+install(TARGETS tsfile LIBRARY DESTINATION ${LIBTSFILE_SDK_DIR})
+
+# set(CMAKE_PREFIX_PATH ../../third-party/zlib-1.2.13/install/lib)
+# set(ZLIB_LIB_DIR ../../third-party/zlib-1.2.13/install/lib)
+# # find_library(my_z_lib NAMES z PATHS ${ZLIB_LIB_DIR} NO_DEFAULT_PATH REQUIRED)
+# # target_link_libraries(tsfile ${my_z_lib})
+
+# set(CMAKE_PREFIX_PATH ../../third-party/lz4-dev/lib)
+# set(LZ4_LIB_DIR ../../third-party/lz4-dev/lib)
+# find_library(my_lz4_lib NAMES lz4 PATHS ${LZ4_LIB_DIR} NO_DEFAULT_PATH REQUIRED)
+# link_directories(${LZ4_LIB_DIR})
+# target_link_libraries(tsfile ${my_lz4_lib})
+
+# if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
+# add_custom_command(TARGET tsfile POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change `otool -L ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib | grep liblz4 | sed 's/dylib.*/dylib/g'` ${my_lz4_lib} ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib)
+# add_custom_command(TARGET tsfile POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change `otool -L ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib | grep libz | sed 's/dylib.*/dylib/g'` ${my_z_lib} ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib)
+# endif()
\ No newline at end of file
diff --git a/cpp/src/common/CMakeLists.txt b/cpp/src/common/CMakeLists.txt
new file mode 100644
index 00000000..7087d03d
--- /dev/null
+++ b/cpp/src/common/CMakeLists.txt
@@ -0,0 +1,35 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
+message("Running in src/common directory")
+
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} common_SRC_LIST)
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/allocator common_allocator_SRC_LIST)
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/container common_container_SRC_LIST)
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/tsblock common_tsblock_SRC_LIST)
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/mutex common_mutex_SRC_LIST)
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/datatype common_datatype_SRC_LIST)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+add_library(common_obj OBJECT ${common_SRC_LIST}
+ ${common_allocator_SRC_LIST}
+ ${common_container_SRC_LIST}
+ ${common_tsblock_SRC_LIST}
+ ${common_mutex_SRC_LIST}
+ ${common_datatype_SRC_LIST})
+
diff --git a/cpp/src/common/allocator/CMakeLists.txt b/cpp/src/common/allocator/CMakeLists.txt
new file mode 100644
index 00000000..e66bd212
--- /dev/null
+++ b/cpp/src/common/allocator/CMakeLists.txt
@@ -0,0 +1,26 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
+
+message("Running in src/common/allocator directory")
+
+message("CMAKE_CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}")
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} common_allocator_SRC_LIST)
+add_library(common_allocator OBJECT ${common_allocator_SRC_LIST})
+
diff --git a/cpp/src/common/allocator/alloc_base.h b/cpp/src/common/allocator/alloc_base.h
new file mode 100644
index 00000000..1ff55e19
--- /dev/null
+++ b/cpp/src/common/allocator/alloc_base.h
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_ALLOCATOR_ALLOC_BASE_H
+#define COMMON_ALLOCATOR_ALLOC_BASE_H
+
+#include <stddef.h>
+
+#include <cstring>
+
+#include "utils/util_define.h"
+
+namespace common {
+
+enum AllocModID {
+ __FIRST_MOD_ID = 0,
+ // if you are sure you will not consume too much memory, you can use
+ // MOD_DEFAULT.
+ MOD_DEFAULT = 0,
+ MOD_MEMTABLE = 1,
+ MOD_SCHEMA = 2,
+ MOD_SQL = 3,
+ MOD_NET = 4,
+ MOD_NET_DATA = 5,
+ MOD_TVLIST_DATA = 6,
+ MOD_TVLIST_OBJ = 7,
+ MOD_TSBLOCK = 8,
+ MOD_CONTAINER = 9,
+ MOD_TSSTORE_OBJ = 10,
+ MOD_FLUSH_TASK_OBJ = 11,
+ MOD_PAGE_WRITER_OUTPUT_STREAM = 12,
+ MOD_CW_PAGES_DATA = 13,
+ MOD_CHUNK_WRITER_OBJ = 14,
+ MOD_STATISTIC_OBJ = 15,
+ MOD_ENCODER_OBJ = 16,
+ MOD_DECODER_OBJ = 17,
+ MOD_TSFILE_WRITER_META = 18,
+ MOD_TSFILE_WRITE_STREAM = 19,
+ MOD_TIMESERIES_INDEX_OBJ = 20,
+ MOD_BLOOM_FILTER = 21,
+ MOD_OPEN_FILE_OBJ = 22,
+ MOD_TSFILE_READER = 23,
+ MOD_CHUNK_READER = 24,
+ MOD_COMPRESSOR_OBJ = 25,
+ MOD_ARRAY = 26,
+ MOD_HASH_TABLE = 27,
+ MOD_WRITER_INDEX_NODE = 28,
+ MOD_TS2DIFF_OBJ = 29,
+ MOD_BITENCODE_OBJ = 30,
+ MOD_DICENCODE_OBJ = 31,
+ MOD_ZIGZAG_OBJ = 32,
+ __LAST_MOD_ID = 33, // prev + 1,
+ __MAX_MOD_ID = 127, // leave 1 bit to detect header size
+};
+
+extern const char *g_mod_names[__LAST_MOD_ID];
+
+/* very basic alloc/free interface in C style */
+void *mem_alloc(uint32_t size, AllocModID mid);
+void mem_free(void *ptr);
+void *mem_realloc(void *ptr, uint32_t size);
+
+class ModStat {
+ public:
+ ModStat() : stat_arr_(NULL) {}
+
+ static ModStat &get_instance() {
+ /*
+ * This is the singleton of Mod Memory Statistic.
+ * gms is short for Global Mod Statistic
+ */
+ static ModStat gms;
+ return gms;
+ }
+ void init();
+ void destroy();
+ INLINE void update_alloc(AllocModID mid, int32_t size) {
+ // ASSERT(mid < __LAST_MOD_ID);
+ // ATOMIC_FAA(get_item(mid), size);
+ }
+ void update_free(AllocModID mid, uint32_t size) {
+ // ASSERT(mid < __LAST_MOD_ID);
+ // ATOMIC_FAA(get_item(mid), 0 - size);
+ }
+ void print_stat();
+
+#ifdef ENABLE_TEST
+ int32_t TEST_get_stat(int8_t mid) { return ATOMIC_FAA(get_item(mid), 0); }
+#endif
+
+ private:
+ INLINE int32_t *get_item(int8_t mid) {
+ return &(stat_arr_[mid * (ITEM_SIZE / sizeof(int32_t))]);
+ }
+
+ private:
+ static const int32_t ITEM_SIZE = CACHE_LINE_SIZE;
+ static const int32_t ITEM_COUNT = __LAST_MOD_ID;
+ int32_t *stat_arr_;
+
+ STATIC_ASSERT((ITEM_SIZE % sizeof(int32_t) == 0), ModStat_ITEM_SIZE_ERROR);
+};
+
+/* base allocator */
+class BaseAllocator {
+ public:
+ void *alloc(uint32_t size, AllocModID mid) { return mem_alloc(size, mid); }
+ void free(void *ptr) { mem_free(ptr); }
+};
+
+extern BaseAllocator g_base_allocator;
+
+} // end namespace common
+
+#endif // COMMON_ALLOCATOR_ALLOC_BASE_H
diff --git a/cpp/src/common/allocator/byte_stream.h b/cpp/src/common/allocator/byte_stream.h
new file mode 100644
index 00000000..04a30098
--- /dev/null
+++ b/cpp/src/common/allocator/byte_stream.h
@@ -0,0 +1,1109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_ALLOCATOR_BYTE_STREAM_H
+#define COMMON_ALLOCATOR_BYTE_STREAM_H
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <iostream>
+#include <string>
+
+#include "common/allocator/alloc_base.h"
+#include "common/allocator/my_string.h"
+#include "utils/errno_define.h"
+
+namespace common {
+
+template <typename T>
+class OptionalAtomic {
+ public:
+ OptionalAtomic(T t, bool enable_atomic = false)
+ : val_(t), enable_atomic_(enable_atomic) {}
+
+ FORCE_INLINE T load() const {
+ if (UNLIKELY(enable_atomic_)) {
+ return ATOMIC_LOAD(&val_);
+ } else {
+ return val_;
+ }
+ }
+
+ FORCE_INLINE void store(const T t) {
+ if (UNLIKELY(enable_atomic_)) {
+ ATOMIC_STORE(&val_, t);
+ } else {
+ val_ = t;
+ }
+ }
+
+ FORCE_INLINE T atomic_faa(const T increament) {
+ if (UNLIKELY(enable_atomic_)) {
+ return ATOMIC_FAA(&val_, increament);
+ } else {
+ T old_val = val_;
+ val_ = val_ + increament;
+ return old_val;
+ }
+ }
+
+ FORCE_INLINE T atomic_aaf(const T increament) {
+ if (UNLIKELY(enable_atomic_)) {
+ return ATOMIC_AAF(&val_, increament);
+ } else {
+ val_ = val_ + increament;
+ return val_;
+ }
+ }
+
+ FORCE_INLINE bool enable_atomic() const { return enable_atomic_; }
+
+ private:
+ T val_;
+ bool enable_atomic_;
+};
+
+FORCE_INLINE int32_t float_to_int(float f) {
+ // return *((int32_t*)(&f));
+ // return *(reinterpret_cast<int32_t*>(&f));
+ union fi {
+ int i_;
+ float f_;
+ };
+
+ fi my_fi;
+ my_fi.f_ = f;
+ return my_fi.i_; // By mimicking the java jdk's implementation, it's
+ // essentially the same as line 75.
+}
+
+FORCE_INLINE float int_to_float(int32_t i) {
+ // return *((float *)(&i));
+ union fi {
+ int i_;
+ float f_;
+ };
+
+ fi my_fi;
+ my_fi.i_ = i;
+ return my_fi.f_;
+}
+
+FORCE_INLINE void float_to_bytes(float f, uint8_t bytes[4]) {
+ /*
+ * See:
+ * floatToBytes in BytesUtils.java of IoTDB project and
+ * Java_java_lang_Float_intBitsToFloat in JDK project
+ */
+ if (UNLIKELY(f != f)) { // this is NaN
+ // IEEE754: 0x7FC00000 for NanN
+ bytes[0] = 0x7F;
+ bytes[1] = 0xC0;
+ bytes[2] = 0x00;
+ bytes[3] = 0x00;
+ return;
+ }
+
+ // follow jdk implementation style.
+ union {
+ int i;
+ float f;
+ } u;
+
+ u.f = f;
+
+ bytes[3] = (uint8_t)(u.i);
+ bytes[2] = (uint8_t)(u.i >> 8);
+ bytes[1] = (uint8_t)(u.i >> 16);
+ bytes[0] = (uint8_t)(u.i >> 24);
+}
+
+FORCE_INLINE float bytes_to_float(uint8_t bytes[4]) {
+ int32_t i = bytes[3];
+ i &= 0xFF;
+ i |= bytes[2] << 8;
+ i &= 0xFFFF;
+ i |= bytes[1] << 16;
+ i &= 0xFFFFFF;
+ i |= bytes[0] << 24;
+
+ union {
+ int i;
+ float f;
+ } u;
+
+ u.i = i;
+ return u.f;
+}
+
+FORCE_INLINE int64_t double_to_long(double d) {
+ // return *((int64_t*)(&d));
+ union dl {
+ double d_;
+ int64_t l_;
+ };
+ dl my_dl;
+ my_dl.d_ = d;
+ return my_dl.l_;
+}
+
+FORCE_INLINE double long_to_double(int64_t l) {
+ // return *((double *)(&l));
+ union dl {
+ double d_;
+ int64_t l_;
+ };
+ dl my_dl;
+ my_dl.l_ = l;
+ return my_dl.d_;
+}
+
+FORCE_INLINE void double_to_bytes(double d, uint8_t bytes[8]) {
+ if (UNLIKELY(d != d)) {
+ // NaN, 0x7FF8000000000000L
+ memset(bytes, 0, 8);
+ bytes[0] = 0x7F;
+ bytes[1] = 0xF8;
+ return;
+ }
+ // follow jdk implementation style.
+ union {
+ long long l;
+ double d;
+ } u;
+
+ u.d = d;
+
+ bytes[7] = (uint8_t)u.l;
+ bytes[6] = (uint8_t)(u.l >> 8);
+ bytes[5] = (uint8_t)(u.l >> 16);
+ bytes[4] = (uint8_t)(u.l >> 24);
+ bytes[3] = (uint8_t)(u.l >> 32);
+ bytes[2] = (uint8_t)(u.l >> 40);
+ bytes[1] = (uint8_t)(u.l >> 48);
+ bytes[0] = (uint8_t)(u.l >> 56);
+}
+
+FORCE_INLINE double bytes_to_double(uint8_t bytes[8]) {
+ int64_t value = bytes[7];
+ value &= 0xFFl;
+ value |= ((int64_t)bytes[6]) << 8;
+ value &= 0xFFFFl;
+ value |= ((int64_t)bytes[5]) << 16;
+ value &= 0xFFFFFFl;
+ value |= ((int64_t)bytes[4]) << 24;
+ value &= 0xFFFFFFFFl;
+ value |= ((int64_t)bytes[3]) << 32;
+ value &= 0xFFFFFFFFFFl;
+ value |= ((int64_t)bytes[2]) << 40;
+ value &= 0xFFFFFFFFFFFFl;
+ value |= ((int64_t)bytes[1]) << 48;
+ value &= 0xFFFFFFFFFFFFFFl;
+ value |= ((int64_t)bytes[0]) << 56;
+
+ // follow jdk implementation style.
+ union {
+ long l;
+ double d;
+ } u;
+
+ u.l = value;
+ return u.d;
+}
+
+// TODO define a WrappedByteStream class
+
+// auto extend buffer for serialization
+class ByteStream {
+ private:
+ struct Page {
+ OptionalAtomic<Page *> next_; // 9 bytes
+ uint8_t *buf_; // 8 bytes
+
+ explicit Page(bool enable_atomic) : next_(nullptr, enable_atomic) {
+ buf_ = (uint8_t
+ *)(this +
+ 1); // I think it should add 17, because the Page
+ // class is 9+8=17 bytes. No, adding one to a
+ // pointer is not adding a byte, but adding the
+ // length of the byte corresponding to the type
+ // pointed to by the pointer, and adding one
+ // here is actually adding 17 bytes. No problem.
+ }
+ Page(bool enable_atomic, uint8_t *wrapped_buf)
+ : next_(nullptr, enable_atomic), buf_(wrapped_buf) {}
+ };
+
+ public:
+ ByteStream(uint32_t page_size, AllocModID mid, bool enable_atomic = false,
+ BaseAllocator &allocator = g_base_allocator)
+ : allocator_(allocator),
+ head_(nullptr, enable_atomic),
+ tail_(nullptr, enable_atomic),
+ read_page_(nullptr),
+ total_size_(0, enable_atomic),
+ read_pos_(0),
+ marked_read_pos_(0),
+ page_size_(page_size),
+ mid_(mid),
+ wrapped_page_(false, nullptr) {
+ // assert(page_size >= 16); // commented out by gxh on 2023.03.09
+ }
+
+ // TODO use a specific construct function to mark it as wrapped use.
+ // for wrap plain buffer to ByteStream
+ ByteStream()
+ : allocator_(g_base_allocator),
+ head_(nullptr, false),
+ tail_(nullptr, false),
+ read_page_(nullptr),
+ total_size_(0, false),
+ read_pos_(0),
+ marked_read_pos_(0),
+ page_size_(0),
+ mid_(MOD_DEFAULT),
+ wrapped_page_(false, nullptr) {}
+
+ ~ByteStream() { destroy(); }
+
+ /* ================ Part 0: wrap from outside buffer ================ */
+ // if you wrap a buffer as a ByteStream, you should
+ // manage the outside buffer yourself.
+ void wrap_from(const char *buf, int32_t buf_len) {
+ wrapped_page_.next_.store(nullptr);
+ wrapped_page_.buf_ = (uint8_t *)buf;
+
+ page_size_ = buf_len;
+ head_.store(&wrapped_page_);
+ tail_.store(&wrapped_page_);
+ total_size_.store(buf_len);
+ marked_read_pos_ = 0;
+ read_pos_ = 0;
+ read_page_ = nullptr;
+ }
+ FORCE_INLINE bool is_wrapped() const {
+ return head_.load() == &wrapped_page_;
+ }
+ char *get_wrapped_buf() { return (char *)wrapped_page_.buf_; }
+ void clear_wrapped_buf() { wrapped_page_.buf_ = nullptr; }
+
+ /* ================ Part 1: basic ================ */
+ FORCE_INLINE uint32_t remaining_size() const {
+ ASSERT(total_size_.load() >= read_pos_);
+ return total_size_.load() - read_pos_;
+ }
+ FORCE_INLINE bool has_remaining() const { return remaining_size() > 0; }
+
+ FORCE_INLINE void mark_read_pos() { marked_read_pos_ = read_pos_; }
+ FORCE_INLINE uint32_t get_mark_len() const {
+ ASSERT(marked_read_pos_ <= read_pos_);
+ return read_pos_ - marked_read_pos_;
+ }
+
+ void destroy() { reset(); }
+ void reset() {
+ // if this ByteStream is wrapped from a plain buffer
+ // do not free the outside buffer.
+ if (!is_wrapped()) {
+ Page *p = head_.load();
+ while (p) {
+ p = head_.load()->next_.load();
+ allocator_.free(head_.load());
+ head_.store(p);
+ }
+ }
+ head_.store(nullptr);
+ tail_.store(nullptr);
+ read_page_ = nullptr;
+ total_size_.store(0);
+ read_pos_ = 0;
+ }
+
+ // never used TODO
+ void shallow_clone_from(ByteStream &other) {
+ this->page_size_ = other.page_size_;
+ this->mid_ = other.mid_;
+ this->head_.store(other.head_.load());
+ this->tail_.store(other.tail_.load());
+ this->total_size_.store(other.total_size_.load());
+ }
+
+ FORCE_INLINE uint32_t total_size() const { return total_size_.load(); }
+ FORCE_INLINE uint32_t read_pos() const { return read_pos_; };
+ FORCE_INLINE void wrapped_buf_advance_read_pos(uint32_t size) {
+ if (size + read_pos_ > total_size_.load()) {
+ read_pos_ = total_size_.load();
+ } else {
+ read_pos_ += size;
+ }
+ }
+
+ /* ================ Part 2: write_xxx and read_xxx ================ */
+ // writer @buf with length @len into this bytestream
+ int write_buf(const uint8_t *buf, const uint32_t len) {
+ int ret = common::E_OK;
+ uint32_t write_len = 0;
+ while (write_len < len) {
+ if (RET_FAIL(prepare_space())) {
+ std::cout << "write_buf error " << ret << std::endl;
+ return ret;
+ }
+ uint32_t remainder = page_size_ - (total_size_.load() % page_size_);
+ uint32_t copy_len =
+ remainder < (len - write_len) ? remainder : (len - write_len);
+ memcpy(tail_.load()->buf_ + total_size_.load() % page_size_,
+ buf + write_len, copy_len);
+ total_size_.atomic_aaf(copy_len);
+ write_len += copy_len;
+ }
+ return ret;
+ }
+
+ // reader @want_len bytes to @buf, @read_len indicates real len we reader.
+ // if ByteStream do not have so many bytes, it will return E_PARTIAL_READ if
+ // no other error occure.
+ int read_buf(uint8_t *buf, const uint32_t want_len, uint32_t &read_len) {
+ int ret = common::E_OK;
+ bool partial_read = (read_pos_ + want_len > total_size_.load());
+ uint32_t want_len_limited =
+ partial_read ? (total_size_.load() - read_pos_) : want_len;
+ read_len = 0;
+ while (read_len < want_len_limited) {
+ if (RET_FAIL(check_space())) {
+ return ret;
+ }
+ uint32_t remainder = page_size_ - (read_pos_ % page_size_);
+ uint32_t copy_len = remainder < want_len_limited - read_len
+ ? remainder
+ : want_len_limited - read_len;
+ memcpy(buf + read_len, read_page_->buf_ + (read_pos_ % page_size_),
+ copy_len);
+ read_len += copy_len;
+ read_pos_ += copy_len;
+ }
+ return partial_read ? common::E_PARTIAL_READ : common::E_OK;
+ }
+
+ FORCE_INLINE int write_buf(const char *buf, const uint32_t len) {
+ return write_buf((const uint8_t *)buf, len);
+ }
+ FORCE_INLINE int read_buf(char *buf, const uint32_t want_len,
+ uint32_t &read_len) {
+ return read_buf((uint8_t *)buf, want_len, read_len);
+ }
+ FORCE_INLINE int read_buf(char *buf, const int32_t want_len,
+ int32_t &read_len) {
+ return read_buf((uint8_t *)buf, (uint32_t &)want_len,
+ (uint32_t &)read_len);
+ }
+
+ void purge_prev_pages(int purge_page_count = INT32_MAX) {
+ Page *cur = head_.load();
+ while (cur != tail_.load() && purge_page_count-- > 0) {
+ Page *next = cur->next_.load();
+ allocator_.free(cur);
+ cur = next;
+ }
+ head_.store(cur);
+ }
+
+ /* ================ Part 3: writing internal buffers ================ */
+ /*
+ * use @acquire_buf function to get a buf, caller will fill the buf, and
+ * after filling, caller use @buffer_used to notice ByteStream to update
+ * internal variables.
+ *
+ * Note: it should be used in single thread.
+ */
+ struct Buffer {
+ char *buf_;
+ uint32_t len_;
+
+ Buffer() : buf_(nullptr), len_(0) {}
+ };
+
+ Buffer acquire_buf() {
+ Buffer b;
+ if (common::E_OK != prepare_space()) {
+ return b;
+ }
+ b.buf_ =
+ (char *)(tail_.load()->buf_ + (total_size_.load() % page_size_));
+ b.len_ = page_size_ - (total_size_.load() % page_size_);
+ return b;
+ }
+
+ void buffer_used(uint32_t used_bytes) {
+ ASSERT(used_bytes >= 1);
+ // would not span page
+ ASSERT((total_size_.load() / page_size_) ==
+ ((total_size_.load() + used_bytes - 1) / page_size_));
+ total_size_.atomic_aaf(used_bytes);
+ }
+
+ /* ================ Part 4: reading internal buffers ================ */
+ /*
+ * one-shot reader iterator
+ * Do not use it if the host_ is still writing
+ */
+ struct BufferIterator {
+ const ByteStream &host_;
+ Page *cur_;
+ Page *end_;
+ int64_t total_size_;
+ BufferIterator(const ByteStream &bs) : host_(bs) {
+ cur_ = bs.head_.load();
+ end_ = bs.tail_.load();
+ total_size_ = bs.total_size_.load();
+ }
+
+ Buffer get_next_buf() {
+ Buffer b;
+ if (cur_ != nullptr) {
+ b.buf_ = (char *)cur_->buf_;
+ if (cur_ == end_ &&
+ host_.total_size_.load() % host_.page_size_ != 0) {
+ b.len_ = host_.total_size_.load() % host_.page_size_;
+ } else {
+ b.len_ = host_.page_size_;
+ }
+ ASSERT(b.len_ > 0);
+ cur_ = cur_->next_.load();
+ }
+ return b;
+ }
+ };
+ BufferIterator init_buffer_iterator() const {
+ return BufferIterator(*this);
+ }
+
+ /*
+ * producer-consumer mode reader iterator
+ * Note:
+ * the host ByteStream should be configured as atomically.
+ */
+ struct Consumer {
+ const ByteStream &host_;
+ Page *cur_;
+ uint32_t read_offset_within_cur_page_;
+ int64_t total_end_offset_; // for DEBUG
+
+ Consumer(const ByteStream &bs) : host_(bs) {
+ ASSERT(bs.head_.enable_atomic());
+ cur_ = nullptr;
+ read_offset_within_cur_page_ = 0;
+ total_end_offset_ = 0;
+ }
+
+ Buffer get_next_buf(ByteStream &host) {
+ Buffer b;
+ if (UNLIKELY(host.head_.load() == nullptr)) {
+ // host is empty, return empty buffer.
+ return b;
+ }
+ if (UNLIKELY(cur_ == nullptr)) {
+ // this consumer did not initialiazed.
+ cur_ = host_.head_.load();
+ read_offset_within_cur_page_ = 0;
+ }
+
+ // get tail position <tail_, total_size_> atomically
+ Page *host_end = nullptr;
+ uint32_t host_total_size = 0;
+ while (true) {
+ host_end = host_.tail_.load();
+ host_total_size = host_.total_size_.load();
+ if (host_end == host_.tail_.load()) {
+ break;
+ }
+ }
+
+ while (true) {
+ if (cur_ == host_end) {
+ if (host_total_size % host_.page_size_ == 0) {
+ if (read_offset_within_cur_page_ == host_.page_size_) {
+ return b;
+ } else {
+ b.buf_ = ((char *)(cur_->buf_)) +
+ read_offset_within_cur_page_;
+ b.len_ =
+ host_.page_size_ - read_offset_within_cur_page_;
+ read_offset_within_cur_page_ = host_.page_size_;
+ total_end_offset_ += b.len_;
+ return b;
+ }
+ } else {
+ if (read_offset_within_cur_page_ ==
+ (host_total_size % host_.page_size_)) {
+ return b;
+ } else {
+ b.buf_ = ((char *)(cur_->buf_)) +
+ read_offset_within_cur_page_;
+ b.len_ = (host_total_size % host_.page_size_) -
+ read_offset_within_cur_page_;
+ read_offset_within_cur_page_ =
+ (host_total_size % host_.page_size_);
+ total_end_offset_ += b.len_;
+ return b;
+ }
+ }
+ } else {
+ if (read_offset_within_cur_page_ == host_.page_size_) {
+ cur_ = cur_->next_.load();
+ read_offset_within_cur_page_ = 0;
+ } else {
+ b.buf_ = ((char *)(cur_->buf_)) +
+ read_offset_within_cur_page_;
+ b.len_ =
+ host_.page_size_ - read_offset_within_cur_page_;
+ cur_ = cur_->next_.load();
+ read_offset_within_cur_page_ = 0;
+ total_end_offset_ += b.len_;
+ return b;
+ }
+ }
+ }
+ ASSERT(b.len_ < (1 << 30));
+ return b;
+ }
+ };
+
+ private:
+ FORCE_INLINE int prepare_space() {
+ int ret = common::E_OK;
+ if (UNLIKELY(tail_.load() == nullptr ||
+ total_size_.load() % page_size_ == 0)) {
+ Page *p = nullptr;
+ if (RET_FAIL(alloc_page(p))) {
+ return ret;
+ }
+ } else {
+ // do nothing
+ }
+ return ret;
+ }
+
+ FORCE_INLINE int check_space() {
+ if (UNLIKELY(read_pos_ >= total_size_.load())) {
+ return common::E_OUT_OF_RANGE;
+ }
+ if (UNLIKELY(read_page_ == nullptr)) {
+ read_page_ = head_.load();
+ } else if (UNLIKELY(read_pos_ % page_size_ == 0)) {
+ read_page_ = read_page_->next_.load();
+ }
+ if (UNLIKELY(read_page_ == nullptr)) {
+ return common::E_OUT_OF_RANGE;
+ }
+ return common::E_OK;
+ }
+
+ FORCE_INLINE int alloc_page(Page *&p) {
+ int ret = common::E_OK;
+ char *buf = (char *)allocator_.alloc(page_size_ + sizeof(Page), mid_);
+ if (UNLIKELY(buf == nullptr)) {
+ ret = common::E_OOM;
+ } else {
+ p = new (buf) Page(head_.enable_atomic());
+ p->next_.store(nullptr);
+ if (head_.load()) {
+ tail_.load()->next_.store(p);
+ tail_.store(p);
+ } else {
+ head_.store(p);
+ tail_.store(p);
+ }
+ }
+ // printf("\nByteStream alloc_page, this=%p, new_page=%p\n", this, p);
+ return ret;
+ }
+
+ DISALLOW_COPY_AND_ASSIGN(ByteStream);
+
+ private:
+ BaseAllocator &allocator_;
+ OptionalAtomic<Page *> head_;
+ OptionalAtomic<Page *> tail_;
+ Page *read_page_; // only one thread is allow to reader this ByteStream
+ OptionalAtomic<uint32_t> total_size_; // total size in byte
+ uint32_t read_pos_; // current reader position
+ uint32_t marked_read_pos_; // current reader position
+ uint32_t page_size_;
+ AllocModID mid_;
+ Page wrapped_page_;
+};
+
+FORCE_INLINE int merge_byte_stream(ByteStream &sea, ByteStream &river,
+ bool purge_river = false) {
+ int ret = common::E_OK;
+ ByteStream::BufferIterator buf_iter = river.init_buffer_iterator();
+ while (true) {
+ ByteStream::Buffer buf = buf_iter.get_next_buf();
+ if (buf.buf_ == nullptr) {
+ break;
+ } else {
+ if (RET_FAIL(sea.write_buf(buf.buf_, buf.len_))) {
+ break;
+ }
+ }
+ if (purge_river) {
+ river.purge_prev_pages(1);
+ }
+ }
+ return ret;
+}
+
+FORCE_INLINE uint32_t get_var_uint_size(
+ uint32_t
+ ui32) // return: the length of usigned number after varint encoding.
+{
+ uint32_t bytes = 0;
+ while ((ui32 & 0xFFFFFF80) != 0) {
+ bytes++;
+ ui32 = ui32 >> 7;
+ }
+ return ++bytes;
+}
+
+#if DEBUG_SE
+FORCE_INLINE void DEBUG_print_byte_stream(const char *print_tag,
+ ByteStream &b) {
+ const int32_t WRAP_COUNT = 16;
+ int32_t print_char_count = 0;
+ ByteStream::BufferIterator buf_iter = b.init_buffer_iterator();
+ while (true) {
+ ByteStream::Buffer buf = buf_iter.get_next_buf();
+ if (buf.buf_ == nullptr) {
+ break;
+ } else {
+ // print_hex(buf.buf_, buf.len_);
+ printf("\n %s: buf.buf_=%p, buf.len_=%u\n", print_tag, buf.buf_,
+ buf.len_);
+ for (uint32_t i = 0; i < buf.len_; i++) {
+ if (print_char_count++ % WRAP_COUNT == 0) {
+ printf("\n %s offset=0x%05x : ", print_tag,
+ print_char_count - 1);
+ }
+ printf("%02x ", (uint8_t)buf.buf_[i]);
+ }
+ }
+ }
+ printf("\n\n");
+}
+
+FORCE_INLINE void DEBUG_hex_dump_buf(const char *print_tag, const char *buf,
+ int32_t len) {
+ const int32_t WRAP_COUNT = 16;
+ for (int i = 0; i < len; i++) {
+ if (i % WRAP_COUNT == 0) {
+ printf("\n%s", print_tag);
+ }
+ printf("%02x ", (uint8_t)(buf[i]));
+ }
+ printf("\n");
+}
+#endif // ifndef NDEBUG
+
+class SerializationUtil {
+ public:
+ FORCE_INLINE static int write_ui8(uint8_t ui8, ByteStream &out) {
+ return out.write_buf(&ui8, 1);
+ }
+ FORCE_INLINE static int write_ui16(uint16_t ui16, ByteStream &out) {
+ uint8_t buf[2];
+ buf[0] = (uint8_t)((ui16 >> 8) & 0xFF);
+ buf[1] = (uint8_t)((ui16)&0xFF);
+ return out.write_buf(buf, 2);
+ }
+ FORCE_INLINE static int write_ui32(uint32_t ui32, ByteStream &out) {
+ uint8_t buf[4];
+ buf[0] = (uint8_t)((ui32 >> 24) & 0xFF);
+ buf[1] = (uint8_t)((ui32 >> 16) & 0xFF);
+ buf[2] = (uint8_t)((ui32 >> 8) & 0xFF);
+ buf[3] = (uint8_t)((ui32)&0xFF);
+ return out.write_buf(buf, 4);
+ }
+ FORCE_INLINE static int write_ui64(uint64_t ui64, ByteStream &out) {
+ // big-endian: most signification byte at smaller address
+ // refer to tsfile.utils.BytesUtil
+ uint8_t buf[8];
+ buf[0] = (uint8_t)((ui64 >> 56) & 0xFF);
+ buf[1] = (uint8_t)((ui64 >> 48) & 0xFF);
+ buf[2] = (uint8_t)((ui64 >> 40) & 0xFF);
+ buf[3] = (uint8_t)((ui64 >> 32) & 0xFF);
+ buf[4] = (uint8_t)((ui64 >> 24) & 0xFF);
+ buf[5] = (uint8_t)((ui64 >> 16) & 0xFF);
+ buf[6] = (uint8_t)((ui64 >> 8) & 0xFF);
+ buf[7] = (uint8_t)((ui64)&0xFF);
+ return out.write_buf(buf, 8);
+ }
+
+ FORCE_INLINE static int read_ui8(uint8_t &ui8, ByteStream &in) {
+ int ret = common::E_OK;
+ char buf[1];
+ uint32_t read_len = 0;
+ ret = in.read_buf(buf, 1, read_len);
+ ui8 = (uint8_t)buf[0];
+ return ret;
+ }
+ FORCE_INLINE static int read_ui16(uint16_t &ui16, ByteStream &in) {
+ int ret = common::E_OK;
+ uint8_t buf[2];
+ uint32_t read_len = 0;
+ if (RET_FAIL(in.read_buf(buf, 2, read_len))) {
+ return ret;
+ }
+ ui16 = buf[0];
+ ui16 = (ui16 << 8) | buf[1];
+ return ret;
+ }
+ FORCE_INLINE static int read_ui32(uint32_t &ui32, ByteStream &in) {
+ int ret = common::E_OK;
+ uint8_t buf[4];
+ uint32_t read_len = 0;
+ if (RET_FAIL(in.read_buf(buf, 4, read_len))) {
+ return ret;
+ }
+ ui32 = buf[0];
+ ui32 = (ui32 << 8) | (buf[1] & 0xFF);
+ ui32 = (ui32 << 8) | (buf[2] & 0xFF);
+ ui32 = (ui32 << 8) | (buf[3] & 0xFF);
+ return ret;
+ }
+ FORCE_INLINE static int read_ui64(uint64_t &ui64, ByteStream &in) {
+ int ret = common::E_OK;
+ uint8_t buf[8];
+ uint32_t read_len = 0;
+ if (RET_FAIL(in.read_buf(buf, 8, read_len))) {
+ return ret;
+ }
+ ui64 = buf[0];
+ ui64 = (ui64 << 8) | (buf[1] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[2] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[3] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[4] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[5] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[6] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[7] & 0xFF);
+ return ret;
+ }
+ // caller guarantee buffer has at least 1 byte
+ FORCE_INLINE static uint8_t read_ui8(char *buffer) {
+ return *(uint8_t *)buffer;
+ }
+
+ // caller guarantee buffer has at least 2 bytes
+ FORCE_INLINE static uint16_t read_ui16(char *buffer) {
+ uint8_t *buf = (uint8_t *)buffer;
+ uint16_t ui16 = buf[0];
+ ui16 = (ui16 << 8) | (buf[1] & 0xFF);
+ return ui16;
+ }
+ // caller guarantee buffer has at least 4 bytes
+ FORCE_INLINE static uint32_t read_ui32(char *buffer) {
+ uint8_t *buf = (uint8_t *)buffer;
+ uint32_t ui32 = buf[0];
+ ui32 = (ui32 << 8) | (buf[1] & 0xFF);
+ ui32 = (ui32 << 8) | (buf[2] & 0xFF);
+ ui32 = (ui32 << 8) | (buf[3] & 0xFF);
+ return ui32;
+ }
+ // caller guarantee buffer has at least 8 bytes
+ FORCE_INLINE static uint64_t read_ui64(char *buffer) {
+ uint8_t *buf = (uint8_t *)buffer;
+ uint64_t ui64 = buf[0];
+ ui64 = (ui64 << 8) | (buf[1] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[2] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[3] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[4] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[5] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[6] & 0xFF);
+ ui64 = (ui64 << 8) | (buf[7] & 0xFF);
+ return ui64;
+ }
+
+ FORCE_INLINE static int write_float(float f, ByteStream &out) {
+ uint8_t bytes[4];
+ float_to_bytes(f, bytes);
+ return out.write_buf(bytes, 4);
+ }
+ FORCE_INLINE static int read_float(float &f, ByteStream &in) {
+ int ret = common::E_OK;
+ uint8_t bytes[4];
+ uint32_t read_len = 0;
+ if (RET_FAIL(in.read_buf(bytes, 4, read_len))) {
+ } else if (read_len != 4) {
+ ret = common::E_BUF_NOT_ENOUGH;
+ } else {
+ f = bytes_to_float(bytes);
+ }
+ return ret;
+ }
+ FORCE_INLINE static float read_float(char *buffer) {
+ uint8_t *buf = (uint8_t *)buffer;
+ return bytes_to_float(buf);
+ }
+ FORCE_INLINE static int write_double(double d, ByteStream &out) {
+ uint8_t bytes[8];
+ double_to_bytes(d, bytes);
+ return out.write_buf(bytes, 8);
+ }
+ FORCE_INLINE static int read_double(double &d, ByteStream &in) {
+ int ret = common::E_OK;
+ uint32_t read_len = 0;
+ uint8_t bytes[8];
+ if (RET_FAIL(in.read_buf(bytes, 8, read_len))) {
+ } else if (read_len != 8) {
+ ret = common::E_BUF_NOT_ENOUGH;
+ } else {
+ d = bytes_to_double(bytes);
+ }
+ return ret;
+ }
+ FORCE_INLINE static double read_double(char *buffer) {
+ uint8_t *buf = (uint8_t *)buffer;
+ return bytes_to_double(buf);
+ }
+
+ FORCE_INLINE static int write_i8(int8_t i8, ByteStream &out) {
+ return write_ui8((uint8_t)i8, out);
+ }
+ FORCE_INLINE static int write_i16(int16_t i16, ByteStream &out) {
+ return write_ui16((uint16_t)i16, out);
+ }
+ FORCE_INLINE static int write_i32(int32_t i32, ByteStream &out) {
+ return write_ui32((uint32_t)i32, out);
+ }
+ FORCE_INLINE static int write_i64(int64_t i64, ByteStream &out) {
+ return write_ui64((uint64_t)i64, out);
+ }
+
+ FORCE_INLINE static int read_i8(int8_t &i8, ByteStream &in) {
+ return read_ui8((uint8_t &)i8, in);
+ }
+ FORCE_INLINE static int read_i16(int16_t &i16, ByteStream &in) {
+ return read_ui16((uint16_t &)i16, in);
+ }
+ FORCE_INLINE static int read_i32(int32_t &i32, ByteStream &in) {
+ return read_ui32((uint32_t &)i32, in);
+ }
+ FORCE_INLINE static int read_i64(int64_t &i64, ByteStream &in) {
+ return read_ui64((uint64_t &)i64, in);
+ }
+
+ // TODO more test on var_xxx
+ FORCE_INLINE static int do_write_var_uint(uint32_t ui32, ByteStream &out) {
+ int ret = common::E_OK;
+ while ((ui32 & 0xFFFFFF80) != 0) {
+ if (RET_FAIL(write_ui8((ui32 & 0x7F) | 0x80, out))) {
+ return ret;
+ }
+ ui32 = ui32 >> 7;
+ }
+ return write_ui8(ui32 & 0x7F, out);
+ }
+ FORCE_INLINE static int do_write_var_uint(uint32_t ui32, char *out_buf,
+ const uint32_t out_buf_len) {
+ uint32_t offset = 0;
+ while ((ui32 & 0xFFFFFF80) != 0) {
+ if (offset >= out_buf_len) {
+ return common::E_BUF_NOT_ENOUGH;
+ }
+ *(out_buf + offset) = (ui32 & 0x7F) | 0x80;
+ ui32 = ui32 >> 7;
+ offset++;
+ }
+ if (offset >= out_buf_len) {
+ return common::E_BUF_NOT_ENOUGH;
+ }
+ *(out_buf + offset) = (ui32 & 0x7F);
+ return common::E_OK;
+ }
+ FORCE_INLINE static int do_read_var_uint(uint32_t &ui32, ByteStream &in) {
+ // Follow readUnsignedVarInt in ReadWriteForEncodingUtils.java
+ int ret = common::E_OK;
+ ui32 = 0;
+ int i = 0;
+ uint8_t ui8 = 0;
+ if (RET_FAIL(read_ui8(ui8, in))) {
+ return ret;
+ }
+ while (ui8 != 0xF && (ui8 & 0x80) != 0) {
+ ui32 = ui32 | ((ui8 & 0x7F) << i);
+ i = i + 7;
+ if (RET_FAIL(read_ui8(ui8, in))) {
+ return ret;
+ }
+ }
+ ui32 = ui32 | (ui8 << i);
+ return ret;
+ }
+ FORCE_INLINE static int do_read_var_uint(uint32_t &ui32, char *in_buf,
+ int in_buf_len, int *ret_offset) {
+ ui32 = 0;
+ int i = 0;
+ uint8_t ui8 = 0;
+ int offset = 0;
+ if (offset < in_buf_len) {
+ ui8 = *(uint8_t *)(in_buf + offset);
+ offset++;
+ } else {
+ return common::E_BUF_NOT_ENOUGH;
+ }
+ while (ui8 != 0xF && (ui8 & 0x80) != 0) {
+ ui32 = ui32 | ((ui8 & 0x7F) << i);
+ i = i + 7;
+ if (offset < in_buf_len) {
+ ui8 = *(uint8_t *)(in_buf + offset);
+ offset++;
+ } else {
+ return common::E_BUF_NOT_ENOUGH;
+ }
+ }
+ ui32 = ui32 | (ui8 << i);
+ if (ret_offset != nullptr) {
+ *ret_offset = offset;
+ }
+ return common::E_OK;
+ }
+ FORCE_INLINE static int write_var_int(int32_t i32, ByteStream &out) {
+ // TODO 8byte to 4byte.
+ // but in IoTDB java, it has only write_var_uint(i32)
+ i32 = i32 << 1;
+ return do_write_var_uint((uint32_t)i32, out);
+ }
+ FORCE_INLINE static int read_var_int(int32_t &i32, ByteStream &in) {
+ int ret = common::E_OK;
+ uint32_t ui32;
+ if (RET_FAIL(do_read_var_uint(ui32, in))) {
+ } else {
+ i32 = (int32_t)(ui32 >> 1);
+ }
+ return ret;
+ }
+ FORCE_INLINE static int write_var_uint(uint32_t ui32, ByteStream &out) {
+ return do_write_var_uint(ui32, out);
+ }
+ FORCE_INLINE static int write_var_uint(uint32_t ui32, char *out_buf,
+ const uint32_t out_buf_len) {
+ return do_write_var_uint(ui32, out_buf, out_buf_len);
+ }
+ FORCE_INLINE static int read_var_uint(uint32_t &ui32, ByteStream &in) {
+ return do_read_var_uint(ui32, in);
+ }
+ FORCE_INLINE static int read_var_uint(uint32_t &ui32, char *in_buf,
+ int in_buf_len,
+ int *ret_offset = nullptr) {
+ return do_read_var_uint(ui32, in_buf, in_buf_len, ret_offset);
+ }
+
+ FORCE_INLINE static int write_str(const std::string &str, ByteStream &out) {
+ int ret = common::E_OK;
+ if (RET_FAIL(write_var_int(((int32_t)str.size()), out))) {
+ } else if (RET_FAIL(out.write_buf(str.c_str(), str.size()))) {
+ }
+ return ret;
+ }
+ FORCE_INLINE static int read_str(std::string &str, ByteStream &in) {
+ int ret = common::E_OK;
+ int32_t len = 0;
+ int32_t read_len = 0;
+ if (RET_FAIL(read_var_int(len, in))) {
+ } else {
+ char *tmp_buf = (char *)malloc(len + 1);
+ tmp_buf[len] = '\0';
+ if (RET_FAIL(in.read_buf(tmp_buf, len, read_len))) {
+ } else if (len != read_len) {
+ ret = E_BUF_NOT_ENOUGH;
+ } else {
+ str = std::string(tmp_buf);
+ }
+ free(tmp_buf);
+ }
+ return ret;
+ }
+ FORCE_INLINE static int write_mystring(const String &str, ByteStream &out) {
+ int ret = common::E_OK;
+ if (RET_FAIL(write_var_int(str.len_, out))) {
+ } else if (RET_FAIL(out.write_buf(str.buf_, str.len_))) {
+ }
+ return ret;
+ }
+ FORCE_INLINE static int read_mystring(String &str, common::PageArena *pa,
+ ByteStream &in) {
+ int ret = common::E_OK;
+ int32_t len = 0;
+ int32_t read_len = 0;
+ if (RET_FAIL(read_var_int(len, in))) {
+ } else {
+ char *buf = (char *)pa->alloc(len);
+ if (IS_NULL(buf)) {
+ ret = common::E_OOM;
+ } else {
+ if (RET_FAIL(in.read_buf(buf, len, read_len))) {
+ } else if (len != read_len) {
+ ret = E_BUF_NOT_ENOUGH;
+ } else {
+ str.buf_ = buf;
+ str.len_ = len;
+ }
+ }
+ }
+ return ret;
+ }
+ FORCE_INLINE static int write_char(char ch, ByteStream &out) {
+ return write_ui8(ch, out);
+ }
+ FORCE_INLINE static int read_char(char &ch, ByteStream &in) {
+ return read_ui8((uint8_t &)ch, in);
+ }
+};
+
+FORCE_INLINE bool deserialize_buf_not_enough(int ret) {
+ return ret == E_OUT_OF_RANGE || ret == E_PARTIAL_READ;
+}
+FORCE_INLINE char *get_bytes_from_bytestream(ByteStream &bs) {
+ if (bs.total_size() == 0) {
+ return nullptr;
+ }
+ uint32_t size = bs.total_size();
+ char *ret_buf = (char *)malloc(size);
+ if (ret_buf == nullptr) {
+ return nullptr;
+ }
+
+ ByteStream::BufferIterator buf_iter = bs.init_buffer_iterator();
+ uint32_t offset = 0;
+ while (true) {
+ ByteStream::Buffer buf = buf_iter.get_next_buf();
+ if (buf.buf_ == nullptr) {
+ break;
+ } else {
+ assert(offset + buf.len_ <= size);
+ memcpy(ret_buf + offset, buf.buf_, buf.len_);
+ offset += buf.len_;
+ }
+ }
+ assert(offset == size);
+ return ret_buf;
+}
+
+} // end namespace common
+#endif // COMMON_ALLOCATOR_BYTE_STREAM_H
diff --git a/cpp/src/common/allocator/mem_alloc.cc b/cpp/src/common/allocator/mem_alloc.cc
new file mode 100644
index 00000000..983b52aa
--- /dev/null
+++ b/cpp/src/common/allocator/mem_alloc.cc
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <execinfo.h>
+#include <string.h>
+
+#include <iostream>
+
+#include "alloc_base.h"
+#include "common/logger/elog.h"
+#include "stdio.h"
+#include "stdlib.h"
+#include "utils/util_define.h"
+
+namespace common {
+
+const char *g_mod_names[__LAST_MOD_ID] = {
+ /* 0 */ "DEFAULT",
+ /* 1 */ "MEMTABLE",
+ /* 2 */ "SCHEMA",
+ /* 3 */ "SQL",
+ /* 4 */ "NET",
+ /* 5 */ "NET_DATA",
+ /* 6 */ "TVLIST_DATA",
+ /* 7 */ "TVLIST_OBJ",
+ /* 8 */ "TSBLOCK",
+ /* 9 */ "CONTAINER",
+ /* 10 */ "TSSTORE_OBJ",
+ /* 11 */ "FLUSH_TASK_OBJ",
+ /* 12 */ "PAGE_WRITER_OUTPUT_STREAM",
+ /* 13 */ "CW_PAGES_DATA",
+ /* 14 */ "CHUNK_WRITER_OBJ",
+ /* 15 */ "STATISTIC_OBJ",
+ /* 16 */ "ENCODER_OBJ",
+ /* 17 */ "DECODER_OBJ",
+ /* 18 */ "TSFILE_WRITER_META",
+ /* 19 */ "TSFILE_WRITE_STREAM",
+ /* 20 */ "TIMESERIES_INDEX_OBJ",
+ /* 21 */ "BLOOM_FILTER",
+ /* 22 */ "OPEN_FILE_OBJ",
+ /* 23 */ "TSFILE_READER",
+ /* 24 */ "CHUNK_READER",
+ /* 25 */ "COMPRESSOR_OBJ",
+ /* 26 */ "ARRAY",
+ /* 27 */ "HASH_TABLE",
+};
+
+const uint32_t HEADER_SIZE_4B = 4;
+const uint32_t HEADER_SIZE_8B = 8;
+
+void *mem_alloc(uint32_t size, AllocModID mid) {
+ // use 7bit at most
+ ASSERT(mid <= 127);
+
+ if (size <= 0xFFFFFF) {
+ // use 3B size + 1B mod
+ char *p = (char *)malloc(size + HEADER_SIZE_4B);
+ if (UNLIKELY(p == nullptr)) {
+ return nullptr;
+ } else {
+ uint32_t header = (size << 8) | ((uint32_t)mid);
+ *((uint32_t *)p) = header;
+ ModStat::get_instance().update_alloc(mid, size);
+ // cppcheck-suppress memleak
+ // cppcheck-suppress unmatchedSuppression
+ return p + HEADER_SIZE_4B;
+ }
+ } else {
+ char *p = (char *)malloc(size + HEADER_SIZE_8B);
+ if (UNLIKELY(p == nullptr)) {
+ std::cout << "alloc big filed for size " << size + HEADER_SIZE_4B
+ << std::endl;
+ return nullptr;
+ } else {
+ uint64_t large_size = size;
+ uint64_t header = ((large_size) << 8) | (((uint32_t)mid) | (0x80));
+ uint32_t low4b = (uint32_t)(header & 0xFFFFFFFF);
+ uint32_t high4b = (uint32_t)(header >> 32);
+ *(uint32_t *)p = high4b;
+ *(uint32_t *)(p + 4) = low4b;
+ ModStat::get_instance().update_alloc(mid, size);
+ // cppcheck-suppress unmatchedSuppression
+ // cppcheck-suppress memleak
+ return p + HEADER_SIZE_8B;
+ }
+ }
+}
+
+void printCallers() {
+ int layers = 0, i = 0;
+ char **symbols = NULL;
+
+ const int64_t MAX_FRAMES = 32;
+
+ void *frames[MAX_FRAMES];
+ memset(frames, 0, sizeof(frames));
+ layers = backtrace(frames, MAX_FRAMES);
+ for (i = 0; i < layers; i++) {
+ printf("Layer %d: %p\n", i, frames[i]);
+ }
+ printf("------------------\n");
+
+ symbols = backtrace_symbols(frames, layers);
+ if (symbols) {
+ for (i = 0; i < layers; i++) {
+ printf("SYMBOL layer %d: %s\n", i, symbols[i]);
+ }
+ free(symbols);
+ } else {
+ printf("Failed to parse function names\n");
+ }
+}
+
+void mem_free(void *ptr) {
+ // try as 4Byte header
+ char *p = (char *)ptr;
+ uint32_t header = *(uint32_t *)(p - HEADER_SIZE_4B);
+ if ((header & 0x80) == 0) {
+ // 4Byte header
+ uint32_t size = header >> 8;
+ AllocModID mid = (AllocModID)(header & 0x7F);
+ ModStat::get_instance().update_free(mid, size);
+ ::free(p - HEADER_SIZE_4B);
+ } else {
+ // 8Byte header
+ uint64_t header8b = ((uint64_t)(*(uint32_t *)(p - 4))) |
+ ((uint64_t)(*(uint32_t *)(p - 8)) << 32);
+ AllocModID mid = (AllocModID)(header8b & 0x7F);
+ uint32_t size = (uint32_t)(header8b >> 8);
+ ModStat::get_instance().update_free(mid, size);
+ ::free(p - HEADER_SIZE_8B);
+ }
+}
+
+void *mem_realloc(void *ptr, uint32_t size) {
+ AllocModID mid_org;
+ uint32_t size_org;
+ char *p = (char *)ptr;
+ uint32_t header_org =
+ *(uint32_t *)(p - HEADER_SIZE_4B); // try as 4Byte header
+ if ((header_org & 0x80) == 0) {
+ // header_org is 4byte
+ size_org = header_org >> 8;
+ mid_org = (AllocModID)(header_org & 0x7F);
+ if (size <= 0xFFFFFF) {
+ p = (char *)realloc(p - HEADER_SIZE_4B, size + HEADER_SIZE_4B);
+ if (UNLIKELY(p == nullptr)) {
+ return nullptr;
+ } else {
+ uint32_t header =
+ (size << 8) | ((uint32_t)mid_org); // size changed
+ *((uint32_t *)p) = header;
+ ModStat::get_instance().update_alloc(
+ mid_org, int32_t(size) - int32_t(size_org));
+ return p + HEADER_SIZE_4B;
+ }
+ } else { // size > 0xFFFFFF, realloc(os_p, size + header_len)
+ p = (char *)realloc(p - HEADER_SIZE_4B, size + HEADER_SIZE_8B);
+ if (UNLIKELY(p == nullptr)) {
+ return nullptr;
+ } else {
+ std::memmove(p + HEADER_SIZE_8B, p + HEADER_SIZE_4B, size_org);
+ // reconstruct 8-byte header
+ uint64_t large_size = size;
+ uint64_t header =
+ ((large_size) << 8) | (((uint32_t)mid_org) | (0x80));
+ uint32_t low4b = (uint32_t)(header & 0xFFFFFFFF);
+ uint32_t high4b = (uint32_t)(header >> 32);
+ *(uint32_t *)p = high4b;
+ *(uint32_t *)(p + 4) = low4b;
+ ModStat::get_instance().update_alloc(
+ mid_org, int32_t(size) - int32_t(size_org));
+ return p + HEADER_SIZE_8B;
+ }
+ }
+ } else { // header_org is 8byte
+ uint64_t header =
+ ((uint64_t)(*(uint32_t *)(p - 4))) |
+ ((uint64_t)(*(uint32_t *)(p - 8)) << 32); // 8Byte header
+ mid_org = (AllocModID)(header & 0x7F);
+ size_org = (uint32_t)(header >> 8);
+ if (size <= 0xFFFFFF) {
+ uint32_t save_data =
+ *(uint32_t *)(p - HEADER_SIZE_8B + HEADER_SIZE_4B + size);
+ p = (char *)realloc(p - HEADER_SIZE_8B, size + HEADER_SIZE_4B);
+ if (UNLIKELY(p == nullptr)) {
+ return nullptr;
+ } else {
+ std::memmove(p + HEADER_SIZE_4B, p + HEADER_SIZE_8B,
+ size - HEADER_SIZE_4B);
+ // reconstruct 4-byte header
+ uint32_t header4b = (size << 8) | (((uint32_t)mid_org));
+ *((uint32_t *)p) = header4b;
+ // reconstruct data
+ *(uint32_t *)((char *)p + size - 4) = save_data;
+ ModStat::get_instance().update_alloc(
+ mid_org, int32_t(size) - int32_t(size_org));
+ return p + HEADER_SIZE_4B;
+ }
+ } else {
+ p = (char *)realloc(p - HEADER_SIZE_8B, size + HEADER_SIZE_8B);
+ if (UNLIKELY(p == nullptr)) {
+ return nullptr;
+ } else {
+ uint64_t large_size = size;
+ uint64_t header8b =
+ ((large_size) << 8) | (((uint32_t)mid_org) | (0x80));
+ uint32_t low4b = (uint32_t)(header8b & 0xFFFFFFFF);
+ uint32_t high4b = (uint32_t)(header8b >> 32);
+ *(uint32_t *)p = high4b;
+ *(uint32_t *)(p + 4) = low4b;
+ ModStat::get_instance().update_alloc(
+ mid_org, int32_t(size) - int32_t(size_org));
+ return p + HEADER_SIZE_8B;
+ }
+ }
+ }
+}
+
+void ModStat::init() {
+ stat_arr_ = (int32_t *)(::malloc(ITEM_SIZE * ITEM_COUNT));
+ for (int8_t i = 0; i < __LAST_MOD_ID; i++) {
+ int32_t *item = get_item(i);
+ *item = 0;
+ }
+}
+
+void ModStat::destroy() { ::free(stat_arr_); }
+
+// TODO return to SQL
+void ModStat::print_stat() {
+ //
+}
+
+BaseAllocator g_base_allocator;
+
+} // end namespace common
\ No newline at end of file
diff --git a/cpp/src/common/allocator/my_string.h b/cpp/src/common/allocator/my_string.h
new file mode 100644
index 00000000..348c55ed
--- /dev/null
+++ b/cpp/src/common/allocator/my_string.h
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_ALLOCATOR_MY_STRING_H
+#define COMMON_ALLOCATOR_MY_STRING_H
+
+#include <string.h>
+
+#include <iostream>
+
+#include "common/allocator/page_arena.h"
+#include "utils/errno_define.h"
+
+namespace common {
+
+// String that use PageArena
+struct String {
+ char *buf_;
+ uint32_t len_;
+
+ String() : buf_(nullptr), len_(0) {}
+ String(char *buf, uint32_t len) : buf_(buf), len_(len) {}
+ FORCE_INLINE bool is_null() const { return buf_ == nullptr && len_ == 0; }
+ FORCE_INLINE void reset() {
+ len_ = 0;
+ buf_ = nullptr;
+ }
+ FORCE_INLINE int dup_from(const std::string &str, common::PageArena &pa) {
+ len_ = str.size();
+ if (UNLIKELY(len_ == 0)) {
+ return common::E_OK;
+ }
+ buf_ = pa.alloc(len_);
+ if (IS_NULL(buf_)) {
+ return common::E_OOM;
+ }
+ memcpy(buf_, str.c_str(), len_);
+ return common::E_OK;
+ }
+ FORCE_INLINE int dup_from(const String &str, common::PageArena &pa) {
+ len_ = str.len_;
+ if (UNLIKELY(len_ == 0)) {
+ return common::E_OK;
+ }
+ buf_ = pa.alloc(len_);
+ if (IS_NULL(buf_)) {
+ return common::E_OOM;
+ }
+ memcpy(buf_, str.buf_, len_);
+ return common::E_OK;
+ }
+ FORCE_INLINE int build_from(const String &s1, const String &s2,
+ common::PageArena &pa) {
+ len_ = s1.len_ + s2.len_;
+ buf_ = pa.alloc(len_);
+ if (IS_NULL(buf_)) {
+ return common::E_OOM;
+ }
+ memcpy(buf_, s1.buf_, s1.len_);
+ memcpy(buf_ + s1.len_, s2.buf_, s2.len_);
+ return common::E_OK;
+ }
+ FORCE_INLINE void shallow_copy_from(const String &src) {
+ buf_ = src.buf_;
+ len_ = src.len_;
+ }
+ FORCE_INLINE bool equal_to(const String &that) const {
+ return (len_ == 0 && that.len_ == 0) ||
+ ((len_ == that.len_) && (0 == memcmp(buf_, that.buf_, len_)));
+ }
+
+ // FORCE_INLINE bool equal_to(const std::string &that)
+ // {
+ // return (len_ == 0 && that.size() == 0)
+ // || ((len_ == that.len_) && (0 == memcmp(buf_, that.c_str(),
+ // len_)));
+ // }
+
+ // strict less than. If @this equals to @that, return false.
+ FORCE_INLINE bool less_than(const String &that) const {
+ if (len_ == 0 || that.len_ == 0) {
+ return false;
+ }
+ uint32_t min_len = std::min(len_, that.len_);
+ int cmp_res = memcmp(buf_, that.buf_, min_len);
+ if (cmp_res < 0) {
+ return true;
+ } else if (cmp_res > 0) {
+ return false;
+ } else {
+ return len_ < that.len_;
+ }
+ }
+
+ // return = 0, if this = that
+ // return < 0, if this < that
+ // return > 0, if this > that
+ FORCE_INLINE int compare(const String &that) const {
+ if (len_ == 0 || that.len_ == 0) {
+ return 0;
+ }
+ uint32_t min_len = std::min(len_, that.len_);
+ int cmp_res = memcmp(buf_, that.buf_, min_len);
+ if (cmp_res == 0) {
+ return len_ - that.len_;
+ } else {
+ return cmp_res;
+ }
+ }
+
+#ifndef NDEBUG
+ friend std::ostream &operator<<(std::ostream &os, const String &s) {
+ os << s.len_ << "@";
+ for (uint32_t i = 0; i < s.len_; i++) {
+ os << s.buf_[i];
+ }
+ return os;
+ }
+#endif // ifndef NDEBUG
+};
+
+struct StringLessThan {
+ FORCE_INLINE bool operator()(const String &s1, const String &s2) const {
+ return s1.less_than(s2);
+ }
+};
+
+} // end namespace common
+#endif // COMMON_ALLOCATOR_MY_STRING_H
diff --git a/cpp/src/common/allocator/object_pool.h b/cpp/src/common/allocator/object_pool.h
new file mode 100644
index 00000000..721823a2
--- /dev/null
+++ b/cpp/src/common/allocator/object_pool.h
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_ALLOCTOR_OBJECT_POOL_H
+#define COMMON_ALLOCTOR_OBJECT_POOL_H
+
+#include "common/allocator/alloc_base.h"
+#include "common/mutex/mutex.h"
+
+namespace common {
+
+template <class T>
+class ObjectPool {
+ private:
+ struct ObjectPoolNode {
+ T data_;
+ ObjectPoolNode *next_;
+
+ ObjectPoolNode() : data_(), next_(nullptr) {}
+ };
+
+ public:
+ /*
+ * max_cache_count is a soft limitation:
+ */
+ ObjectPool(const uint32_t max_cache_count, const AllocModID mid,
+ BaseAllocator &allocator = g_base_allocator)
+ : max_cache_count_(max_cache_count),
+ cur_alloc_count_(0),
+ mid_(mid),
+ allocator_(allocator),
+ mutex_(),
+ head_(nullptr) {
+ assert(max_cache_count > 1);
+ }
+
+ ~ObjectPool() { destroy(); }
+
+ void destroy() {
+ ObjectPoolNode *cur = head_;
+ while (cur) {
+ head_ = cur->next_;
+ allocator_.free(cur);
+ cur = head_;
+ cur_alloc_count_--;
+ }
+ ASSERT(cur_alloc_count_ == 0);
+ }
+
+ T *alloc() {
+ T *ret_obj = nullptr;
+ common::MutexGuard g(mutex_);
+ if (head_) {
+ ret_obj = &(head_->data_);
+ head_ = head_->next_;
+ return ret_obj;
+ } else {
+ void *buf = allocator_.alloc(sizeof(ObjectPoolNode), mid_);
+ if (UNLIKELY(buf == nullptr)) {
+ return nullptr;
+ }
+ cur_alloc_count_++;
+ ret_obj = &(new (buf) ObjectPoolNode)->data_;
+ return ret_obj;
+ }
+ }
+
+ void free(T *obj) {
+ ASSERT(obj != nullptr);
+ common::MutexGuard g(mutex_);
+ if (cur_alloc_count_ > max_cache_count_) {
+ allocator_.free(obj);
+ cur_alloc_count_--;
+ ASSERT(cur_alloc_count_ >= 0);
+ } else {
+ ObjectPoolNode *n = (ObjectPoolNode *)obj;
+ n->next_ = head_;
+ head_ = n;
+ }
+ }
+
+ uint32_t get_cur_alloc_count() const { return cur_alloc_count_; }
+
+ private:
+ uint32_t max_cache_count_;
+ uint32_t cur_alloc_count_;
+ AllocModID mid_;
+ BaseAllocator allocator_;
+ common::Mutex mutex_;
+ ObjectPoolNode *head_; // freelist head
+};
+
+} // namespace common
+#endif // COMMON_ALLOCTOR_OBJECT_POOL_H
diff --git a/cpp/src/common/allocator/page_arena.cc b/cpp/src/common/allocator/page_arena.cc
new file mode 100644
index 00000000..731b0b0b
--- /dev/null
+++ b/cpp/src/common/allocator/page_arena.cc
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "page_arena.h"
+
+#include <stdio.h>
+
+#include <new>
+
+namespace common {
+
+char *PageArena::alloc(uint32_t size) {
+ if (LIKELY(size <= page_size_)) {
+ Page *cur_page = dummy_head_.next_;
+ if (LIKELY(cur_page != nullptr)) {
+ char *ret_ptr = cur_page->alloc(size);
+ if (LIKELY(ret_ptr != nullptr)) {
+ return ret_ptr;
+ }
+ }
+
+ /*
+ * cur_page is null OR can not alloc @size from cur_page,
+ * then alloc new page
+ */
+ void *ptr = base_allocator_.alloc(page_size_ + sizeof(Page), mid_);
+ if (UNLIKELY(ptr == nullptr)) {
+ return nullptr;
+ }
+ Page *new_page = new (ptr) Page(page_size_, cur_page);
+ dummy_head_.next_ = new_page;
+ return new_page->alloc(size);
+ } else {
+ void *ptr = base_allocator_.alloc(size + sizeof(Page), mid_);
+ Page *new_page = new (ptr) Page(dummy_head_.next_);
+ dummy_head_.next_ = new_page;
+ return reinterpret_cast<char *>(new_page) + sizeof(Page);
+ }
+}
+
+void PageArena::reset() {
+ Page *p = dummy_head_.next_;
+ while (p) {
+ dummy_head_.next_ = p->next_;
+ base_allocator_.free(p);
+ p = dummy_head_.next_;
+ }
+}
+
+} // end namespace common
\ No newline at end of file
diff --git a/cpp/src/common/allocator/page_arena.h b/cpp/src/common/allocator/page_arena.h
new file mode 100644
index 00000000..b8053cfb
--- /dev/null
+++ b/cpp/src/common/allocator/page_arena.h
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_ALLOCATOR_PAGE_ARENA_H
+#define COMMON_ALLOCATOR_PAGE_ARENA_H
+
+#include "alloc_base.h"
+
+namespace common {
+
+/*
+ * Not Thread Safe
+ */
+class PageArena {
+ public:
+ explicit PageArena(BaseAllocator &base_allocator = g_base_allocator)
+ : page_size_(0),
+ mid_(__FIRST_MOD_ID),
+ base_allocator_(base_allocator),
+ dummy_head_() {}
+ ~PageArena() { destroy(); }
+ void init(uint32_t page_size, AllocModID mid) {
+ page_size_ = page_size;
+ mid_ = mid;
+ }
+
+ char *alloc(uint32_t size);
+ FORCE_INLINE void destroy() { reset(); }
+ void reset();
+
+#ifdef ENABLE_TEST
+ int TEST_get_page_count() const {
+ int count = 0;
+ Page *p = dummy_head_.next_;
+ while (p) {
+ p = p->next_;
+ count++;
+ }
+ return count;
+ }
+#endif
+
+ private:
+ class Page {
+ public:
+ Page() : next_(nullptr), page_end_(nullptr), cur_alloc_(nullptr) {}
+ explicit Page(Page *next_page)
+ : next_(next_page), page_end_(nullptr), cur_alloc_(nullptr) {}
+ Page(uint32_t page_size, Page *next_page) {
+ next_ = next_page;
+ cur_alloc_ =
+ (char *)this + sizeof(Page); // equals to (char*)(this+1)
+ page_end_ = cur_alloc_ + page_size;
+ }
+ INLINE char *alloc(uint32_t size) {
+ if (cur_alloc_ + size > page_end_) {
+ return nullptr;
+ } else {
+ char *ret = cur_alloc_;
+ cur_alloc_ += size;
+ return ret;
+ // char *ret = cur_alloc_;
+ // cur_alloc_ += size;
+ // int address = reinterpret_cast<uintptr_t>(cur_alloc_);
+ // int new_addr = (address + 3) & (~3);
+ // cur_alloc_ = reinterpret_cast<char *>(new_addr);
+ // return ret;
+ }
+ }
+
+ public:
+ Page *next_;
+ char *page_end_;
+ char *cur_alloc_; // buf_'s current offset
+ };
+
+ private:
+ uint32_t page_size_;
+ AllocModID mid_;
+ BaseAllocator &base_allocator_;
+ Page dummy_head_;
+};
+
+} // end namespace common
+#endif // COMMON_ALLOCATOR_PAGE_ARENA_H
diff --git a/cpp/src/common/allocator/stl_allocator.h b/cpp/src/common/allocator/stl_allocator.h
new file mode 100644
index 00000000..e8583985
--- /dev/null
+++ b/cpp/src/common/allocator/stl_allocator.h
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_ALLOCATOR_STL_ALLOCATOR_H
+#define COMMON_ALLOCATOR_STL_ALLOCATOR_H
+
+#include "alloc_base.h"
+
+namespace common {
+
+template <class T, AllocModID Mid, class TAllocator = BaseAllocator>
+class StlAllocator {
+ public:
+ typedef size_t size_type;
+ typedef ptrdiff_t difference_type;
+ typedef T *pointer;
+ typedef const T *const_pointer;
+ typedef T &reference;
+ typedef const T &const_reference;
+ typedef T value_type;
+
+ /*
+ * rebind provides a way to obtain an allocator for a different type.
+ * For example, std::list will alloc object Node<T> beside alloc object T.
+ */
+ template <class U>
+ struct rebind {
+ typedef StlAllocator<U, Mid, TAllocator> other;
+ };
+
+ StlAllocator() {}
+ StlAllocator(const StlAllocator &) {}
+
+ template <class T2, AllocModID Mid2>
+ StlAllocator(const StlAllocator<T2, Mid2, TAllocator> &) {}
+
+ StlAllocator(TAllocator base_allocator) : base_allocator_(base_allocator) {}
+
+ pointer address(reference x) const { return &x; }
+ const_pointer address(const_reference x) { return &x; }
+
+ pointer allocate(size_type n, const void *hint = 0) {
+ return (pointer)base_allocator_.alloc(n * sizeof(T), Mid);
+ };
+ void deallocate(void *p, size_type) { base_allocator_.free(p); }
+ size_type max_size() const { return uint32_t(-1); }
+
+ void construct(pointer p, const T &val) { new ((T *)p) T(val); }
+ void destroy(pointer p) { p->~T(); }
+
+ private:
+ TAllocator base_allocator_;
+};
+
+/*
+ * According to the manual, allocator is stateless.
+ * Although we define a base_allocator_ here, but base_allocator_ is also
+ * stateless. so '==' is always true and '!=' is always false. refer to
+ * https://en.cppreference.com/w/cpp/memory/allocator/operator_cmp.
+ */
+template <class T1, AllocModID Mid1, class T2, AllocModID Mid2>
+bool operator==(const StlAllocator<T1, Mid1> &a1,
+ const StlAllocator<T2, Mid2> &a2) {
+ return true;
+}
+
+template <class T1, AllocModID Mid1, class T2, AllocModID Mid2>
+bool operator!=(const StlAllocator<T1, Mid1> &a1,
+ const StlAllocator<T2, Mid2> &a2) {
+ return false;
+}
+
+} // end namespace common
+#endif // COMMON_ALLOCATOR_STL_ALLOCATOR_H
diff --git a/cpp/tsfile/src/common/allocator/util_define.h b/cpp/src/common/allocator/util_define.h
similarity index 51%
rename from cpp/tsfile/src/common/allocator/util_define.h
rename to cpp/src/common/allocator/util_define.h
index 4d85ef52..0526c08b 100644
--- a/cpp/tsfile/src/common/allocator/util_define.h
+++ b/cpp/src/common/allocator/util_define.h
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
/*
* This file defines some basic macros
@@ -10,53 +28,47 @@
#include <stdint.h>
#include <stdlib.h>
-
/* ======== cmake config define ======== */
/* ======== unsued ======== */
#define UNUSED(v) ((void)(v))
-
/* ======== inline ======== */
#ifdef __GNUC__
#define FORCE_INLINE inline __attribute__((always_inline))
#else
#define FORCE_INLINE inline
-#endif // __GNUC__
+#endif // __GNUC__
#ifdef BUILD_FOR_SMALL_BINARY
#define INLINE FORCE_INLINE
#else
#define INLINE
-#endif // BUILD_FOR_SMALL_BINARY
-
+#endif // BUILD_FOR_SMALL_BINARY
/* ======== likely ======== */
#if defined(__GNUC__) && __GNUC__ >= 4
-#define LIKELY(x) (__builtin_expect((x), 1))
+#define LIKELY(x) (__builtin_expect((x), 1))
#define UNLIKELY(x) (__builtin_expect((x), 0))
#else
-#define LIKELY(x) (x)
+#define LIKELY(x) (x)
#define UNLIKELY(x) (x)
-#endif // __GNUC__ >= 4
-
+#endif // __GNUC__ >= 4
/* ======== nullptr ======== */
#if __cplusplus < 201103L
#ifndef nullptr
#define nullptr NULL
#endif
-#define OVERRIDE
+#define OVERRIDE
#else
#define OVERRIDE override
-#endif // __cplusplus < 201103L
-
+#endif // __cplusplus < 201103L
/* ======== cache line ======== */
#ifndef CACHE_LINE_SIZE
#define CACHE_LINE_SIZE 64
-#endif // CACHE_LINE_SIZE
-
+#endif // CACHE_LINE_SIZE
/* ======== assert ======== */
#ifdef NDEBUG
@@ -73,28 +85,33 @@
*/
#if __cplusplus < 201103L
// TODO only define this when DEBUG
-#define STATIC_ASSERT(cond, msg) typedef char static_assertion_##msg[(cond) ? 1 : -1] __attribute__((unused))
+#define STATIC_ASSERT(cond, msg) \
+ typedef char static_assertion_##msg[(cond) ? 1 : -1] __attribute__((unused))
#else
#define STATIC_ASSERT(cond, msg) static_assert((cond), #msg)
-#endif // __cplusplus < 201103L
-
+#endif // __cplusplus < 201103L
/* ======== atomic operation ======== */
-#define ATOMIC_FAA(val_addr, addv) __atomic_fetch_add((val_addr), (addv), __ATOMIC_SEQ_CST)
-#define ATOMIC_AAF(val_addr, addv) __atomic_add_fetch((val_addr), (addv), __ATOMIC_SEQ_CST)
+#define ATOMIC_FAA(val_addr, addv) \
+ __atomic_fetch_add((val_addr), (addv), __ATOMIC_SEQ_CST)
+#define ATOMIC_AAF(val_addr, addv) \
+ __atomic_add_fetch((val_addr), (addv), __ATOMIC_SEQ_CST)
/*
* It implements an atomic compare and exchange operation.
* This compares the contents of *ptr with the contents of *expected.
- * - If equal, the operation is a read-modify-write operation that writes desired into *ptr.
- * - If they are not equal, the operation is a read and the current contents of *ptr are written into *expected
+ * - If equal, the operation is a reader-modify-writer operation that writes
+ * desired into *ptr.
+ * - If they are not equal, the operation is a reader and the current contents
+ * of *ptr are written into *expected
*/
-#define ATOMIC_CAS(val_addr, expected, desired) \
- __atomic_compare_exchange_n((val_addr), (expected),(desired), \
- /* weak = */ false, \
- /* success_memorder = */ __ATOMIC_SEQ_CST, \
- /* failure_memorder = */ __ATOMIC_SEQ_CST)
-#define ATOMIC_LOAD(val_addr) __atomic_load_n((val_addr), __ATOMIC_SEQ_CST)
-#define ATOMIC_STORE(val_addr, val) __atomic_store_n((val_addr), (val), __ATOMIC_SEQ_CST)
+#define ATOMIC_CAS(val_addr, expected, desired) \
+ __atomic_compare_exchange_n((val_addr), (expected), (desired), \
+ /* weak = */ false, \
+ /* success_memorder = */ __ATOMIC_SEQ_CST, \
+ /* failure_memorder = */ __ATOMIC_SEQ_CST)
+#define ATOMIC_LOAD(val_addr) __atomic_load_n((val_addr), __ATOMIC_SEQ_CST)
+#define ATOMIC_STORE(val_addr, val) \
+ __atomic_store_n((val_addr), (val), __ATOMIC_SEQ_CST)
/* ======== align ======== */
#define ALIGNED(a) __attribute__((aligned(a)))
@@ -104,12 +121,12 @@
/* ======== disallow copy and assign ======== */
#if __cplusplus < 201103L
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
- TypeName(const TypeName&); \
- void operator=(const TypeName&)
+ TypeName(const TypeName&); \
+ void operator=(const TypeName&)
#else
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
- TypeName(const TypeName&) = delete; \
- TypeName& operator=(const TypeName&) = delete;
+ TypeName(const TypeName&) = delete; \
+ TypeName& operator=(const TypeName&) = delete;
#endif
/* ======== return value check ======== */
@@ -132,5 +149,4 @@
*/
#define INT64_TO_BASE10_MAX_LEN 24
-#endif // COMMON_UTIL_DEFINE_H
-
+#endif // COMMON_UTIL_DEFINE_H
diff --git a/cpp/src/common/config/CMakeLists.txt b/cpp/src/common/config/CMakeLists.txt
new file mode 100644
index 00000000..e7ef66fa
--- /dev/null
+++ b/cpp/src/common/config/CMakeLists.txt
@@ -0,0 +1,20 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
+
+
diff --git a/cpp/src/common/config/config.h b/cpp/src/common/config/config.h
new file mode 100644
index 00000000..679a69b0
--- /dev/null
+++ b/cpp/src/common/config/config.h
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONFIG_CONFIG_H
+#define COMMON_CONFIG_CONFIG_H
+
+#include <stdint.h>
+
+#include "common/mutex/mutex.h"
+#include "utils/db_utils.h"
+#include "utils/util_define.h"
+
+namespace common {
+enum ConfigLevel {
+ INIT, // Unchangeable, initialized during database init
+ RESTART, // Can be changed, but the database must be restarted to take
+ // effect
+ USERSET // Session level update
+};
+
+typedef struct ConfigValue {
+ uint32_t
+ tsblock_mem_inc_step_size_; // tsblock memory self-increment step size
+ uint32_t tsblock_max_memory_; // the maximum memory of a single tsblock
+ const char *rest_service_ip_;
+ int32_t rest_service_port_;
+ WALFlushPolicy wal_flush_policy_;
+ uint32_t seqtvlist_primary_array_size_;
+ uint32_t seqtvlist_max_record_count_;
+ uint32_t page_writer_max_point_num_;
+ uint32_t page_writer_max_memory_bytes_;
+ uint32_t max_degree_of_index_node_;
+ double tsfile_index_bloom_filter_error_percent_;
+ const char *tsfile_prefix_path_;
+ TSEncoding time_encoding_type_;
+ uint32_t memtable_flusher_poll_interval_seconds_;
+} ConfigValue;
+
+extern void init_config_value();
+
+// In the future, configuration items need to be dynamically adjusted according
+// to the level
+extern void set_config_value();
+extern void config_set_page_max_point_count(uint32_t page_max_ponint_count);
+extern void config_set_max_degree_of_index_node(
+ uint32_t max_degree_of_index_node);
+
+// FORCE_INLINE bool wal_cfg_enabled() { return
+// g_config_value_.wal_flush_policy_ != WAL_DISABLED; } FORCE_INLINE bool
+// wal_cfg_should_wait_persisted() { return g_config_value_.wal_flush_policy_ >=
+// WAL_FLUSH; }
+
+} // namespace common
+
+#endif // COMMON_CONFIG_CONFIG_H
diff --git a/cpp/src/common/container/CMakeLists.txt b/cpp/src/common/container/CMakeLists.txt
new file mode 100644
index 00000000..f92dd5ba
--- /dev/null
+++ b/cpp/src/common/container/CMakeLists.txt
@@ -0,0 +1,22 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
+message("Running in src/common/container directory")
+
+aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} common_container_SRC_LIST)
+add_library(common_container OBJECT ${common_container_SRC_LIST})
diff --git a/cpp/src/common/container/array.h b/cpp/src/common/container/array.h
new file mode 100644
index 00000000..52412949
--- /dev/null
+++ b/cpp/src/common/container/array.h
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONTAINER_ARRAY_H
+#define COMMON_CONTAINER_ARRAY_H
+
+#include <stdint.h>
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+
+#include "common/allocator/alloc_base.h"
+#include "common/logger/elog.h"
+#include "utils/errno_define.h"
+#include "utils/util_define.h"
+
+#define ARRAY_INIT_CAPACITY 1000 // TODO: configurable mode
+#define ARRAY_GROWTH_THRESHOLD 50 // TODO: configurable mode
+#define ARRAY_SHRINK_THRESHOLD 50 // TODO: configurable mode
+
+namespace common {
+
+template <class ValueType>
+class Array {
+ public:
+#ifndef NDEBUG
+ friend std::ostream &operator<<(std::ostream &out, Array<ValueType> &sa) {
+ for (size_t idx = 0; idx < sa.size(); idx++) {
+ if (idx > 0) {
+ out << ' ';
+ }
+ out << sa[idx];
+ }
+ out << std::endl;
+ return out;
+ }
+#endif // NDEBUG
+
+ Array() {
+ capacity_ = ARRAY_INIT_CAPACITY;
+ size_ = 0;
+ is_inited_ = false;
+ array_ = nullptr;
+ }
+
+ Array(size_t capacity) {
+ capacity_ = capacity;
+ size_ = 0;
+ is_inited_ = false;
+ array_ = nullptr;
+ }
+
+ ~Array() {
+ if (is_inited_) {
+ destroy();
+ }
+ capacity_ = 0;
+ size_ = 0;
+ }
+
+ // int deep_copy(const Array &other)
+ // {
+ // this->capacity_ = other.capacity_;
+ // this->size_ = other.size_;
+ // if (init() == E_OK) {
+ // memcpy(this->array_, other.array_, this->capacity_ *
+ // sizeof(*(this->array_))); return E_OK;
+ // }
+ // return E_OOM;
+ // }
+
+ int init() {
+ if (is_inited_) {
+ // log_err("init repeated.");
+ ASSERT(!is_inited_);
+ }
+ array_ =
+ (ValueType *)mem_alloc(capacity_ * sizeof(*(array_)), MOD_ARRAY);
+ if (UNLIKELY(nullptr == array_)) {
+ is_inited_ = false;
+ // log_err("malloc failed.");
+ return E_OOM;
+ }
+ is_inited_ = true;
+ return E_OK;
+ }
+
+ void destroy() {
+ capacity_ = 0;
+ size_ = 0;
+ if (is_inited_) {
+ mem_free(array_);
+ array_ = nullptr;
+ }
+ is_inited_ = false;
+ }
+
+ private:
+ static size_t linear_calc(size_t input) {
+ if (input < ARRAY_GROWTH_THRESHOLD) {
+ return input;
+ }
+ size_t res = 1;
+ input /= 10;
+ while (input) {
+ input /= 10;
+ res *= 10;
+ }
+ return res;
+ }
+
+ public:
+ int extend() {
+ // Check for a potential overflow
+ uint64_t tmp = (uint64_t)(linear_calc(capacity_) + capacity_);
+ if (tmp > SIZE_MAX) {
+ // log_err("size overflow.");
+ return E_OVERFLOW;
+ }
+
+ size_t new_capacity = (size_t)tmp;
+ array_ =
+ (ValueType *)mem_realloc(array_, new_capacity * sizeof(*(array_)));
+ if (UNLIKELY(nullptr == array_)) {
+ // log_err("realloc failed.");
+ return E_OOM;
+ }
+ capacity_ = new_capacity;
+ return E_OK;
+ }
+
+ int shrink() {
+ size_t new_capacity = (size_t)(capacity_ / 2);
+ // if the size passed to realloc is smaller than before, OS will
+ // automatically release the rest memory
+ array_ =
+ (ValueType *)mem_realloc(array_, new_capacity * sizeof(*(array_)));
+ if (UNLIKELY(nullptr == array_)) {
+ // log_err("malloc failed.");
+ return E_OOM;
+ }
+ capacity_ = new_capacity;
+ return E_OK;
+ }
+
+ int insert(size_t idx, const ValueType &value) {
+ if (idx > size_) {
+ // log_err("index %lu is out of range, because size is %lu.", idx,
+ // size_);
+ return E_OUT_OF_RANGE;
+ }
+ if (size_ >= capacity_) {
+ int ret = extend();
+ if (ret != E_OK) {
+ return ret;
+ }
+ }
+ ASSERT(idx <= size_);
+ ASSERT(size_ < capacity_);
+ if (idx == size_) {
+ array_[idx] = value;
+ size_++;
+ return E_OK;
+ }
+
+ memmove(array_ + (idx + 1), array_ + idx,
+ (size_ - idx) * sizeof(*(array_)));
+ array_[idx] = value;
+ size_++;
+ return E_OK;
+ }
+
+ // int update(size_t idx, const ValueType &value)
+ // {
+ // if (idx >= size_) {
+ // //log_err("index %lu is out of range, because size is %lu.", idx,
+ // size_); return E_OUT_OF_RANGE;
+ // }
+ // array_[idx] = value;
+ // return E_OK;
+ // }
+
+ int append(const ValueType &value) {
+ if (size_ >= capacity_) {
+ int ret = extend();
+ if (ret != E_OK) {
+ return ret;
+ }
+ }
+ array_[size_] = value;
+ size_++;
+ return E_OK;
+ }
+
+ FORCE_INLINE ValueType &at(size_t idx) {
+ if (idx >= size_) {
+ // log_err("index %lu is out of range, because size is %lu.", idx,
+ // size_);
+ ASSERT(idx < size_);
+ return dummy_;
+ }
+ return array_[idx];
+ }
+
+ FORCE_INLINE ValueType &operator[](size_t idx) {
+ if (idx >= size_) {
+ // log_err("index %lu is out of range, because size is %lu.", idx,
+ // size_);
+ ASSERT(idx < size_);
+ return dummy_;
+ }
+ return array_[idx];
+ }
+
+ bool contain(const ValueType &value) {
+ for (int i = 0; i < size_; i++) {
+ if (array_[i] == value) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /*
+ * if the value is not found, then set 'found' is false.
+ */
+ size_t find(const ValueType &target_val, bool &found) {
+ for (size_t i = 0; i < size_; i++) {
+ if (array_[i] == target_val) {
+ found = true;
+ return i;
+ }
+ }
+ found = false;
+ return 0;
+ }
+
+ ValueType remove(size_t idx) {
+ if (idx >= size_) {
+ // log_err("index %lu is out of range, because size is %lu.", idx,
+ // size_);
+ ASSERT(idx < size_);
+ return dummy_;
+ }
+ if (capacity_ > ARRAY_SHRINK_THRESHOLD && size_ <= capacity_ / 4) {
+ shrink();
+ }
+ ValueType ret = array_[idx];
+ memmove(array_ + idx, array_ + idx + 1,
+ (size_ - idx - 1) * sizeof(*(array_)));
+ size_--;
+
+ return ret;
+ }
+
+ int remove_value(const ValueType &value) {
+ bool found = false;
+ size_t pos = find(value, found);
+ if (!found) {
+ // log_warn("the value is not exist in the array.");
+ return E_NOT_EXIST;
+ }
+ while (found) {
+ remove(pos);
+ pos = find(value, found);
+ }
+
+ return E_OK;
+ }
+
+ FORCE_INLINE void clear() { size_ = 0; }
+
+ FORCE_INLINE bool empty() const { return ATOMIC_LOAD(&size_) == 0; }
+
+ FORCE_INLINE size_t size() const { return ATOMIC_LOAD(&size_); }
+
+ FORCE_INLINE size_t capacity() const { return ATOMIC_LOAD(&capacity_); }
+
+ private:
+ size_t capacity_;
+ size_t size_;
+ ValueType *array_;
+ ValueType dummy_;
+ bool is_inited_;
+};
+
+} // end namespace common
+#endif // COMMON_CONTAINER_ARRAY_H
diff --git a/cpp/src/common/container/bit_map.cc b/cpp/src/common/container/bit_map.cc
new file mode 100644
index 00000000..407605e5
--- /dev/null
+++ b/cpp/src/common/container/bit_map.cc
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "bit_map.h"
+
+#include "common/allocator/alloc_base.h"
+
+namespace common {
+
+BitMap::~BitMap() {
+ if (bitmap_ && (size_ > 0)) {
+ mem_free(bitmap_);
+ bitmap_ = nullptr;
+ size_ = 0;
+ }
+}
+
+int BitMap::init(uint32_t item_size, bool init_as_zero) {
+ uint32_t size = (item_size + 7) / 8;
+ bitmap_ = static_cast<char*>(mem_alloc(size, MOD_TSBLOCK));
+ // need set to 0, otherwise there will be wrong data
+ const char initial_char = init_as_zero ? 0x00 : 0xFF;
+ memset(bitmap_, initial_char, size);
+ size_ = size;
+ init_as_zero_ = init_as_zero;
+ return common::E_OK;
+}
+
+} // namespace common
diff --git a/cpp/src/common/container/bit_map.h b/cpp/src/common/container/bit_map.h
new file mode 100644
index 00000000..371f29cd
--- /dev/null
+++ b/cpp/src/common/container/bit_map.h
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONTAINER_BIT_MAP_H
+#define COMMON_CONTAINER_BIT_MAP_H
+
+#include <string.h>
+
+#include "utils/errno_define.h"
+#include "utils/util_define.h"
+
+namespace common {
+
+class BitMap {
+ public:
+ BitMap() : bitmap_(nullptr), size_(0), init_as_zero_(true) {}
+ ~BitMap();
+ int init(uint32_t item_size, bool init_as_zero = true);
+
+ FORCE_INLINE void reset() {
+ const char initial_char = init_as_zero_ ? 0x00 : 0xFF;
+ memset(bitmap_, initial_char, size_);
+ }
+
+ FORCE_INLINE void set(uint32_t index) {
+ uint32_t offset = index >> 3;
+ ASSERT(offset < size_);
+
+ char *start_addr = bitmap_ + offset;
+ uint8_t bit_mask = get_bit_mask(index);
+ *start_addr = (*start_addr) | (bit_mask);
+ }
+
+ FORCE_INLINE void clear(uint32_t index) {
+ uint32_t offset = index >> 3;
+ ASSERT(offset < size_);
+
+ char *start_addr = bitmap_ + offset;
+ uint8_t bit_mask = get_bit_mask(index);
+ *start_addr = (*start_addr) ^ (~bit_mask);
+ }
+
+ FORCE_INLINE bool test(uint32_t index) {
+ uint32_t offset = index >> 3;
+ ASSERT(offset < size_);
+
+ char *start_addr = bitmap_ + offset;
+ uint8_t bit_mask = get_bit_mask(index);
+ return (*start_addr & bit_mask);
+ }
+
+ FORCE_INLINE uint32_t get_size() { return size_; }
+
+ FORCE_INLINE char *get_bitmap() { return bitmap_; } // for debug
+
+ private:
+ FORCE_INLINE uint8_t get_bit_mask(uint32_t index) {
+ return 1 << (index & 7);
+ }
+
+ private:
+ char *bitmap_;
+ uint32_t size_;
+ bool init_as_zero_;
+};
+} // namespace common
+
+#endif // COMMON_CONTAINER_BIT_MAP_H
diff --git a/cpp/src/common/container/blocking_queue.cc b/cpp/src/common/container/blocking_queue.cc
new file mode 100644
index 00000000..f5123d32
--- /dev/null
+++ b/cpp/src/common/container/blocking_queue.cc
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "blocking_queue.h"
+
+namespace common {
+
+BlockingQueue::BlockingQueue() : queue_(), mutex_(), cond_() {
+ pthread_mutex_init(&mutex_, NULL);
+ pthread_cond_init(&cond_, NULL);
+}
+
+BlockingQueue::~BlockingQueue() {
+ pthread_mutex_destroy(&mutex_);
+ pthread_cond_destroy(&cond_);
+}
+
+void BlockingQueue::push(void *data) {
+ pthread_mutex_lock(&mutex_);
+ queue_.push(data);
+ pthread_mutex_unlock(&mutex_);
+ /*
+ * it is safe to signal after unlock.
+ * since pthread_cond_wait is guarantee to unlock and sleep atomically.
+ */
+ pthread_cond_signal(&cond_);
+}
+
+void *BlockingQueue::pop() {
+ void *ret_data = NULL;
+ pthread_mutex_lock(&mutex_);
+ while (queue_.empty()) {
+ pthread_cond_wait(&cond_, &mutex_);
+ }
+ ret_data = queue_.front();
+ queue_.pop();
+ pthread_mutex_unlock(&mutex_);
+ return ret_data;
+}
+
+} // end namespace common
\ No newline at end of file
diff --git a/cpp/common/include/common.hpp b/cpp/src/common/container/blocking_queue.h
similarity index 55%
rename from cpp/common/include/common.hpp
rename to cpp/src/common/container/blocking_queue.h
index 6eb9830c..e03ea3a4 100644
--- a/cpp/common/include/common.hpp
+++ b/cpp/src/common/container/blocking_queue.h
@@ -4,10 +4,10 @@
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
+ * License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * https://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -16,15 +16,29 @@
* specific language governing permissions and limitations
* under the License.
*/
-#ifndef TSFILE_COMMON_H
-#define TSFILE_COMMON_H
+#ifndef COMMON_CONTAINER_BLOCKING_QUEUE_H
+#define COMMON_CONTAINER_BLOCKING_QUEUE_H
-class Common
-{
-public:
- Common();
+#include <pthread.h>
- int doSomething(int a, int b) const;
+#include <queue>
+
+namespace common {
+
+class BlockingQueue {
+ public:
+ BlockingQueue();
+ ~BlockingQueue();
+
+ void push(void* data);
+ // if empty, blocking
+ void* pop();
+
+ private:
+ std::queue<void*> queue_;
+ pthread_mutex_t mutex_;
+ pthread_cond_t cond_;
};
-#endif
\ No newline at end of file
+} // end namespace common
+#endif // COMMON_CONTAINER_BLOCKING_QUEUE_H
diff --git a/cpp/src/common/container/byte_buffer.h b/cpp/src/common/container/byte_buffer.h
new file mode 100644
index 00000000..967d3640
--- /dev/null
+++ b/cpp/src/common/container/byte_buffer.h
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONTAINER_BYTE_BUFFER_H
+#define COMMON_CONTAINER_BYTE_BUFFER_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "common/allocator/alloc_base.h"
+#include "common/global.h"
+#include "utils/util_define.h"
+
+namespace common {
+class ByteBuffer {
+ public:
+ ByteBuffer()
+ : data_(nullptr),
+ variable_type_len_(sizeof(uint32_t)),
+ real_data_size_(0),
+ reserved_size_(0) {}
+
+ ~ByteBuffer() {
+ if (data_ && (reserved_size_ > 0)) {
+ mem_free(data_);
+ data_ = nullptr;
+ real_data_size_ = 0;
+ reserved_size_ = 0;
+ }
+ }
+
+ FORCE_INLINE void init(uint32_t size) {
+ data_ = static_cast<char *>(mem_alloc(size, MOD_TSBLOCK));
+ reserved_size_ = size;
+ }
+
+ FORCE_INLINE void reset() { real_data_size_ = 0; }
+
+ FORCE_INLINE void extend_memory(uint32_t new_size) {
+ ASSERT(new_size > reserved_size_);
+ data_ = static_cast<char *>(mem_realloc(data_, new_size));
+ reserved_size_ = new_size;
+ }
+
+ FORCE_INLINE void append_variable_value(const char *value, uint32_t len) {
+ // dynamic growth
+ if (UNLIKELY((real_data_size_ + len + variable_type_len_) >
+ reserved_size_)) {
+ // extreme scenarios, when encountering very long string
+ uint32_t growth_size =
+ g_config_value_.tsblock_mem_inc_step_size_ > len
+ ? g_config_value_.tsblock_mem_inc_step_size_
+ : (len + 1);
+ extend_memory(reserved_size_ + growth_size);
+ }
+
+ ASSERT(data_);
+ // append len
+ memcpy(&data_[real_data_size_], reinterpret_cast<char *>(&len),
+ variable_type_len_);
+ real_data_size_ += variable_type_len_;
+ if (len > 0) {
+ // append data
+ memcpy(&data_[real_data_size_], value, len);
+ real_data_size_ += len;
+ }
+ }
+
+ FORCE_INLINE void append_fixed_value(const char *value, uint32_t len) {
+ // dynamic growth
+ if (UNLIKELY(real_data_size_ + len > reserved_size_)) {
+ // extreme scenarios, when encountering very long string
+ uint32_t growth_size =
+ g_config_value_.tsblock_mem_inc_step_size_ > len
+ ? g_config_value_.tsblock_mem_inc_step_size_
+ : (len + 1);
+ extend_memory(reserved_size_ + growth_size);
+ }
+
+ ASSERT(data_);
+ memcpy(&data_[real_data_size_], value, len);
+ real_data_size_ += len;
+ }
+
+ // for fixed len value
+ FORCE_INLINE char *read(uint32_t offset, uint32_t len) {
+ ASSERT((offset + len) <= real_data_size_);
+ char *p = &data_[offset];
+ return p;
+ }
+
+ // for variable len value
+ FORCE_INLINE char *read(uint32_t offset, uint32_t *len) {
+ // get len
+ *len = *reinterpret_cast<uint32_t *>(&data_[offset]);
+ // get value
+ char *p = &data_[offset + variable_type_len_];
+ return p;
+ }
+
+ FORCE_INLINE char *get_data() { return data_; }
+
+ private:
+ char *data_;
+ uint8_t variable_type_len_;
+ uint32_t real_data_size_;
+ uint32_t reserved_size_; // malloc memory size from system
+};
+
+} // namespace common
+#endif // COMMON_CONTAINER_BYTE_BUFFER_H
\ No newline at end of file
diff --git a/cpp/src/common/container/hash_func.h b/cpp/src/common/container/hash_func.h
new file mode 100644
index 00000000..8bac3170
--- /dev/null
+++ b/cpp/src/common/container/hash_func.h
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONTAINER_HASH_FUNC_H
+#define COMMON_CONTAINER_HASH_FUNC_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "common/container/hash_table.h"
+#include "common/container/murmur_hash3.h"
+#include "common/container/slice.h"
+#include "utils/db_utils.h"
+
+#define TSID_MAX_LEN 20
+
+namespace common {
+
+/*
+ * using djb2 hash algorithm
+ */
+struct StringHashFunc {
+ uint32_t hash_fmix32(uint32_t h) {
+ h ^= h >> 16;
+ h *= 0x3243f6a9U;
+ h ^= h >> 16;
+ return h;
+ }
+
+ uint32_t operator()(const void *data) {
+ uint32_t hash_ = (const uint32_t)5381;
+ const char *str = (const char *)data;
+ char c;
+ while ((c = *str++)) {
+ hash_ = ((hash_ << 5) + hash_) + c;
+ }
+ return hash_fmix32(hash_);
+ }
+};
+
+/*
+ * using djb2 hash algorithm
+ */
+struct SliceHashFunc {
+ uint32_t hash_fmix32(uint32_t h) {
+ h ^= h >> 16;
+ h *= 0x3243f6a9U;
+ h ^= h >> 16;
+ return h;
+ }
+
+ uint32_t operator()(const Slice &slice) {
+ uint32_t hash_ = (const uint32_t)5381;
+ const char *str = (const char *)slice.data();
+ char c;
+ while ((c = *str++)) {
+ hash_ = ((hash_ << 5) + hash_) + c;
+ }
+ return hash_fmix32(hash_);
+ }
+};
+
+/*
+ * using murmur_hash hash algorithm
+ */
+struct TsIDHashFunc {
+ uint32_t hash_fmix32(uint32_t h) {
+ h ^= h >> 16;
+ h *= 0x3243f6a9U;
+ h ^= h >> 16;
+ return h;
+ }
+
+ uint32_t operator()(const TsID &data) {
+ int32_t tmp = data.db_nid_ * 10000 + data.device_nid_ * 100 +
+ data.measurement_nid_;
+ return Murmur128Hash::hash(tmp, 0);
+ }
+};
+
+/*
+ * using murmur_hash hash algorithm
+ */
+struct NodeIDHashFunc {
+ uint32_t operator()(const NodeID &data) {
+ return Murmur128Hash::hash(static_cast<int32_t>(data), 0);
+ }
+};
+
+} // end namespace common
+#endif // COMMON_CONTAINER_HASH_FUNC_H
diff --git a/cpp/src/common/container/hash_node.h b/cpp/src/common/container/hash_node.h
new file mode 100644
index 00000000..b51d0113
--- /dev/null
+++ b/cpp/src/common/container/hash_node.h
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONTAINER_HASH_NODE_H
+#define COMMON_CONTAINER_HASH_NODE_H
+
+#include "utils/util_define.h"
+
+namespace common {
+
+template <class KeyType, class ValueType>
+class HashNode {
+ public:
+ HashNode() {
+ is_valid_ = false;
+ rehash_ = 0;
+ next_ = nullptr;
+ }
+
+ HashNode(const KeyType &key, const ValueType &value)
+ : key_(key),
+ value_(value),
+ is_valid_(true),
+ rehash_(0),
+ next_(nullptr) {}
+
+ ~HashNode() { is_valid_ = false; }
+
+ FORCE_INLINE KeyType get_key() const { return key_; }
+
+ FORCE_INLINE ValueType get_value() const { return value_; }
+
+ FORCE_INLINE void set_key(const KeyType &key) { key_ = key; }
+
+ FORCE_INLINE void set_value(const ValueType &value) { value_ = value; }
+
+ FORCE_INLINE HashNode *get_next() const { return next_; }
+
+ FORCE_INLINE void set_next(HashNode *next) { next_ = next; }
+
+ FORCE_INLINE bool get_valid() const { return is_valid_; }
+
+ FORCE_INLINE void set_valid(bool vd) { is_valid_ = vd; }
+
+ FORCE_INLINE uint64_t get_rehashid() const { return rehash_; }
+
+ FORCE_INLINE void set_rehashid(uint64_t id) { rehash_ = id; }
+
+ public:
+ bool is_valid_;
+ uint64_t rehash_;
+ KeyType key_;
+ ValueType value_;
+ HashNode *next_; // pointer to the next node (same bucket)
+};
+
+} // end namespace common
+#endif // COMMON_CONTAINER_HASH_NODE_H
\ No newline at end of file
diff --git a/cpp/src/common/container/hash_segm.h b/cpp/src/common/container/hash_segm.h
new file mode 100644
index 00000000..df0e5396
--- /dev/null
+++ b/cpp/src/common/container/hash_segm.h
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONTAINER_HASH_SEGM_H
+#define COMMON_CONTAINER_HASH_SEGM_H
+
+#include <stddef.h>
+
+#include "common/container/hash_node.h"
+#include "common/mutex/mutex.h"
+#include "utils/errno_define.h"
+#include "utils/util_define.h"
+
+#define SEGMENT_CAPACITY 100
+
+namespace common {
+
+template <class KeyType, class ValueType>
+class HashSegm {
+ public:
+ HashSegm() {}
+ ~HashSegm() {}
+
+ FORCE_INLINE HashNode<KeyType, ValueType>& operator[](size_t idx) {
+ if (idx >= SEGMENT_CAPACITY) {
+ // log_err("index %lu is out of range.", idx);
+ ASSERT(idx < SEGMENT_CAPACITY);
+ }
+ return segm_pointer_[idx];
+ }
+
+ public:
+ Mutex mutexes_[SEGMENT_CAPACITY];
+ HashNode<KeyType, ValueType> segm_pointer_[SEGMENT_CAPACITY];
+};
+
+} // end namespace common
+#endif // COMMON_CONTAINER_HASH_SEGM_H
\ No newline at end of file
diff --git a/cpp/src/common/container/hash_table.h b/cpp/src/common/container/hash_table.h
new file mode 100644
index 00000000..b344706b
--- /dev/null
+++ b/cpp/src/common/container/hash_table.h
@@ -0,0 +1,792 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONTAINER_HASH_TABLE_H
+#define COMMON_CONTAINER_HASH_TABLE_H
+
+#include "common/allocator/alloc_base.h"
+#include "common/container/array.h"
+#include "common/container/hash_func.h"
+#include "common/container/hash_node.h"
+#include "common/container/hash_segm.h"
+
+#define TABLE_CAPACITY 100
+#define TABLE_GROWTH 2
+
+namespace common {
+
+template <typename KeyType, typename ValueType, typename F>
+class HashTable {
+ public:
+ HashTable() : table_(TABLE_CAPACITY) {
+ new_table_capacity_ = TABLE_CAPACITY;
+ old_table_capacity_ = TABLE_CAPACITY;
+ size_ = 0;
+ is_inited_ = false;
+ hash_version_ = 0;
+ is_extending_ = 0;
+ }
+
+ HashTable(size_t total_capacity)
+ : table_(total_capacity / SEGMENT_CAPACITY) {
+ new_table_capacity_ = total_capacity / SEGMENT_CAPACITY;
+ old_table_capacity_ = new_table_capacity_;
+ size_ = 0;
+ is_inited_ = false;
+ hash_version_ = 0;
+ is_extending_ = 0;
+ }
+ ~HashTable() {
+ new_table_capacity_ = 0;
+ size_ = 0;
+ is_inited_ = false;
+ hash_version_ = 0;
+ is_extending_ = 0;
+ }
+
+ int init() {
+ if (is_inited_) {
+ // log_err("init repeated.");
+ ASSERT(!is_inited_);
+ }
+
+ int ret = E_OK;
+ ret = table_.init();
+ if (ret != E_OK) {
+ return E_OOM;
+ }
+ for (size_t i = 0; i < new_table_capacity_; i++) {
+ void* buf =
+ mem_alloc(sizeof(HashSegm<KeyType, ValueType>), MOD_HASH_TABLE);
+ if (UNLIKELY(nullptr == buf)) {
+ // log_err("malloc() failed.");
+ return E_OOM;
+ }
+ HashSegm<KeyType, ValueType>* tmp_segm =
+ new (buf) HashSegm<KeyType, ValueType>;
+ table_.append(tmp_segm);
+ }
+ is_inited_ = true;
+ return E_OK;
+ }
+
+ void destroy() {
+ if (is_inited_) {
+ for (size_t i = 0; i < new_table_capacity_; i++) {
+ for (size_t j = 0; j < SEGMENT_CAPACITY; j++) {
+ if ((*table_[i])[j].get_valid()) {
+ HashNode<KeyType, ValueType>* prev = nullptr;
+ HashNode<KeyType, ValueType>* entry =
+ (*table_[i])[j].get_next();
+ while (entry != nullptr) {
+ prev = entry;
+ entry = entry->get_next();
+ prev->~HashNode();
+ mem_free(prev);
+ prev = nullptr;
+ }
+ (*table_[i])[j].set_valid(false);
+ }
+ }
+ table_[i]->~HashSegm();
+ mem_free(table_[i]);
+ table_[i] = nullptr;
+ }
+ table_.destroy();
+ }
+ is_inited_ = false;
+ }
+
+ int table_extend() {
+ int ret = table_.extend();
+ if (ret != E_OK) {
+ return ret;
+ }
+ for (size_t i = ATOMIC_LOAD(&new_table_capacity_);
+ i < table_.capacity(); i++) {
+ void* buf =
+ mem_alloc(sizeof(HashSegm<KeyType, ValueType>), MOD_HASH_TABLE);
+ if (UNLIKELY(nullptr == buf)) {
+ // log_err("malloc() failed.");
+ return E_OOM;
+ }
+ HashSegm<KeyType, ValueType>* tmp_segm =
+ new (buf) HashSegm<KeyType, ValueType>;
+ table_.append(tmp_segm);
+ }
+ return E_OK;
+ }
+
+ int move_bucket_head(size_t from_segment_idx, size_t from_bucket_idx) {
+ HashNode<KeyType, ValueType>* prev =
+ &((*table_[from_segment_idx])[from_bucket_idx]);
+ HashNode<KeyType, ValueType>* entry =
+ ((*table_[from_segment_idx])[from_bucket_idx]).get_next();
+
+ while (
+ (*table_[from_segment_idx])[from_bucket_idx].get_rehashid() !=
+ ATOMIC_LOAD(&hash_version_)) { // if the hashnode hasn't rehashed
+ uint32_t hash_value = hashfunc_(
+ (*table_[from_segment_idx])[from_bucket_idx].get_key());
+ size_t to_segment_idx =
+ hash_value % ATOMIC_LOAD(&new_table_capacity_);
+ size_t to_bucket_idx = hash_value % SEGMENT_CAPACITY;
+ if (from_segment_idx == to_segment_idx &&
+ from_bucket_idx == to_bucket_idx) {
+ ((*table_[to_segment_idx])[to_bucket_idx])
+ .set_rehashid(
+ ATOMIC_LOAD(&hash_version_)); // only update rehashid
+ } else {
+ // put
+ table_[to_segment_idx]->mutexes_[to_bucket_idx].lock();
+ int ret = put_nolock(
+ (*table_[from_segment_idx])[from_bucket_idx].get_key(),
+ (*table_[from_segment_idx])[from_bucket_idx].get_value(),
+ to_segment_idx, to_bucket_idx);
+ table_[to_segment_idx]->mutexes_[to_bucket_idx].unlock();
+ if (ret != E_OK) {
+ return ret;
+ }
+ // remove
+ if (entry != nullptr) {
+ ((*table_[from_segment_idx])[from_bucket_idx])
+ .set_key(entry->get_key());
+ ((*table_[from_segment_idx])[from_bucket_idx])
+ .set_value(entry->get_value());
+ ((*table_[from_segment_idx])[from_bucket_idx])
+ .set_next(entry->get_next());
+ ((*table_[from_segment_idx])[from_bucket_idx])
+ .set_valid(true);
+ ((*table_[from_segment_idx])[from_bucket_idx])
+ .set_rehashid(entry->get_rehashid());
+ ATOMIC_AAF(&size_, -1);
+ entry->~HashNode();
+ mem_free(entry);
+ entry = prev->get_next();
+ } else {
+ ((*table_[from_segment_idx])[from_bucket_idx])
+ .set_valid(false);
+ ATOMIC_AAF(&size_, -1);
+ break;
+ }
+ }
+ }
+
+ return E_OK;
+ }
+
+ int move_bucket_chain(size_t from_segment_idx, size_t from_bucket_idx) {
+ HashNode<KeyType, ValueType>* prev =
+ &((*table_[from_segment_idx])[from_bucket_idx]);
+ HashNode<KeyType, ValueType>* entry =
+ ((*table_[from_segment_idx])[from_bucket_idx]).get_next();
+
+ while (entry != nullptr) {
+ if (entry->get_rehashid() != ATOMIC_LOAD(&hash_version_)) {
+ uint32_t hash_value = hashfunc_(entry->get_key());
+ size_t to_segment_idx =
+ hash_value % ATOMIC_LOAD(&new_table_capacity_);
+ size_t to_bucket_idx = hash_value % SEGMENT_CAPACITY;
+ if (from_segment_idx == to_segment_idx &&
+ from_bucket_idx == to_bucket_idx) {
+ entry->set_rehashid(ATOMIC_LOAD(&hash_version_));
+ prev = entry;
+ entry = entry->get_next();
+ } else {
+ table_[to_segment_idx]->mutexes_[to_bucket_idx].lock();
+ if ((*table_[to_segment_idx])[to_bucket_idx].get_valid() ==
+ false) {
+ ((*table_[to_segment_idx])[to_bucket_idx])
+ .set_key(entry->get_key());
+ ((*table_[to_segment_idx])[to_bucket_idx])
+ .set_value(entry->get_value());
+ ((*table_[to_segment_idx])[to_bucket_idx])
+ .set_next(nullptr);
+ ((*table_[to_segment_idx])[to_bucket_idx])
+ .set_valid(true);
+ ((*table_[to_segment_idx])[to_bucket_idx])
+ .set_rehashid(ATOMIC_LOAD(&hash_version_));
+ prev->set_next(entry->get_next());
+ entry->~HashNode();
+ mem_free(entry);
+ entry = prev->get_next();
+ // entry = entry->get_next();
+ } else {
+ entry->set_rehashid(ATOMIC_LOAD(&hash_version_));
+ prev->set_next(entry->get_next());
+ entry->set_next((*table_[to_segment_idx])[to_bucket_idx]
+ .get_next());
+ (*table_[to_segment_idx])[to_bucket_idx].set_next(
+ entry);
+ entry = prev->get_next();
+ }
+ table_[to_segment_idx]->mutexes_[to_bucket_idx].unlock();
+ }
+ } else {
+ prev = prev->get_next();
+ entry = entry->get_next();
+ }
+ }
+ return E_OK;
+ }
+
+ int extend() {
+ int ret = table_extend();
+ if (ret != E_OK) {
+ return ret;
+ }
+ ATOMIC_STORE(&old_table_capacity_, ATOMIC_LOAD(&new_table_capacity_));
+ ATOMIC_STORE(&new_table_capacity_, table_.capacity());
+ // ATOMIC_STORE(&hash_total_capacity_, table_.capacity() *
+ // SEGMENT_CAPACITY);
+
+ int64_t next_hash_version =
+ ATOMIC_LOAD(&hash_version_) +
+ 1; // the highest position of next_hash_version is 0
+ ATOMIC_STORE(&hash_version_,
+ next_hash_version |
+ (1UL << 63)); // set the highest position to 1
+ // which means 'extend' is beginning.
+
+ // rehash
+ for (size_t from_segment_idx = 0;
+ from_segment_idx < ATOMIC_LOAD(&old_table_capacity_);
+ from_segment_idx++) {
+ for (size_t from_bucket_idx = 0; from_bucket_idx < SEGMENT_CAPACITY;
+ from_bucket_idx++) {
+ table_[from_segment_idx]->mutexes_[from_bucket_idx].lock();
+ if ((*table_[from_segment_idx])[from_bucket_idx].get_valid()) {
+ ret = move_bucket_head(from_segment_idx, from_bucket_idx);
+ if (ret != E_OK) {
+ table_[from_segment_idx]
+ ->mutexes_[from_bucket_idx]
+ .unlock();
+ return ret;
+ }
+ ret = move_bucket_chain(from_segment_idx, from_bucket_idx);
+ if (ret != E_OK) {
+ table_[from_segment_idx]
+ ->mutexes_[from_bucket_idx]
+ .unlock();
+ return ret;
+ }
+ }
+ table_[from_segment_idx]->mutexes_[from_bucket_idx].unlock();
+ }
+ }
+
+ ATOMIC_STORE(
+ &hash_version_,
+ next_hash_version & (~(1UL << 63))); // set extending flag is false
+
+ return E_OK;
+ }
+
+ /*
+ * when the key already exists, just update old value to new value;
+ */
+ int put(const KeyType& key, const ValueType& value) {
+ int ret;
+ uint32_t hash_value = hashfunc_(key);
+
+ while (true) {
+ uint8_t cur_table_capacity = ATOMIC_LOAD(&new_table_capacity_);
+ size_t segment_idx = hash_value % cur_table_capacity;
+ size_t bucket_idx = hash_value % SEGMENT_CAPACITY;
+
+ table_[segment_idx]->mutexes_[bucket_idx].lock();
+ ret = put_nolock(key, value, segment_idx, bucket_idx);
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ if (ret != E_OK) {
+ return ret;
+ }
+
+ if (UNLIKELY(
+ cur_table_capacity !=
+ ATOMIC_LOAD(
+ &new_table_capacity_))) { // resolve the problem of
+ // insert into old position
+ ret = remove_from_wrong_bucket(key, segment_idx, bucket_idx);
+ // if (ret != E_OK) {
+ // return ret;
+ // }
+ continue;
+ } else {
+ break;
+ }
+ }
+
+ if (ATOMIC_LOAD(&size_) > ATOMIC_LOAD(&new_table_capacity_) *
+ SEGMENT_CAPACITY * TABLE_GROWTH) {
+ int tmp = 0;
+ if (ATOMIC_CAS(&is_extending_, &tmp, 1)) {
+ ret = extend();
+ if (ret != E_OK) {
+ ATOMIC_AAF(&is_extending_, -1);
+ return ret;
+ }
+ ATOMIC_AAF(&is_extending_, -1);
+ }
+ }
+
+ return ret;
+ }
+
+ /*
+ * when the key already exists, just update old value to new value;
+ */
+ int put_nolock(const KeyType& key, const ValueType& value,
+ size_t segment_idx, size_t bucket_idx) {
+ if (((*table_[segment_idx])[bucket_idx]).get_valid() ==
+ false) { // if current segment has no valid node
+ ((*table_[segment_idx])[bucket_idx]).set_key(key);
+ ((*table_[segment_idx])[bucket_idx]).set_value(value);
+ ((*table_[segment_idx])[bucket_idx]).set_next(nullptr);
+ ((*table_[segment_idx])[bucket_idx]).set_valid(true);
+ ((*table_[segment_idx])[bucket_idx])
+ .set_rehashid(ATOMIC_LOAD(&hash_version_));
+ ATOMIC_AAF(&size_, 1);
+ } else if (((*table_[segment_idx])[bucket_idx]).get_key() ==
+ key) { // if current segment's first node's key is same as
+ // the target key
+ ((*table_[segment_idx])[bucket_idx])
+ .set_value(value); // update value of existing node
+ ((*table_[segment_idx])[bucket_idx])
+ .set_rehashid(ATOMIC_LOAD(&hash_version_));
+ } else { // follow the link list
+ HashNode<KeyType, ValueType>* prev =
+ &((*table_[segment_idx])[bucket_idx]);
+ HashNode<KeyType, ValueType>* entry =
+ ((*table_[segment_idx])[bucket_idx]).get_next();
+ while (entry != nullptr && entry->get_key() != key) {
+ prev = entry;
+ entry = entry->get_next();
+ }
+ if (LIKELY(entry == nullptr)) {
+ void* buf = mem_alloc(sizeof(HashNode<KeyType, ValueType>),
+ MOD_HASH_TABLE);
+ if (UNLIKELY(nullptr == buf)) {
+ // log_err("malloc() failed.");
+ return E_OOM;
+ }
+ entry = new (buf) HashNode<KeyType, ValueType>;
+ entry->set_key(key);
+ entry->set_value(value);
+ entry->set_valid(true);
+ entry->set_rehashid(ATOMIC_LOAD(&hash_version_));
+ prev->set_next(entry);
+ ATOMIC_AAF(&size_, 1);
+ } else {
+ entry->set_value(value); // update value of existing node
+ entry->set_rehashid(ATOMIC_LOAD(&hash_version_));
+ }
+ }
+ return E_OK;
+ }
+
+ /*
+ * when the key already exists, then remain the old value and return the old
+ * value and set 'exist' to true
+ */
+ int put_no_update(const KeyType& key, ValueType& value, bool& exist) {
+ int ret;
+ uint32_t hash_value = hashfunc_(key);
+
+ while (true) {
+ uint8_t cur_table_capacity = ATOMIC_LOAD(&new_table_capacity_);
+ size_t segment_idx = hash_value % cur_table_capacity;
+ size_t bucket_idx = hash_value % SEGMENT_CAPACITY;
+
+ table_[segment_idx]->mutexes_[bucket_idx].lock();
+ ret = put_nolock_no_update(key, value, segment_idx, bucket_idx,
+ exist);
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ if (ret != E_OK) {
+ return ret;
+ }
+
+ if (UNLIKELY(
+ cur_table_capacity !=
+ ATOMIC_LOAD(
+ &new_table_capacity_))) { // resolve the problem of
+ // insert into old position
+ ret = remove_from_wrong_bucket(key, segment_idx, bucket_idx);
+ // if (ret != E_OK) {
+ // return ret;
+ // }
+ continue;
+ } else {
+ break;
+ }
+ }
+
+ if (ATOMIC_LOAD(&size_) > ATOMIC_LOAD(&new_table_capacity_) *
+ SEGMENT_CAPACITY * TABLE_GROWTH) {
+ int tmp = 0;
+ if (ATOMIC_CAS(&is_extending_, &tmp, 1)) {
+ ret = extend();
+ if (ret != E_OK) {
+ return ret;
+ }
+ ATOMIC_AAF(&is_extending_, -1);
+ }
+ }
+
+ return ret;
+ }
+
+ /*
+ * when the key already exists, then remain the old value and return the old
+ * value and set 'exist' to true
+ */
+ int put_nolock_no_update(const KeyType& key, ValueType& value,
+ size_t segment_idx, size_t bucket_idx,
+ bool& exist) {
+ if (((*table_[segment_idx])[bucket_idx]).get_valid() ==
+ false) { // if current segment has no valid node
+ ((*table_[segment_idx])[bucket_idx]).set_key(key);
+ ((*table_[segment_idx])[bucket_idx]).set_value(value);
+ ((*table_[segment_idx])[bucket_idx]).set_next(nullptr);
+ ((*table_[segment_idx])[bucket_idx]).set_valid(true);
+ ((*table_[segment_idx])[bucket_idx])
+ .set_rehashid(ATOMIC_LOAD(&hash_version_));
+ ATOMIC_AAF(&size_, 1);
+ } else if (((*table_[segment_idx])[bucket_idx]).get_key() ==
+ key) { // if current segment's first node's key is same as
+ // the target key
+ exist = true;
+ value = ((*table_[segment_idx])[bucket_idx]).get_value();
+ ((*table_[segment_idx])[bucket_idx])
+ .set_rehashid(ATOMIC_LOAD(&hash_version_));
+ } else { // follow the link list
+ HashNode<KeyType, ValueType>* prev =
+ &((*table_[segment_idx])[bucket_idx]);
+ HashNode<KeyType, ValueType>* entry =
+ ((*table_[segment_idx])[bucket_idx]).get_next();
+ while (entry != nullptr && entry->get_key() != key) {
+ prev = entry;
+ entry = entry->get_next();
+ }
+ if (LIKELY(entry == nullptr)) {
+ void* buf = mem_alloc(sizeof(HashNode<KeyType, ValueType>),
+ MOD_HASH_TABLE);
+ if (UNLIKELY(nullptr == buf)) {
+ // log_err("malloc() failed.");
+ return E_OOM;
+ }
+ entry = new (buf) HashNode<KeyType, ValueType>;
+ entry->set_key(key);
+ entry->set_value(value);
+ entry->set_valid(true);
+ entry->set_rehashid(ATOMIC_LOAD(&hash_version_));
+ prev->set_next(entry);
+ ATOMIC_AAF(&size_, 1);
+ } else {
+ exist = true;
+ value = entry->get_value();
+ entry->set_rehashid(ATOMIC_LOAD(&hash_version_));
+ }
+ }
+ return E_OK;
+ }
+
+ int get(const KeyType& key, ValueType& result_value) {
+ if (!ATOMIC_LOAD(&size_)) {
+ return E_NOT_EXIST;
+ }
+ int ret;
+ ValueType result1;
+ ValueType result2;
+ ValueType result3;
+ while (true) {
+ uint64_t hv1 = ATOMIC_LOAD(&hash_version_);
+
+ if ((hv1 & (1UL << 63))) {
+ ret = get_by_bucket(key, result1,
+ ATOMIC_LOAD(&old_table_capacity_));
+ if (ret == E_OK) {
+ result_value = result1;
+ return ret;
+ }
+ ret = get_by_bucket(key, result2,
+ ATOMIC_LOAD(&new_table_capacity_));
+ if (ret == E_OK) {
+ result_value = result2;
+ return ret;
+ }
+ } else { //
+ ret = get_by_bucket(key, result3,
+ ATOMIC_LOAD(&new_table_capacity_));
+ if (ret == E_OK) {
+ result_value = result3;
+ return ret;
+ }
+ }
+
+ uint64_t hv2 = ATOMIC_LOAD(&hash_version_);
+ if ((hv1 & (~(1UL << 63))) != (hv2 & (~(1UL << 63)))) {
+ continue;
+ } else if (ATOMIC_LOAD(&is_extending_)) {
+ continue;
+ } else {
+ return E_NOT_EXIST;
+ }
+ }
+ }
+
+ int get_by_bucket(const KeyType& key, ValueType& result_value,
+ size_t table_capacity) {
+ uint32_t hash_value = hashfunc_(key);
+ size_t segment_idx = hash_value % table_capacity;
+ size_t bucket_idx = hash_value % SEGMENT_CAPACITY;
+
+ table_[segment_idx]->mutexes_[bucket_idx].lock();
+
+ if (!((*table_[segment_idx])[bucket_idx])
+ .get_valid()) { // if current segment has no node
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ return E_NOT_EXIST;
+ } else if (((*table_[segment_idx])[bucket_idx]).get_key() == key) {
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ result_value = ((*table_[segment_idx])[bucket_idx]).get_value();
+ return E_OK;
+ } else {
+ HashNode<KeyType, ValueType>* entry =
+ ((*table_[segment_idx])[bucket_idx]).get_next();
+ while (entry != nullptr) {
+ if (entry->get_key() == key) {
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ result_value = entry->get_value();
+ return E_OK;
+ }
+ entry = entry->get_next();
+ }
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ return E_NOT_EXIST;
+ }
+ }
+
+ int remove(const KeyType& key) {
+ if (!ATOMIC_LOAD(&size_)) {
+ return E_NOT_EXIST;
+ }
+ int ret;
+ while (true) {
+ uint64_t hv1 = ATOMIC_LOAD(&hash_version_);
+
+ if ((hv1 & (1UL << 63))) {
+ ret = remove_by_bucket(key, ATOMIC_LOAD(&old_table_capacity_));
+ if (ret == E_OK) {
+ return ret;
+ }
+ ret = remove_by_bucket(key, ATOMIC_LOAD(&new_table_capacity_));
+ if (ret == E_OK) {
+ return ret;
+ }
+ } else {
+ ret = remove_by_bucket(key, ATOMIC_LOAD(&new_table_capacity_));
+ if (ret == E_OK) {
+ return ret;
+ }
+ }
+
+ uint64_t hv2 = ATOMIC_LOAD(&hash_version_);
+ if ((hv1 & (~(1UL << 63))) != (hv2 & (~(1UL << 63)))) {
+ continue;
+ } else if (ATOMIC_LOAD(&is_extending_)) {
+ continue;
+ } else {
+ return E_NOT_EXIST;
+ }
+ }
+ }
+
+ int remove_by_bucket(const KeyType& key, size_t table_capacity) {
+ uint32_t hash_value = hashfunc_(key);
+ size_t segment_idx = hash_value % table_capacity;
+ size_t bucket_idx = hash_value % SEGMENT_CAPACITY;
+
+ table_[segment_idx]->mutexes_[bucket_idx].lock();
+
+ if (!((*table_[segment_idx])[bucket_idx])
+ .get_valid()) { // if current segment has no node
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ return E_NOT_EXIST;
+ } else if (((*table_[segment_idx])[bucket_idx]).get_key() == key) {
+ HashNode<KeyType, ValueType>* entry =
+ ((*table_[segment_idx])[bucket_idx]).get_next();
+ if (entry != nullptr) {
+ ((*table_[segment_idx])[bucket_idx]).set_key(entry->get_key());
+ ((*table_[segment_idx])[bucket_idx])
+ .set_value(entry->get_value());
+ ((*table_[segment_idx])[bucket_idx])
+ .set_next(entry->get_next());
+ ((*table_[segment_idx])[bucket_idx]).set_valid(true);
+ ((*table_[segment_idx])[bucket_idx])
+ .set_rehashid(entry->get_rehashid());
+ entry->~HashNode();
+ mem_free(entry);
+ entry = nullptr;
+ } else {
+ ((*table_[segment_idx])[bucket_idx]).set_valid(false);
+ }
+ ATOMIC_AAF(&size_, -1);
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ return E_OK;
+ } else {
+ HashNode<KeyType, ValueType>* prev =
+ &((*table_[segment_idx])[bucket_idx]);
+ HashNode<KeyType, ValueType>* entry =
+ ((*table_[segment_idx])[bucket_idx]).get_next();
+ while (entry != nullptr) {
+ if (entry->get_key() == key) {
+ prev->set_next(
+ entry->get_next()); // remove node and reconnect
+ entry->~HashNode();
+ mem_free(entry);
+ entry = nullptr;
+ ATOMIC_AAF(&size_, -1);
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ return E_OK;
+ }
+ prev = entry;
+ entry = entry->get_next();
+ }
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ return E_NOT_EXIST;
+ }
+ }
+
+ int remove_from_wrong_bucket(const KeyType& key, size_t segment_idx,
+ size_t bucket_idx) {
+ if (!ATOMIC_LOAD(&size_)) {
+ return E_NO_MORE_DATA;
+ }
+
+ table_[segment_idx]->mutexes_[bucket_idx].lock();
+
+ if (!((*table_[segment_idx])[bucket_idx])
+ .get_valid()) { // if current segment has no node
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ return E_NOT_EXIST;
+ } else if (((*table_[segment_idx])[bucket_idx]).get_key() == key) {
+ HashNode<KeyType, ValueType>* entry =
+ ((*table_[segment_idx])[bucket_idx]).get_next();
+ if (entry != nullptr) {
+ ((*table_[segment_idx])[bucket_idx]).set_key(entry->get_key());
+ ((*table_[segment_idx])[bucket_idx])
+ .set_value(entry->get_value());
+ ((*table_[segment_idx])[bucket_idx])
+ .set_next(entry->get_next());
+ ((*table_[segment_idx])[bucket_idx]).set_valid(true);
+ ((*table_[segment_idx])[bucket_idx])
+ .set_rehashid(entry->get_rehashid());
+ entry->~HashNode();
+ mem_free(entry);
+ entry = nullptr;
+ } else {
+ ((*table_[segment_idx])[bucket_idx]).set_valid(false);
+ }
+ ATOMIC_AAF(&size_, -1);
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ return E_OK;
+ } else {
+ HashNode<KeyType, ValueType>* prev =
+ &((*table_[segment_idx])[bucket_idx]);
+ HashNode<KeyType, ValueType>* entry =
+ ((*table_[segment_idx])[bucket_idx]).get_next();
+ while (entry != nullptr) {
+ if (entry->get_key() == key) {
+ prev->set_next(
+ entry->get_next()); // remove node and reconnect
+ entry->~HashNode();
+ mem_free(entry);
+ entry = nullptr;
+ ATOMIC_AAF(&size_, -1);
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ return E_OK;
+ }
+ prev = entry;
+ entry = entry->get_next();
+ }
+ table_[segment_idx]->mutexes_[bucket_idx].unlock();
+ return E_NOT_EXIST;
+ }
+ }
+
+#ifndef NDEBUG
+ void print() {
+ printf("<< The Hash Table >>\n");
+ if (!ATOMIC_AAF(&size_, 0)) {
+ printf(" Empty now!\n");
+ return;
+ }
+ size_t bucket_num = 0;
+ for (size_t i = 0; i < ATOMIC_AAF(&new_table_capacity_, 0); i++) {
+ for (size_t j = 0; j < SEGMENT_CAPACITY; j++) {
+ // printf("----(%ld)segment id is %ld, bucket id is %ld----\n",
+ // bucket_num++, i, j);
+ if ((*table_[i])[j].get_valid()) {
+ printf("----(%ld)segment id is %ld, bucket id is %ld----\n",
+ bucket_num++, i, j);
+ std::cout << "| "
+ << "key is " << (*table_[i])[j].get_key()
+ << ", value is " << (*table_[i])[j].get_value()
+ << ", rehashid is "
+ << (int)(*table_[i])[j].get_rehashid()
+ << std::endl;
+ HashNode<KeyType, ValueType>* entry =
+ ((*table_[i])[j]).get_next();
+ while (entry != nullptr) {
+ std::cout << "| "
+ << "key is " << entry->get_key()
+ << ", value is " << entry->get_value()
+ << ", rehashid is "
+ << (int)entry->get_rehashid() << std::endl;
+ entry = entry->get_next();
+ }
+ } else {
+ bucket_num++;
+ }
+ }
+ }
+ }
+#endif // #ifndef NDEBUG
+
+ FORCE_INLINE bool empty() const { return ATOMIC_LOAD(&size_) == 0; }
+
+ FORCE_INLINE size_t size() { return ATOMIC_LOAD(&size_); }
+
+ FORCE_INLINE size_t total_capacity() {
+ return ATOMIC_LOAD(&new_table_capacity_) * SEGMENT_CAPACITY;
+ }
+
+ private:
+ size_t new_table_capacity_;
+ size_t old_table_capacity_;
+ size_t size_;
+ uint64_t hash_version_;
+ Array<HashSegm<KeyType, ValueType>*>
+ table_; // the element in Array is pointer.
+ bool is_inited_;
+ uint8_t is_extending_;
+ F hashfunc_;
+};
+
+} // end namespace common
+#endif // COMMON_CONTAINER_HASH_TABLE_H
diff --git a/cpp/src/common/container/list.h b/cpp/src/common/container/list.h
new file mode 100644
index 00000000..271b9881
--- /dev/null
+++ b/cpp/src/common/container/list.h
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_CONTAINER_LIST_H
+#define COMMON_CONTAINER_LIST_H
+
+#include "common/allocator/page_arena.h"
+#include "utils/errno_define.h"
+
+namespace common {
+
+#define INVALID_NODE_PTR ((SimpleListNode *)0xABCDEF)
+
+/*
+ * A very simple list.
+ * There are such scenarios:
+ * We prepare all data in the list, and pass the list as immutable
+ * parameters to other module or caller. We never do delete or complex
+ * push operations.
+ * for example:
+ * SchemaMgr::get_timeseries_by_tsid(TsID, list<ColumnDesc>&)
+ * SchemaMgr::create_aligned_timeseries(..., const list<char*>
+ * &measurement_list, ...)
+ *
+ * Note: not thread safe!
+ */
+template <class T>
+class SimpleList {
+ private:
+ struct SimpleListNode {
+ T data_;
+ SimpleListNode *next_;
+
+ SimpleListNode() : next_(nullptr) {}
+ SimpleListNode(const T &data) : data_(data), next_(nullptr) {}
+ };
+
+ public:
+ class Iterator {
+ public:
+ Iterator(SimpleListNode *node) : cur_(node) {}
+ Iterator() : cur_(INVALID_NODE_PTR) {}
+ T &get() { return cur_->data_; }
+ FORCE_INLINE bool is_inited() const { return cur_ != INVALID_NODE_PTR; }
+ FORCE_INLINE Iterator &operator++(int n) {
+ if (LIKELY(cur_ != nullptr)) {
+ cur_ = cur_->next_;
+ }
+ return *this;
+ }
+ FORCE_INLINE bool operator!=(const Iterator &other) const {
+ return this->cur_ != other.cur_;
+ }
+ FORCE_INLINE bool operator==(const Iterator &other) const {
+ return this->cur_ == other.cur_;
+ }
+
+ private:
+ SimpleListNode *cur_;
+ };
+
+ public:
+ SimpleList(const uint32_t page_size, AllocModID mid)
+ : head_(nullptr), tail_(nullptr), size_(0) {
+ own_page_arena_.init(page_size, mid);
+ page_arena_ = &own_page_arena_;
+ }
+
+ SimpleList(PageArena *page_arena)
+ : page_arena_(page_arena), head_(nullptr), tail_(nullptr), size_(0) {
+ // page_arena_ should be destroy outside by caller
+ }
+
+ int push_back(const T &data) {
+ void *buf = page_arena_->alloc(sizeof(SimpleListNode));
+ if (UNLIKELY(buf == nullptr)) {
+ return common::E_OOM;
+ }
+ SimpleListNode *node = new (buf) SimpleListNode(data);
+ if (head_ == nullptr) {
+ head_ = node;
+ tail_ = node;
+ } else {
+ assert(tail_ != nullptr);
+ tail_->next_ = node;
+ tail_ = node;
+ }
+ size_++;
+ return common::E_OK;
+ }
+
+ FORCE_INLINE T &front() {
+ ASSERT(size_ > 0 && head_ != nullptr);
+ return head_->data_;
+ }
+
+ int remove(T target) {
+ if (head_ == nullptr) {
+ return common::E_NOT_EXIST;
+ }
+ SimpleListNode *prev = head_;
+ SimpleListNode *cur = head_->next_;
+ while (cur && cur->data_ != target) {
+ cur = cur->next_;
+ }
+ if (cur) {
+ return common::E_NOT_EXIST;
+ }
+ prev->next_ = cur->next_;
+ // cur is allocated from PageArena, it should not reclaim now
+ return common::E_OK;
+ }
+
+ FORCE_INLINE Iterator begin() const { return Iterator(head_); }
+ FORCE_INLINE Iterator end() const { return Iterator(nullptr); }
+ FORCE_INLINE uint32_t size() const { return size_; }
+ FORCE_INLINE void clear() {
+ head_ = nullptr;
+ tail_ = nullptr;
+ size_ = 0;
+ own_page_arena_.destroy();
+ }
+
+ private:
+ PageArena *page_arena_;
+ PageArena own_page_arena_;
+ SimpleListNode *head_;
+ SimpleListNode *tail_;
+ uint32_t size_;
+};
+
+} // end namespace common
+#endif // COMMON_CONTAINER_LIST_H
diff --git a/cpp/src/common/container/murmur_hash3.cc b/cpp/src/common/container/murmur_hash3.cc
new file mode 100644
index 00000000..14a06022
--- /dev/null
+++ b/cpp/src/common/container/murmur_hash3.cc
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "murmur_hash3.h"
+
+namespace common {
+
+/* ================ Murmur128Hash ================ */
+// follow Java IoTDB exactly.
+int64_t Murmur128Hash::inner_hash(const char *buf, int32_t len, int64_t seed) {
+ const int block_count = len >> 4; // as 128-bit blocks
+ int64_t h1 = seed;
+ int64_t h2 = seed;
+ int64_t c1 = 0x87c37b91114253d5L;
+ int64_t c2 = 0x4cf5ad432745937fL;
+
+ for (int i = 0; i < block_count; i++) {
+ int64_t k1 = get_block(buf, i * 2);
+ int64_t k2 = get_block(buf, i * 2 + 1);
+ k1 *= c1;
+ k1 = rotl64(k1, 31);
+ k1 *= c2;
+ h1 ^= k1;
+ h1 = rotl64(h1, 27);
+ h1 += h2;
+ h1 = h1 * 5 + 0x52dce729;
+ k2 *= c2;
+ k2 = rotl64(k2, 33);
+ k2 *= c1;
+ h2 ^= k2;
+ h2 = rotl64(h2, 31);
+ h2 += h1;
+ h2 = h2 * 5 + 0x38495ab5;
+ }
+
+ int offset = block_count * 16;
+ int64_t k1 = 0;
+ int64_t k2 = 0;
+ switch (len & 15) {
+ case 15:
+ k2 ^= ((int64_t)(buf[offset + 14])) << 48;
+ // fallthrough
+ case 14:
+ k2 ^= ((int64_t)(buf[offset + 13])) << 40;
+ // fallthrough
+ case 13:
+ k2 ^= ((int64_t)(buf[offset + 12])) << 32;
+ // fallthrough
+ case 12:
+ k2 ^= ((int64_t)(buf[offset + 11])) << 24;
+ // fallthrough
+ case 11:
+ k2 ^= ((int64_t)(buf[offset + 10])) << 16;
+ // fallthrough
+ case 10:
+ k2 ^= ((int64_t)(buf[offset + 9])) << 8;
+ // fallthrough
+ case 9:
+ k2 ^= buf[offset + 8];
+ k2 *= c2;
+ k2 = rotl64(k2, 33);
+ k2 *= c1;
+ h2 ^= k2;
+ // fallthrough
+ case 8:
+ k1 ^= ((int64_t)buf[offset + 7]) << 56;
+ // fallthrough
+ case 7:
+ k1 ^= ((int64_t)buf[offset + 6]) << 48;
+ // fallthrough
+ case 6:
+ k1 ^= ((int64_t)buf[offset + 5]) << 40;
+ // fallthrough
+ case 5:
+ k1 ^= ((int64_t)buf[offset + 4]) << 32;
+ // fallthrough
+ case 4:
+ k1 ^= ((int64_t)buf[offset + 3]) << 24;
+ // fallthrough
+ case 3:
+ k1 ^= ((int64_t)buf[offset + 2]) << 16;
+ // fallthrough
+ case 2:
+ k1 ^= ((int64_t)buf[offset + 1]) << 8;
+ // fallthrough
+ case 1:
+ k1 ^= buf[offset];
+ k1 *= c1;
+ k1 = rotl64(k1, 31);
+ k1 *= c2;
+ h1 ^= k1;
+ break;
+ default: // 0
+ // do nothing
+ break;
+ }
+
+ h1 ^= len;
+ h2 ^= len;
+ h1 += h2;
+ h2 += h1;
+ h1 = fmix(h1);
+ h2 = fmix(h2);
+ h1 += h2;
+ h2 += h1;
+ return h1 + h2;
+}
+
+int64_t Murmur128Hash::get_block(const char *buf, int32_t index) {
+ int block_offset = index << 3;
+ int64_t res = 0;
+ res += ((int64_t)(buf[block_offset + 0] & 0xFF));
+ res += (((int64_t)(buf[block_offset + 1] & 0xFF)) << 8);
+ res += (((int64_t)(buf[block_offset + 2] & 0xFF)) << 16);
+ res += (((int64_t)(buf[block_offset + 3] & 0xFF)) << 24);
+ res += (((int64_t)(buf[block_offset + 4] & 0xFF)) << 32);
+ res += (((int64_t)(buf[block_offset + 5] & 0xFF)) << 40);
+ res += (((int64_t)(buf[block_offset + 6] & 0xFF)) << 48);
+ res += (((int64_t)(buf[block_offset + 7] & 0xFF)) << 56);
+ return res;
+}
+
+} // end namespace common
\ No newline at end of file
diff --git a/cpp/src/common/container/murmur_hash3.h b/cpp/src/common/container/murmur_hash3.h
new file mode 100644
index 00000000..1e2bf44b
--- /dev/null
+++ b/cpp/src/common/container/murmur_hash3.h
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONTAINER_MURMUR3_MURMUR_HASH3_H
+#define COMMON_CONTAINER_MURMUR3_MURMUR_HASH3_H
+
+#include <string>
+
+#include "common/allocator/my_string.h"
+#include "utils/db_utils.h"
+
+namespace common {
+
+/*
+ * Murmur128Hash has to follow the logic in Java IoTDB,
+ * since the bitmap buffer will be serialized to tsfile.
+ */
+class Murmur128Hash {
+ public:
+ FORCE_INLINE static int32_t hash(int32_t value, uint32_t seed) {
+ return inner_hash(reinterpret_cast<const char *>(&value), 4, seed);
+ }
+
+ FORCE_INLINE static int32_t hash(int64_t value, uint32_t seed) {
+ return inner_hash(reinterpret_cast<const char *>(&value), 8, seed);
+ }
+
+ FORCE_INLINE static int32_t hash(double value, uint32_t seed) {
+ return inner_hash(reinterpret_cast<const char *>(&value), 8, seed);
+ }
+
+ FORCE_INLINE static int32_t hash(std::string &value, uint32_t seed) {
+ return inner_hash(value.data(), static_cast<uint32_t>(value.size()),
+ seed);
+ }
+
+ FORCE_INLINE static int32_t hash(const common::String &buf, int32_t seed) {
+ return inner_hash(buf.buf_, buf.len_ - 1, seed);
+ }
+
+ private:
+ FORCE_INLINE static int64_t rotl64(int64_t v, int n) {
+ uint64_t uv = (uint64_t)v;
+ return ((uv << n) | (uv >> (64 - n)));
+ }
+ FORCE_INLINE static int64_t fmix(int64_t k) {
+ uint64_t uk = (uint64_t)k;
+ uk ^= uk >> 33;
+ uk *= 0xff51afd7ed558ccdL;
+ uk ^= uk >> 33;
+ uk *= 0xc4ceb9fe1a85ec53L;
+ uk ^= uk >> 33;
+ return (int64_t)uk;
+ }
+ static int64_t get_block(const char *buf, int32_t index);
+ static int64_t inner_hash(const char *buf, int32_t len, int64_t seed);
+};
+
+} // end namespace common
+#endif // COMMON_CONTAINER_MURMUR3_MURMUR_HASH3_H
diff --git a/cpp/src/common/container/simple_vector.h b/cpp/src/common/container/simple_vector.h
new file mode 100644
index 00000000..977d6421
--- /dev/null
+++ b/cpp/src/common/container/simple_vector.h
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONTAINER_VECTOR_H
+#define COMMON_CONTAINER_VECTOR_H
+
+#include "utils/util_define.h"
+
+namespace common {
+
+template <typename T>
+class SimpleVector {
+ public:
+ SimpleVector() : std_vec_(), size_(0) {}
+ void push_back(T t) {
+ if (LIKELY(size_ < STACKED_ITEM_COUNT)) {
+ stacked_items_[size_] = t;
+ } else {
+ std_vec_.push_back(t);
+ }
+ size_++;
+ }
+ size_t size() { return size_; }
+ T operator[](int index) {
+ if (UNLIKELY((size_t)index > size_)) {
+ abort();
+ }
+ if (index < STACKED_ITEM_COUNT) {
+ return stacked_items_[index];
+ } else {
+ return std_vec_[index - STACKED_ITEM_COUNT];
+ }
+ }
+
+ private:
+ static const int STACKED_ITEM_COUNT = 16;
+ T stacked_items_[STACKED_ITEM_COUNT];
+ std::vector<T> std_vec_;
+ size_t size_;
+};
+
+} // end namespace common
+#endif
diff --git a/cpp/src/common/container/slice.h b/cpp/src/common/container/slice.h
new file mode 100644
index 00000000..4f678d78
--- /dev/null
+++ b/cpp/src/common/container/slice.h
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_CONTAINER_SLICE_H
+#define COMMON_CONTAINER_SLICE_H
+
+#include <cassert>
+#include <cstddef>
+#include <cstring>
+#include <string>
+
+#include "common/allocator/alloc_base.h"
+#include "common/logger/elog.h"
+#include "utils/errno_define.h"
+#include "utils/util_define.h"
+
+namespace common {
+
+class Slice {
+ public:
+ // Create an empty slice.
+ // Slice() : data_(""), size_(0) {}
+ Slice() {
+ data_ = nullptr;
+ size_ = 0;
+ }
+
+ // Create a slice that refers to the contents of "s"
+ // Slice(const std::string& s) : data_(s.data()), size_(s.size()) {}
+ Slice(const std::string &s) {
+ int len = s.size();
+ data_ = (char *)malloc(len + 1); // TODO: user ourself's mem_alloc()
+ if (UNLIKELY(nullptr == data_)) {
+ // log_err("malloc() failed.");
+ }
+ strcpy(data_, s.c_str());
+ size_ = len;
+ }
+
+ // Create a slice that refers to s[0,strlen(s)-1]
+ // Slice(const char* s) : data_(s), size_(strlen(s)) {}
+ Slice(const char *s) {
+ int len = strlen(s);
+ data_ = (char *)malloc(len + 1); // TODO: user ourself's mem_alloc()
+ if (UNLIKELY(nullptr == data_)) {
+ // log_err("malloc() failed.");
+ }
+ strcpy(data_, s);
+ size_ = len;
+ }
+
+ ~Slice() {
+ free(data_); // TODO: user ourself's mem_free()
+ data_ = nullptr;
+ size_ = 0;
+ }
+
+ // Intentionally copyable.
+ // Slice(const Slice&) = default;
+ // Slice& operator=(const Slice&) = default;
+ Slice(const Slice &other) {
+ int len = other.size_;
+ data_ = (char *)malloc(len + 1); // TODO: user ourself's mem_alloc()
+ if (UNLIKELY(nullptr == data_)) {
+ // log_err("malloc() failed.");
+ }
+ strcpy(data_, other.data_);
+ size_ = len;
+ }
+ Slice &operator=(const Slice &other) {
+ if (this->data_ != nullptr) {
+ free(data_); // TODO: user ourself's mem_free()
+ data_ = nullptr;
+ }
+ int len = other.size_;
+ this->data_ =
+ (char *)malloc(len + 1); // TODO: user ourself's mem_alloc()
+ if (UNLIKELY(nullptr == data_)) {
+ // log_err("malloc() failed.");
+ }
+ strcpy(this->data_, other.data_);
+ this->size_ = len;
+ return *this;
+ }
+
+ // Return a pointer to the beginning of the referenced data
+ char *data() const { return data_; }
+
+ // Return the length (in bytes) of the referenced data
+ size_t size() const { return size_; }
+
+ // Return true iff the length of the referenced data is zero
+ bool empty() const { return size_ == 0; }
+
+ // Return the ith byte in the referenced data.
+ char operator[](size_t n) const {
+ ASSERT(n < size());
+ return data_[n];
+ }
+
+ // Return a string that contains the copy of the referenced data.
+ std::string to_string() const { return std::string(data_, size_); }
+
+ friend bool operator==(const Slice &x, const Slice &y) {
+ return ((x.size() == y.size()) &&
+ (memcmp(x.data(), y.data(), x.size()) == 0));
+ }
+
+ friend bool operator!=(const Slice &x, const Slice &y) { return !(x == y); }
+
+ // Three-way comparison. Returns value:
+ // < 0 iff "*this" < "b",
+ // == 0 iff "*this" == "b",
+ // > 0 iff "*this" > "b"
+ int compare(const Slice &b) const {
+ const size_t min_len = (size_ < b.size_) ? size_ : b.size_;
+ int r = memcmp(data_, b.data_, min_len);
+ if (r == 0) {
+ if (size_ < b.size_)
+ r = -1;
+ else if (size_ > b.size_)
+ r = +1;
+ }
+ return r;
+ }
+
+ private:
+ char *data_;
+ size_t size_;
+};
+
+} // end namespace common
+#endif // COMMON_CONTAINER_SLICE_H
diff --git a/cpp/src/common/container/sorted_array.h b/cpp/src/common/container/sorted_array.h
new file mode 100644
index 00000000..175f9637
--- /dev/null
+++ b/cpp/src/common/container/sorted_array.h
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * no duplicated elements
+ */
+#ifndef COMMON_CONTAINER_SORTEDARRAY_H
+#define COMMON_CONTAINER_SORTEDARRAY_H
+
+#include <stdint.h>
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+
+#include "common/allocator/alloc_base.h"
+#include "common/logger/elog.h"
+#include "utils/errno_define.h"
+#include "utils/util_define.h"
+
+#define SORTED_ARRAY_INIT_CAPACITY 1000 // TODO: configurable mode
+#define SORTED_ARRAY_GROWTH_THRESHOLD 50 // TODO: configurable mode
+#define SORTED_ARRAY_SHRINK_THRESHOLD 50 // TODO: configurable mode
+
+namespace common {
+
+template <class ValueType>
+class SortedArray {
+ public:
+#ifndef NDEBUG
+ friend std::ostream &operator<<(std::ostream &out,
+ SortedArray<ValueType> &sa) {
+ for (size_t idx = 0; idx < sa.size(); idx++) {
+ if (idx > 0) {
+ out << ' ';
+ }
+ out << sa.at(idx);
+ }
+ out << std::endl;
+ return out;
+ }
+#endif // NDEBUG
+
+ SortedArray() {
+ capacity_ = SORTED_ARRAY_INIT_CAPACITY;
+ size_ = 0;
+ is_inited_ = false;
+ array_ = nullptr;
+ }
+
+ SortedArray(size_t capacity) {
+ capacity_ = capacity;
+ size_ = 0;
+ is_inited_ = false;
+ array_ = nullptr;
+ }
+
+ ~SortedArray() {
+ if (is_inited_) {
+ destroy();
+ }
+ capacity_ = 0;
+ size_ = 0;
+ }
+
+ // int deep_copy(const SortedArray &other)
+ // {
+ // this->capacity_ = other.capacity_;
+ // this->size_ = other.size_;
+ // if (init() == E_OK) {
+ // memcpy(this->array_, other.array_, this->capacity_ *
+ // sizeof(*(this->array_))); return E_OK;
+ // }
+ // return E_OOM;
+ // }
+
+ int init() {
+ if (is_inited_) {
+ // log_err("init repeated.");
+ ASSERT(!is_inited_);
+ }
+ array_ =
+ (ValueType *)mem_alloc(capacity_ * sizeof(*(array_)), MOD_ARRAY);
+ if (UNLIKELY(nullptr == array_)) {
+ is_inited_ = false;
+ // log_err("malloc failed.");
+ return E_OOM;
+ }
+ is_inited_ = true;
+ return E_OK;
+ }
+
+ void destroy() {
+ capacity_ = 0;
+ size_ = 0;
+ if (is_inited_) {
+ mem_free(array_);
+ array_ = nullptr;
+ }
+ is_inited_ = false;
+ }
+
+ private:
+ static size_t linear_calc(size_t input) {
+ if (input < SORTED_ARRAY_GROWTH_THRESHOLD) {
+ return input;
+ }
+ size_t res = 1;
+ input /= 10;
+ while (input) {
+ input /= 10;
+ res *= 10;
+ }
+ return res;
+ }
+
+ int extend() {
+ // Check for a potential overflow
+ uint64_t tmp = (uint64_t)(linear_calc(capacity_) + capacity_);
+ if (tmp > SIZE_MAX) {
+ // log_err("size overflow.");
+ return E_OVERFLOW;
+ }
+
+ size_t new_capacity = (size_t)tmp;
+ array_ =
+ (ValueType *)mem_realloc(array_, new_capacity * sizeof(*(array_)));
+ if (UNLIKELY(nullptr == array_)) {
+ // log_err("realloc failed.");
+ return E_OOM;
+ }
+ capacity_ = new_capacity;
+ return E_OK;
+ }
+
+ int shrink() {
+ size_t new_capacity = (size_t)(capacity_ / 2);
+ // if the size passed to realloc is smaller than before, OS will
+ // automatically release the rest memory
+ array_ = (ValueType *)mem_realloc(
+ array_,
+ new_capacity *
+ sizeof(*(array_))); // TODO: user ourself's mem_alloc()
+ if (UNLIKELY(nullptr == array_)) {
+ // log_err("malloc failed.");
+ return E_OOM;
+ }
+ capacity_ = new_capacity;
+ return E_OK;
+ }
+
+ public:
+ int insert(const ValueType &value) {
+ if (size_ >= capacity_) {
+ int ret = extend();
+ if (ret != E_OK) {
+ return ret;
+ }
+ }
+
+ bool found = false;
+ size_t pos = find(value, found);
+ if (found) {
+ // log_warn("the new value has already in the sorted array.");
+ return E_ALREADY_EXIST;
+ }
+
+ if (pos == size_) {
+ array_[pos] = value;
+ size_++;
+ return E_OK;
+ }
+
+ memmove(array_ + (pos + 1), array_ + pos,
+ (size_ - pos) * sizeof(*(array_)));
+ array_[pos] = value;
+ size_++;
+ return E_OK;
+ }
+
+ FORCE_INLINE ValueType at(size_t idx) {
+ if (idx >= size_) {
+ // log_err("index %lu is out of range, because size is %lu.", idx,
+ // size_);
+ ASSERT(idx < size_);
+ return dummy_;
+ }
+ return array_[idx];
+ }
+
+ FORCE_INLINE ValueType operator[](size_t idx) {
+ if (idx >= size_) {
+ // log_err("index %lu is out of range, because size is %lu.", idx,
+ // size_);
+ ASSERT(idx < size_);
+ return dummy_;
+ }
+ return array_[idx];
+ }
+
+ bool contain(const ValueType &target_val) {
+ if (size_ == 0) {
+ return false;
+ }
+ int64_t left = 0;
+ int64_t right = size_ - 1;
+ while (left <= right) {
+ size_t mid = (left + right) / 2;
+ ValueType mid_val = array_[mid];
+ if (mid_val == target_val) {
+ return true;
+ } else if (mid_val < target_val) {
+ left = mid + 1;
+ } else if (mid_val > target_val) {
+ right = mid - 1;
+ }
+ }
+ return false;
+ }
+
+ /*
+ * if the value is not found, then set 'found' is false ant return the first
+ * bigger one's location
+ */
+ size_t find(const ValueType &target_val, bool &found) {
+ if (size_ == 0) {
+ found = false;
+ return 0;
+ }
+ int64_t left = 0;
+ int64_t right = size_ - 1;
+ while (left <= right) {
+ size_t mid = (left + right) / 2;
+ ValueType mid_val = array_[mid];
+ if (mid_val == target_val) {
+ found = true;
+ return mid;
+ } else if (mid_val < target_val) {
+ left = mid + 1;
+ } else if (mid_val > target_val) {
+ right = mid - 1;
+ }
+ }
+ found = false;
+ return (size_t)left;
+ }
+
+ ValueType remove(size_t idx) {
+ if (idx >= size_) {
+ // log_err("index %lu is out of range, because size is %lu.", idx,
+ // size_);
+ ASSERT(idx < size_);
+ }
+ if (capacity_ > SORTED_ARRAY_SHRINK_THRESHOLD &&
+ size_ <= capacity_ / 4) {
+ shrink();
+ }
+ ValueType ret = array_[idx];
+ memmove(array_ + idx, array_ + idx + 1,
+ (size_ - idx - 1) * sizeof(*(array_)));
+ size_--;
+
+ return ret;
+ }
+
+ int remove_value(const ValueType &value) {
+ bool found = false;
+ size_t pos = find(value, found);
+ if (found) {
+ remove(pos);
+ } else {
+ // log_warn("the value is not exist in the array.");
+ return E_NOT_EXIST;
+ }
+ return E_OK;
+ }
+
+ FORCE_INLINE void clear() { size_ = 0; }
+
+ ValueType get_min() {
+ if (size_ == 0) {
+ // log_warn("The array is empty now.");
+ ASSERT(size_ > 0);
+ return dummy_;
+ }
+ return array_[0];
+ }
+
+ ValueType &get_max() {
+ if (size_ == 0) {
+ // log_warn("The array is empty now.");
+ ASSERT(size_ > 0);
+ return dummy_;
+ }
+ return array_[size_ - 1];
+ }
+
+ FORCE_INLINE bool empty() const { return ATOMIC_LOAD(&size_) == 0; }
+
+ FORCE_INLINE size_t size() const { return ATOMIC_LOAD(&size_); }
+
+ FORCE_INLINE size_t capacity() const { return ATOMIC_LOAD(&capacity_); }
+
+ private:
+ size_t capacity_;
+ size_t size_;
+ ValueType *array_;
+ ValueType dummy_;
+ bool is_inited_;
+};
+
+} // end namespace common
+#endif // COMMON_CONTAINER_SORTEDARRAY_H
diff --git a/cpp/src/common/datatype/CMakeLists.txt b/cpp/src/common/datatype/CMakeLists.txt
new file mode 100644
index 00000000..f7db3afc
--- /dev/null
+++ b/cpp/src/common/datatype/CMakeLists.txt
@@ -0,0 +1,18 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
diff --git a/cpp/src/common/datatype/value.h b/cpp/src/common/datatype/value.h
new file mode 100644
index 00000000..5d91d776
--- /dev/null
+++ b/cpp/src/common/datatype/value.h
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_DATATYPE_VALUE_H
+#define COMMON_DATATYPE_VALUE_H
+
+#include <float.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <utility>
+#include <vector>
+
+#include "common/logger/elog.h"
+#include "utils/db_utils.h"
+
+namespace common {
+
+struct Value {
+ Value(TSDataType type) : type_(type) {}
+
+ ~Value() {
+ if (value_.sval_) {
+ free(value_.sval_);
+ }
+ }
+
+ FORCE_INLINE void free_memory() {
+ if (value_.sval_) {
+ free(value_.sval_);
+ value_.sval_ = nullptr;
+ }
+ }
+
+ FORCE_INLINE bool is_type(TSDataType type) const { return type == type_; }
+
+ FORCE_INLINE bool is_literal() const {
+ return is_type(BOOLEAN) || is_type(DOUBLE) || is_type(TEXT) ||
+ is_type(INT64) || is_type(NULL_TYPE);
+ }
+
+ template <class T>
+ FORCE_INLINE void set_value(TSDataType type, T val) {
+ type_ = type;
+ switch (type) {
+ case common::BOOLEAN: {
+ value_.bval_ = *(bool *)val;
+ break;
+ }
+ case common::INT32: {
+ value_.ival_ = *(int32_t *)val;
+ break;
+ }
+ case common::INT64: {
+ value_.lval_ = *(int64_t *)val;
+ break;
+ }
+ case common::FLOAT: {
+ value_.fval_ = *(float *)val;
+ break;
+ }
+ case common::DOUBLE: {
+ value_.dval_ = *(double *)val;
+ break;
+ }
+ case common::TEXT: {
+ value_.sval_ = strdup(val);
+ break;
+ }
+ default: {
+ LOGE("unknown data type");
+ }
+ }
+ }
+
+ public:
+ TSDataType type_;
+
+ union {
+ bool bval_;
+ int64_t lval_;
+ int32_t ival_;
+ float fval_;
+ double dval_;
+ char *sval_;
+ } value_;
+};
+
+FORCE_INLINE Value *make(TSDataType type) {
+ Value *value = new Value(type);
+ return value;
+}
+
+FORCE_INLINE Value *make_literal(int64_t val) {
+ Value *value = new Value(INT64);
+ value->value_.lval_ = val;
+ return value;
+}
+
+FORCE_INLINE Value *make_literal(double val) {
+ Value *value = new Value(DOUBLE);
+ value->value_.dval_ = val;
+ return value;
+}
+
+FORCE_INLINE Value *make_literal(char *string) {
+ Value *value = new Value(TEXT);
+ value->value_.sval_ = string;
+ return value;
+}
+
+FORCE_INLINE Value *make_literal(bool val) {
+ Value *value = new Value(BOOLEAN);
+ value->value_.bval_ = val;
+ return value;
+}
+
+FORCE_INLINE Value *make_null_literal() {
+ Value *value = new Value(NULL_TYPE);
+ return value;
+}
+
+FORCE_INLINE std::string value_to_string(Value *value) {
+ if (value->type_ == common::TEXT) {
+ return std::string(value->value_.sval_);
+ } else {
+ std::stringstream ss;
+ switch (value->type_) {
+ case common::BOOLEAN:
+ ss << (value->value_.bval_ ? "true" : "false");
+ break;
+ case common::INT32:
+ case common::INT64:
+ ss << value->value_.lval_;
+ break;
+ case common::FLOAT:
+ case common::DOUBLE:
+ ss << value->value_.dval_;
+ break;
+ case common::NULL_TYPE:
+ ss << "NULL";
+ default:
+ ASSERT(false);
+ break;
+ }
+ return ss.str();
+ }
+}
+
+// return true if cast succ.
+template <typename T>
+FORCE_INLINE int get_typed_data_from_value(Value *value, T &ret_data) {
+ return E_OK;
+}
+
+template <>
+FORCE_INLINE int get_typed_data_from_value<bool>(Value *value, bool &ret_data) {
+ ret_data = value->value_.bval_;
+ return (value->type_ == BOOLEAN) ? E_OK : E_TYPE_NOT_MATCH;
+}
+
+template <>
+FORCE_INLINE int get_typed_data_from_value<int32_t>(Value *value,
+ int32_t &ret_data) {
+ if (value->value_.lval_ > INT32_MAX || value->value_.lval_ < INT32_MIN) {
+ return E_OVERFLOW;
+ }
+ ret_data = (int32_t)value->value_.lval_;
+ // number value will be parsed as INT64
+ return (value->type_ == INT64) ? E_OK : E_TYPE_NOT_MATCH;
+}
+
+template <>
+FORCE_INLINE int get_typed_data_from_value<int64_t>(Value *value,
+ int64_t &ret_data) {
+ ret_data = value->value_.lval_;
+ return (value->type_ == INT64) ? E_OK : E_TYPE_NOT_MATCH;
+}
+
+template <>
+FORCE_INLINE int get_typed_data_from_value<float>(Value *value,
+ float &ret_data) {
+ if (value->value_.dval_ > FLT_MAX || value->value_.dval_ < FLT_MIN) {
+ return E_OVERFLOW;
+ }
+ ret_data = (float)value->value_.dval_;
+ return (value->type_ == DOUBLE) ? E_OK : E_TYPE_NOT_MATCH;
+}
+
+template <>
+FORCE_INLINE int get_typed_data_from_value<double>(Value *value,
+ double &ret_data) {
+ ret_data = value->value_.dval_;
+ return (value->type_ == DOUBLE) ? E_OK : E_TYPE_NOT_MATCH;
+}
+
+// template<> FORCE_INLINE int get_typed_data_from_value<Text>(Value *value,
+// Text &ret_data)
+// {
+// ret_data.buf_ = value->value_.sval_;
+// ret_data.len_ = strlen(ret_data.buf_);
+// return value->type_ == TEXT;
+// }
+
+} // namespace common
+
+#endif // COMMON_DATATYPE_VALUE_H
diff --git a/cpp/src/common/db_common.h b/cpp/src/common/db_common.h
new file mode 100644
index 00000000..de6c7e71
--- /dev/null
+++ b/cpp/src/common/db_common.h
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_DB_COMMON_H
+#define COMMON_DB_COMMON_H
+
+#include <iostream>
+
+#include "utils/util_define.h"
+
+namespace common {
+
+enum TSDataType {
+ BOOLEAN = 0,
+ INT32 = 1,
+ INT64 = 2,
+ FLOAT = 3,
+ DOUBLE = 4,
+ TEXT = 5,
+ VECTOR = 6,
+ NULL_TYPE = 254,
+ INVALID_DATATYPE = 255
+};
+
+enum TSEncoding {
+ PLAIN = 0,
+ DICTIONARY = 1,
+ RLE = 2,
+ DIFF = 3,
+ TS_2DIFF = 4,
+ BITMAP = 5,
+ GORILLA_V1 = 6,
+ REGULAR = 7,
+ GORILLA = 8,
+ ZIGZAG = 9,
+ FREQ = 10,
+ INVALID_ENCODING = 255
+};
+
+enum CompressionType {
+ UNCOMPRESSED = 0,
+ SNAPPY = 1,
+ GZIP = 2,
+ LZO = 3,
+ SDT = 4,
+ PAA = 5,
+ PLA = 6,
+ LZ4 = 7,
+ INVALID_COMPRESSION = 255
+};
+
+extern const char* s_data_type_names[7];
+extern const char* s_encoding_names[12];
+extern const char* s_compression_names[8];
+
+FORCE_INLINE const char* get_data_type_name(TSDataType type) {
+ ASSERT(type >= BOOLEAN && type <= VECTOR);
+ return s_data_type_names[type];
+}
+
+FORCE_INLINE const char* get_encoding_name(TSEncoding encoding) {
+ ASSERT(encoding >= PLAIN && encoding <= FREQ);
+ return s_encoding_names[encoding];
+}
+
+FORCE_INLINE const char* get_compression_name(CompressionType type) {
+ return s_compression_names[type];
+}
+
+FORCE_INLINE TSEncoding get_default_encoding_for_type(TSDataType type) {
+ if (type == common::BOOLEAN) {
+ return PLAIN;
+ } else if (type == common::INT32) {
+ return PLAIN;
+ } else if (type == common::INT64) {
+ return PLAIN;
+ } else if (type == common::FLOAT) {
+ return PLAIN;
+ } else if (type == common::DOUBLE) {
+ return PLAIN;
+ } else if (type == common::TEXT) {
+ return PLAIN;
+ } else {
+ ASSERT(false);
+ }
+ return INVALID_ENCODING;
+}
+
+FORCE_INLINE CompressionType get_default_compression_for_type(TSDataType type) {
+ if (type == common::BOOLEAN) {
+ return UNCOMPRESSED;
+ } else if (type == common::INT32) {
+ return UNCOMPRESSED;
+ } else if (type == common::INT64) {
+ return UNCOMPRESSED;
+ } else if (type == common::FLOAT) {
+ return UNCOMPRESSED;
+ } else if (type == common::DOUBLE) {
+ return UNCOMPRESSED;
+ } else if (type == common::TEXT) {
+ return UNCOMPRESSED;
+ } else {
+ ASSERT(false);
+ }
+ return INVALID_COMPRESSION;
+}
+
+enum Ordering { DESC, ASC };
+
+template <typename T>
+FORCE_INLINE common::TSDataType GetDataTypeFromTemplateType() {
+ return common::INVALID_DATATYPE;
+}
+
+template <>
+FORCE_INLINE common::TSDataType GetDataTypeFromTemplateType<bool>() {
+ return common::BOOLEAN;
+}
+template <>
+FORCE_INLINE common::TSDataType GetDataTypeFromTemplateType<int32_t>() {
+ return common::INT32;
+}
+template <>
+FORCE_INLINE common::TSDataType GetDataTypeFromTemplateType<int64_t>() {
+ return common::INT64;
+}
+template <>
+FORCE_INLINE common::TSDataType GetDataTypeFromTemplateType<float>() {
+ return common::FLOAT;
+}
+template <>
+FORCE_INLINE common::TSDataType GetDataTypeFromTemplateType<double>() {
+ return common::DOUBLE;
+}
+
+FORCE_INLINE size_t get_data_type_size(TSDataType data_type) {
+ switch (data_type) {
+ case common::BOOLEAN:
+ return 1;
+ case common::INT32:
+ case common::FLOAT:
+ return 4;
+ case common::INT64:
+ case common::DOUBLE:
+ return 8;
+ default:
+ ASSERT(false);
+ return 8;
+ }
+}
+
+} // end namespace common
+
+namespace storage {
+
+struct TimeRange {
+ TimeRange() : start_time_(INT64_MAX), end_time_(INT64_MIN) {}
+ TimeRange(int64_t start_time, int64_t end_time)
+ : start_time_(start_time), end_time_(end_time) {}
+
+ int64_t start_time_;
+ int64_t end_time_;
+};
+
+} // namespace storage
+
+#endif
diff --git a/cpp/src/common/global.cc b/cpp/src/common/global.cc
new file mode 100644
index 00000000..fc63677c
--- /dev/null
+++ b/cpp/src/common/global.cc
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "global.h"
+
+#include <execinfo.h>
+#include <stdlib.h>
+
+#include "utils/injection.h"
+
+namespace common {
+
+ColumnDesc g_time_column_desc;
+ConfigValue g_config_value_;
+
+// TODO move to server.cc ?
+void init_config_value() {
+ g_config_value_.tsblock_mem_inc_step_size_ = 8000; // 8k
+ g_config_value_.tsblock_max_memory_ = 64000; // 64k
+ // g_config_value_.tsblock_max_memory_ = 32;
+ g_config_value_.rest_service_ip_ = "0.0.0.0";
+
+ char* timechodb_port = getenv("TIMECHODB_PORT");
+ if (nullptr == timechodb_port) {
+ g_config_value_.rest_service_port_ = 8899;
+ } else {
+ g_config_value_.rest_service_port_ = atoi(timechodb_port);
+ }
+
+ g_config_value_.wal_flush_policy_ = WAL_DISABLED;
+ g_config_value_.seqtvlist_primary_array_size_ = 32; // 32;
+ g_config_value_.seqtvlist_max_record_count_ = 1024; // 64;
+ g_config_value_.page_writer_max_point_num_ = 5;
+ g_config_value_.page_writer_max_memory_bytes_ = 128 * 1024; // 128 k
+ g_config_value_.max_degree_of_index_node_ = 256;
+ g_config_value_.tsfile_index_bloom_filter_error_percent_ = 0.05;
+ // g_config_value_.tsfile_prefix_path_ = "./data";
+ g_config_value_.tsfile_prefix_path_ = "";
+ // g_config_value_.time_encoding_type_ = TS_2DIFF;
+ g_config_value_.time_encoding_type_ = PLAIN;
+ g_config_value_.memtable_flusher_poll_interval_seconds_ = 1;
+}
+
+void config_set_page_max_point_count(uint32_t page_max_ponint_count) {
+ g_config_value_.page_writer_max_point_num_ = page_max_ponint_count;
+}
+
+void config_set_max_degree_of_index_node(uint32_t max_degree_of_index_node) {
+ g_config_value_.max_degree_of_index_node_ = max_degree_of_index_node;
+}
+
+void set_config_value() {}
+
+const char* s_data_type_names[7] = {"BOOLEAN", "INT32", "INT64", "FLOAT",
+ "DOUBLE", "TEXT", "VECTOR"};
+
+const char* s_encoding_names[12] = {
+ "PLAIN", "DICTIONARY", "RLE", "DIFF", "TS_2DIFF", "BITMAP",
+ "GORILLA_V1", "REGULAR", "GORILLA", "ZIGZAG", "FREQ"};
+
+const char* s_compression_names[8] = {
+ "UNCOMPRESSED", "SNAPPY", "GZIP", "LZO", "SDT", "PAA", "PLA", "LZ4",
+};
+
+int init_common() {
+ int ret = E_OK;
+ common::init_config_value();
+ g_time_column_desc.type_ = INT64;
+ g_time_column_desc.encoding_ = PLAIN;
+ g_time_column_desc.compression_ = UNCOMPRESSED;
+ g_time_column_desc.ttl_ = INT64_MAX; // TODO
+ g_time_column_desc.column_name_ = std::string("time");
+ g_time_column_desc.ts_id_ = TsID(0, 0, 0);
+ return ret;
+}
+
+bool is_timestamp_column_name(const char* time_col_name) {
+ // both "time" and "timestamp" refer to timestmap column.
+ int32_t len = strlen(time_col_name);
+ if (len == 4) {
+ return strncasecmp(time_col_name, "time", 4) == 0;
+ } else if (len == 9) {
+ return strncasecmp(time_col_name, "timestamp", 9) == 0;
+ } else {
+ return false;
+ }
+}
+
+void cols_to_json(ByteStream* byte_stream,
+ std::vector<common::ColumnDesc>& ret_ts_list) {
+ // 1. append start tag
+ byte_stream->write_buf("{\n", 2);
+
+ size_t ts_count = ret_ts_list.size();
+ for (size_t i = 0; i < ts_count; ++i) {
+ // 2. append timeseries name
+ std::string name = ret_ts_list[i].column_name_;
+ byte_stream->write_buf(" \"", 3);
+ byte_stream->write_buf(name.c_str(), name.length());
+ byte_stream->write_buf("\" : {\n", 6);
+
+ // 3. append DataType
+ const char* data_type = get_data_type_name(ret_ts_list[i].type_);
+ byte_stream->write_buf(" \"DataType\" : \"", 18);
+ byte_stream->write_buf(data_type, strlen(data_type));
+ byte_stream->write_buf("\",\n", 3);
+
+ // 4. append Encoding
+ const char* encoding = get_encoding_name(ret_ts_list[i].encoding_);
+ byte_stream->write_buf(" \"Encoding\" : \"", 18);
+ byte_stream->write_buf(encoding, strlen(encoding));
+ byte_stream->write_buf("\",\n", 3);
+
+ // 5. append CompressionType
+ const char* compression =
+ get_compression_name(ret_ts_list[i].compression_);
+ byte_stream->write_buf(" \"Compression\" : \"", 21);
+ byte_stream->write_buf(compression, strlen(compression));
+ byte_stream->write_buf("\",\n", 3);
+
+ // 6. append TTL
+ std::string ttl_str = to_string(ret_ts_list[i].ttl_);
+ if (ret_ts_list[i].ttl_ == INVALID_TTL) {
+ byte_stream->write_buf(" \"TTL\" : \"INVALID\",\n",
+ 23); // if it is the end, delete ','
+ } else {
+ byte_stream->write_buf(" \"TTL\" : \"", 13);
+ byte_stream->write_buf(ttl_str.c_str(), ttl_str.length());
+ byte_stream->write_buf("\",\n", 3); // if it is the end, delete ','
+ }
+
+ // TODO : this is used for Debug
+ // 7. append TsId
+ std::string tsid_str = ret_ts_list[i].ts_id_.to_string();
+ byte_stream->write_buf(" \"TsID\" : \"", 14);
+ byte_stream->write_buf(tsid_str.c_str(), tsid_str.length());
+ byte_stream->write_buf("\"\n", 2);
+
+ // 8. append footer
+ if (i == ts_count - 1) {
+ byte_stream->write_buf(" }\n", 4);
+ } else {
+ byte_stream->write_buf(" },\n", 5);
+ }
+ }
+
+ // 9. end
+ byte_stream->write_buf("}\n", 2);
+
+ // DEBUG_print_byte_stream(*byte_stream); // for debug
+}
+
+void print_backtrace() {
+ const int MAX_FRAMES = 32;
+ int layers = 0;
+ char** symbols = NULL;
+ void* frames[MAX_FRAMES];
+
+ memset(frames, 0, sizeof(frames));
+ layers = backtrace(frames, MAX_FRAMES);
+ symbols = backtrace_symbols(frames, layers);
+ if (symbols) {
+ for (int i = 0; i < layers; i++) {
+ printf("SYMBOL layer %d: %s\n", i, symbols[i]);
+ }
+ free(symbols);
+ }
+}
+
+Mutex g_all_inject_points_mutex;
+std::map<std::string, InjectPoint> g_all_inject_points;
+
+} // namespace common
diff --git a/cpp/src/common/global.h b/cpp/src/common/global.h
new file mode 100644
index 00000000..bf973fc9
--- /dev/null
+++ b/cpp/src/common/global.h
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_GLOBAL_H
+#define COMMON_GLOBAL_H
+
+#include <string>
+#include <vector>
+
+#include "common/allocator/byte_stream.h"
+#include "common/config/config.h"
+
+namespace common {
+
+extern ConfigValue g_config_value_;
+
+FORCE_INLINE bool wal_cfg_enabled() {
+ return g_config_value_.wal_flush_policy_ != WAL_DISABLED;
+}
+FORCE_INLINE bool wal_cfg_should_wait_persisted() {
+ return g_config_value_.wal_flush_policy_ >= WAL_FLUSH;
+}
+
+extern ColumnDesc g_time_column_desc;
+extern int init_common();
+extern bool is_timestamp_column_name(const char *time_col_name);
+extern void cols_to_json(ByteStream *byte_stream,
+ std::vector<common::ColumnDesc> &ret_ts_list);
+extern void print_backtrace();
+
+} // namespace common
+
+#endif // COMMON_GLOBAL_H
diff --git a/cpp/examples/src/example.cpp b/cpp/src/common/logger/elog.h
similarity index 61%
rename from cpp/examples/src/example.cpp
rename to cpp/src/common/logger/elog.h
index 07deb8e7..62ccf1aa 100644
--- a/cpp/examples/src/example.cpp
+++ b/cpp/src/common/logger/elog.h
@@ -4,10 +4,10 @@
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
+ * License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * https://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@@ -16,14 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
-#include <cstdio>
+#include <iostream>
-#include "tsfile.hpp"
+#ifdef DEBUG_LOG
+#define LOG(X) std::cout << X << std::endl
+#else
+#define LOG(X) ((void)0)
+#endif
-int main()
-{
- TsFile* tsFile = new TsFile();
- int result = tsFile->doSomethingElse(10, 20, 3);
- printf("Result = %d", result);
- return 0;
-}
\ No newline at end of file
+#ifdef DEBUG_ERROR
+#define LOGE(X) std::cout << X << std::endl
+#else
+#define LOGE(X) ((void)0)
+#endif
+
+#ifdef DEBUG_WARN
+#define LOGW(X) std::cout << X << std::endl
+#else
+#define LOGW(X) ((void)0)
+#endif
diff --git a/cpp/src/common/mutex/CMakeLists.txt b/cpp/src/common/mutex/CMakeLists.txt
new file mode 100644
index 00000000..e7ef66fa
--- /dev/null
+++ b/cpp/src/common/mutex/CMakeLists.txt
@@ -0,0 +1,20 @@
+#[[
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+]]
+
+
diff --git a/cpp/src/common/mutex/mutex.h b/cpp/src/common/mutex/mutex.h
new file mode 100644
index 00000000..18bb736e
--- /dev/null
+++ b/cpp/src/common/mutex/mutex.h
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_MUTEX_MUTEX_H
+#define COMMON_MUTEX_MUTEX_H
+
+#include <errno.h>
+#include <pthread.h>
+
+#include "utils/util_define.h"
+
+namespace common {
+
+class Mutex {
+ public:
+ Mutex() : mutex_() { pthread_mutex_init(&mutex_, NULL); }
+ ~Mutex() { pthread_mutex_destroy(&mutex_); }
+
+ void lock() {
+ int ret = EBUSY;
+ do {
+ ret = pthread_mutex_lock(&mutex_);
+ } while (UNLIKELY(ret == EBUSY || ret == EAGAIN));
+ ASSERT(ret == 0);
+ }
+
+ void unlock() {
+ int ret = pthread_mutex_unlock(&mutex_);
+ ASSERT(ret == 0);
+ }
+
+ bool try_lock() {
+ int ret = pthread_mutex_trylock(&mutex_);
+ if (ret == 0) {
+ return true;
+ } else if (ret == EBUSY || ret == EAGAIN) {
+ return false;
+ } else {
+ ASSERT(false);
+ return false;
+ }
+ }
+
+ private:
+ pthread_mutex_t mutex_;
+};
+
+class MutexGuard {
+ public:
+ MutexGuard(Mutex &m) : m_(m) { m_.lock(); }
+ ~MutexGuard() { m_.unlock(); }
+
+ private:
+ Mutex &m_;
+};
+
+} // end namespace common
+#endif // COMMON_MUTEX_MUTEX_H
diff --git a/cpp/src/common/path.h b/cpp/src/common/path.h
new file mode 100644
index 00000000..7a3b55b0
--- /dev/null
+++ b/cpp/src/common/path.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_READ_COMMON_PATH_H
+#define COMMON_READ_COMMON_PATH_H
+
+#include <string>
+
+namespace storage {
+
+struct Path {
+ std::string measurement_;
+ std::string device_;
+ std::string full_path_;
+
+ Path() {}
+
+ Path(std::string &device, std::string &measurement)
+ : measurement_(measurement), device_(device) {
+ full_path_ = device + "." + measurement;
+ }
+
+ bool operator==(const Path &path) {
+ if (measurement_.compare(path.measurement_) == 0 &&
+ device_.compare(path.device_) == 0) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+};
+
+} // namespace storage
+
+#endif // COMMON_READ_COMMON_PATH_H
diff --git a/cpp/src/common/record.h b/cpp/src/common/record.h
new file mode 100644
index 00000000..f87df047
--- /dev/null
+++ b/cpp/src/common/record.h
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_RECORD_H
+#define COMMON_RECORD_H
+
+#include <string>
+#include <vector>
+
+#include "common/db_common.h"
+
+namespace storage {
+
+// TODO: use std::move
+// #define MOVE(var) std::move(var)
+
+// TODO use common/allocator/my_string.h
+struct TextType {
+ char *buf_;
+ int32_t len_;
+
+ TextType() : buf_(nullptr), len_(0) {}
+};
+
+/*
+ * DataPoint is a data value of one measurement of some device.
+ */
+struct DataPoint {
+ std::string measurement_name_;
+ common::TSDataType data_type_;
+ union {
+ bool bool_val_;
+ int32_t i32_val_;
+ int64_t i64_val_;
+ float float_val_;
+ double double_val_;
+ } u_;
+ TextType text_val_;
+
+ DataPoint(const std::string &measurement_name, bool b)
+ : measurement_name_(measurement_name),
+ data_type_(common::BOOLEAN),
+ text_val_() {
+ u_.bool_val_ = b;
+ }
+
+ DataPoint(const std::string &measurement_name, int32_t i32)
+ : measurement_name_(measurement_name),
+ data_type_(common::INT32),
+ text_val_() {
+ u_.i32_val_ = i32;
+ }
+
+ DataPoint(const std::string &measurement_name, int64_t i64)
+ : measurement_name_(measurement_name),
+ data_type_(common::INT64),
+ text_val_() {
+ u_.i64_val_ = i64;
+ }
+
+ DataPoint(const std::string &measurement_name, float f)
+ : measurement_name_(measurement_name),
+ data_type_(common::FLOAT),
+ text_val_() {
+ u_.float_val_ = f;
+ }
+
+ DataPoint(const std::string &measurement_name, double d)
+ : measurement_name_(measurement_name),
+ data_type_(common::DOUBLE),
+ text_val_() {
+ u_.double_val_ = d;
+ }
+
+ // DataPoint(const std::string &measurement_name, Text &text),
+ // : measurement_name_(measurement_name),
+ // data_type_(common::TEXT),
+ // text_val_(text) {}
+
+ DataPoint(const std::string &measurement_name)
+ : measurement_name_(measurement_name) {}
+ void set_i32(int32_t i32) {
+ data_type_ = common::INT32;
+ u_.i32_val_ = i32;
+ }
+ void set_i64(int64_t i64) {
+ data_type_ = common::INT64;
+ u_.i64_val_ = i64;
+ }
+ void set_float(float f) {
+ data_type_ = common::FLOAT;
+ u_.float_val_ = f;
+ }
+ void set_double(double d) {
+ data_type_ = common::DOUBLE;
+ u_.double_val_ = d;
+ }
+};
+
+struct TsRecord {
+ int64_t timestamp_;
+ std::string device_name_;
+ std::vector<DataPoint> points_;
+
+ TsRecord(const std::string &device_name) : device_name_(device_name) {}
+
+ TsRecord(int64_t timestamp, const std::string &device_name,
+ int32_t point_count_in_row = 0)
+ : timestamp_(timestamp), device_name_(device_name), points_() {
+ if (point_count_in_row > 0) {
+ points_.reserve(point_count_in_row);
+ }
+ }
+
+ void append_data_point(const DataPoint &point) {
+ // points_.emplace_back(point); C++11
+ points_.push_back(point);
+ }
+};
+
+} // end namespace storage
+#endif // COMMON_RECORD_H
diff --git a/cpp/src/common/row_record.h b/cpp/src/common/row_record.h
new file mode 100644
index 00000000..6901b08d
--- /dev/null
+++ b/cpp/src/common/row_record.h
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef COMMON_READ_COMMON_ROW_RECORD_H
+#define COMMON_READ_COMMON_ROW_RECORD_H
+
+#include <sstream>
+#include <vector>
+
+#include "common/db_common.h"
+
+namespace storage {
+struct Field {
+ Field() : type_(common::INVALID_DATATYPE) {}
+ Field(common::TSDataType type) : type_(type) {}
+
+ ~Field() {
+ if (type_ == common::TEXT && value_.sval_) {
+ free(value_.sval_);
+ }
+ }
+
+ FORCE_INLINE void free_memory() {
+ if (value_.sval_) {
+ free(value_.sval_);
+ value_.sval_ = nullptr;
+ }
+ }
+
+ FORCE_INLINE bool is_type(common::TSDataType type) const {
+ return type == type_;
+ }
+
+ FORCE_INLINE bool is_literal() const {
+ return is_type(common::BOOLEAN) || is_type(common::DOUBLE) ||
+ is_type(common::TEXT) || is_type(common::INT64) ||
+ is_type(common::NULL_TYPE);
+ }
+
+ template <class T>
+ FORCE_INLINE void set_value(common::TSDataType type, T val) {
+ type_ = type;
+ switch (type) {
+ case common::BOOLEAN: {
+ value_.bval_ = *(bool *)val;
+ break;
+ }
+ case common::INT32: {
+ value_.ival_ = *(int32_t *)val;
+ break;
+ }
+ case common::INT64: {
+ value_.lval_ = *(int64_t *)val;
+ break;
+ }
+ case common::FLOAT: {
+ value_.fval_ = *(float *)val;
+ break;
+ }
+ case common::DOUBLE: {
+ value_.dval_ = *(double *)val;
+ break;
+ }
+ // case common::TEXT: {
+ // value_.sval_ = strdup(val);
+ // break;
+ // }
+ default: {
+ std::cout << "unknown data type" << std::endl;
+ }
+ }
+ }
+
+ public:
+ common::TSDataType type_;
+
+ union {
+ bool bval_;
+ int64_t lval_;
+ int32_t ival_;
+ float fval_;
+ double dval_;
+ char *sval_;
+ } value_;
+};
+
+FORCE_INLINE Field *make(common::TSDataType type) {
+ Field *value = new Field(type);
+ return value;
+}
+
+FORCE_INLINE Field *make_literal(int64_t val) {
+ Field *value = new Field(common::INT64);
+ value->value_.lval_ = val;
+ return value;
+}
+
+FORCE_INLINE Field *make_literal(double val) {
+ Field *value = new Field(common::DOUBLE);
+ value->value_.dval_ = val;
+ return value;
+}
+
+FORCE_INLINE Field *make_literal(char *string) {
+ Field *value = new Field(common::TEXT);
+ value->value_.sval_ = string;
+ return value;
+}
+
+FORCE_INLINE Field *make_literal(bool val) {
+ Field *value = new Field(common::BOOLEAN);
+ value->value_.bval_ = val;
+ return value;
+}
+
+FORCE_INLINE Field *make_null_literal() {
+ Field *value = new Field(common::NULL_TYPE);
+ return value;
+}
+
+class RowRecord {
+ public:
+ explicit RowRecord(uint32_t col_num) : col_num_(col_num) {
+ fields_ = new std::vector<Field *>();
+ fields_->reserve(col_num);
+ for (uint32_t i = 0; i < col_num; ++i) {
+ Field *val = make_null_literal();
+ fields_->push_back(val);
+ }
+ }
+
+ RowRecord(int64_t time, uint32_t col_num) : time_(time), col_num_(col_num) {
+ fields_ = new std::vector<Field *>();
+ fields_->reserve(col_num_);
+ for (uint32_t i = 0; i < col_num_; ++i) {
+ Field *val = make_null_literal();
+ fields_->push_back(val);
+ }
+ }
+
+ ~RowRecord() {
+ if (fields_) {
+ int size = fields_->size();
+ for (int i = 0; i < size; ++i) {
+ delete fields_->at(i);
+ }
+ delete fields_;
+ }
+ }
+
+ FORCE_INLINE void reset() {
+ for (uint32_t i = 0; i < col_num_; ++i) {
+ if ((*fields_)[i]->type_ == common::TEXT) {
+ (*fields_)[i]->free_memory();
+ }
+ (*fields_)[i]->type_ = common::NULL_TYPE;
+ }
+ }
+
+ FORCE_INLINE void add_field(Field *field) { fields_->push_back(field); }
+
+ FORCE_INLINE void set_timestamp(int64_t time) { time_ = time; }
+
+ FORCE_INLINE int64_t get_timestamp() { return time_; }
+
+ FORCE_INLINE Field *get_field(uint32_t index) { return (*fields_)[index]; }
+
+ FORCE_INLINE std::vector<Field *> *get_fields() { return fields_; }
+
+ private:
+ int64_t time_; // time value
+ uint32_t col_num_; // measurement num
+ std::vector<Field *> *fields_; // measurement value
+};
+
+} // namespace storage
+
+#endif // COMMON_READ_COMMON_ROW_RECORD_H
diff --git a/cpp/src/common/schema.h b/cpp/src/common/schema.h
new file mode 100644
index 00000000..ff733b12
--- /dev/null
+++ b/cpp/src/common/schema.h
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_SCHEMA_H
+#define COMMON_SCHEMA_H
+
+#include <map> // use unordered_map instead
+#include <string>
+
+#include "common/db_common.h"
+
+namespace storage {
+class ChunkWriter;
+}
+
+namespace storage {
+
+/* schema information for one measurement */
+struct MeasurementSchema {
+ std::string measurement_name_; // for example: "s1"
+ common::TSDataType data_type_;
+ common::TSEncoding encoding_;
+ common::CompressionType compression_type_;
+ storage::ChunkWriter *chunk_writer_;
+
+ MeasurementSchema()
+ : measurement_name_(),
+ data_type_(common::INVALID_DATATYPE),
+ encoding_(common::INVALID_ENCODING),
+ compression_type_(common::INVALID_COMPRESSION),
+ chunk_writer_(NULL) {}
+
+ MeasurementSchema(const std::string &measurement_name,
+ common::TSDataType data_type, common::TSEncoding encoding,
+ common::CompressionType compression_type)
+ : measurement_name_(measurement_name),
+ data_type_(data_type),
+ encoding_(encoding),
+ compression_type_(compression_type),
+ chunk_writer_(NULL) {}
+};
+
+typedef std::map<std::string, MeasurementSchema *> MeasurementSchemaMap;
+typedef std::map<std::string, MeasurementSchema *>::iterator
+ MeasurementSchemaMapIter;
+typedef std::pair<MeasurementSchemaMapIter, bool>
+ MeasurementSchemaMapInsertResult;
+
+/* schema information for a device */
+struct MeasurementSchemaGroup {
+ // measurement_name -> MeasurementSchema
+ MeasurementSchemaMap measurement_schema_map_;
+ bool is_aligned_; // currently not used.
+};
+
+} // end namespace storage
+#endif // COMMON_SCHEMA_H
diff --git a/cpp/src/common/seq_tvlist.h b/cpp/src/common/seq_tvlist.h
new file mode 100644
index 00000000..57f63760
--- /dev/null
+++ b/cpp/src/common/seq_tvlist.h
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_SEQ_TVLIST_H
+#define COMMON_SEQ_TVLIST_H
+
+#include "common/allocator/alloc_base.h"
+#include "common/allocator/page_arena.h"
+#include "common/mutex/mutex.h"
+#include "utils/db_utils.h"
+#include "utils/errno_define.h"
+#include "utils/storage_utils.h"
+#include "utils/util_define.h"
+
+namespace storage {
+
+class SeqTVListBase {
+ public:
+ SeqTVListBase()
+ : data_type_(common::VECTOR),
+ mutex_(),
+ ref_count_(0),
+ primary_array_size_(0),
+ list_size_(0),
+ write_count_(0),
+ page_arena_(common::g_base_allocator),
+ use_page_arena_(false),
+ is_immutable_(false) {}
+ virtual ~SeqTVListBase() {}
+ virtual void destroy() {}
+
+ FORCE_INLINE void ref() { ATOMIC_AAF(&ref_count_, 1); }
+ FORCE_INLINE bool unref() { return 0 == ATOMIC_AAF(&ref_count_, -1); }
+
+ FORCE_INLINE void lock() { mutex_.lock(); }
+ FORCE_INLINE void unlock() { mutex_.unlock(); }
+
+ int32_t get_total_count() const { return write_count_; }
+ common::TSDataType get_data_type() const { return data_type_; }
+ virtual TimeRange get_time_range() const = 0;
+ void mark_immutable() { is_immutable_ = true; }
+ bool is_immutable() const { return is_immutable_; }
+
+ protected:
+ common::TSDataType data_type_;
+ mutable common::Mutex mutex_;
+ int32_t ref_count_;
+ int32_t primary_array_size_;
+ int32_t list_size_;
+ int32_t write_count_;
+ common::PageArena page_arena_;
+ bool use_page_arena_;
+ bool is_immutable_;
+};
+
+template <typename Type>
+class SeqTVList : public SeqTVListBase {
+ public:
+ typedef struct TV {
+ int64_t time_;
+ Type value_;
+ } TV;
+
+ struct Iterator {
+ SeqTVList *host_list_;
+ int32_t read_idx_;
+ int32_t end_idx_;
+
+ Iterator() : host_list_(nullptr), read_idx_(UINT32_MAX), end_idx_(0) {}
+
+ INLINE void init(SeqTVList *host, int32_t start_idx, int32_t end_idx) {
+ host_list_ = host;
+ read_idx_ = start_idx;
+ end_idx_ = end_idx;
+ }
+
+ int next(TV &tv) {
+ if (read_idx_ >= end_idx_) {
+ return common::E_NO_MORE_DATA;
+ }
+ tv = host_list_->at(read_idx_);
+ read_idx_++;
+ return common::E_OK;
+ }
+ };
+
+ public:
+ SeqTVList() : tv_array_list_(nullptr), last_time_(-1) {
+ data_type_ = common::GetDataTypeFromTemplateType<Type>();
+ }
+ virtual ~SeqTVList() {}
+
+ int init(int32_t primary_array_size, int32_t max_count,
+ bool use_page_arena);
+ void destroy() OVERRIDE;
+
+ int push(int64_t time, Type value);
+ int push_without_lock(int64_t time, Type value);
+ Iterator scan_without_lock(int64_t start_time, int64_t end_time);
+ Iterator scan_without_lock();
+
+ TimeRange get_time_range() const OVERRIDE {
+ TimeRange time_range;
+ common::MutexGuard mg(mutex_);
+ if (write_count_ > 0) {
+ time_range.start_time_ = time_at(0);
+ time_range.end_time_ = time_at(write_count_ - 1);
+ ASSERT(time_range.start_time_ <= time_range.end_time_);
+ }
+ return time_range;
+ }
+
+ FORCE_INLINE TV at(int32_t tv_idx) const {
+ ASSERT(tv_idx < write_count_);
+ int32_t list_idx = tv_idx / primary_array_size_;
+ int32_t list_offset = tv_idx % primary_array_size_;
+ return tv_array_list_[list_idx][list_offset];
+ }
+
+ FORCE_INLINE int64_t time_at(int32_t tv_idx) const {
+ return at(tv_idx).time_;
+ }
+
+#ifdef ENABLE_TEST
+ int32_t TEST_binary_search_upper(int64_t time) {
+ return binary_search_upper(time);
+ }
+ int32_t TEST_binary_search_lower(int64_t time) {
+ return binary_search_lower(time);
+ }
+#endif
+
+ private:
+ FORCE_INLINE void *alloc(uint32_t size) {
+ if (use_page_arena_) {
+ return page_arena_.alloc(size);
+ } else {
+ return common::mem_alloc(size, common::MOD_TVLIST_DATA);
+ }
+ }
+
+ // return the first tv which is larger or equal to @time
+ int32_t binary_search_upper(int64_t time);
+ // return the last tv which is less or equal to @time
+ int32_t binary_search_lower(int64_t time);
+
+ private:
+ TV **tv_array_list_;
+ int64_t last_time_;
+};
+
+} // namespace storage
+
+#include "seq_tvlist.inc"
+
+#endif // COMMON_SEQ_TVLIST_H
diff --git a/cpp/tsfile/src/tsfile/seq_tvlist.inc b/cpp/src/common/seq_tvlist.inc
similarity index 81%
rename from cpp/tsfile/src/tsfile/seq_tvlist.inc
rename to cpp/src/common/seq_tvlist.inc
index 272d7a60..0e723ea3 100644
--- a/cpp/tsfile/src/tsfile/seq_tvlist.inc
+++ b/cpp/src/common/seq_tvlist.inc
@@ -1,4 +1,21 @@
-
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
// #include "seq_tvlist.h"
#include <stdio.h>
#include <string.h>
@@ -6,8 +23,7 @@
#include "common/mutex/mutex.h"
#include "common/logger/elog.h"
-namespace timecho
-{
+
namespace storage
{
@@ -17,7 +33,7 @@ int SeqTVList<Type>::init(int32_t primary_array_size,
bool use_page_arena)
{
if (primary_array_size > max_count) {
- common::log_error("TVList init error, primary_array_size=%u, max_count=%u", primary_array_size, max_count);
+ //common:://log_err("TVList init error, primary_array_size=%u, max_count=%u", primary_array_size, max_count);
return common::E_INVALID_ARG;
}
use_page_arena_ = use_page_arena;
@@ -155,5 +171,4 @@ int32_t SeqTVList<Type>::binary_search_upper(int64_t time)
}
} // namepsace storage
-} // namespace timecho
diff --git a/cpp/src/common/statistic.h b/cpp/src/common/statistic.h
new file mode 100644
index 00000000..1a2a91d6
--- /dev/null
+++ b/cpp/src/common/statistic.h
@@ -0,0 +1,839 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_STATISTIC_H
+#define COMMON_STATISTIC_H
+
+#include <inttypes.h>
+
+#include "common/allocator/alloc_base.h"
+#include "common/allocator/byte_stream.h"
+#include "common/db_common.h"
+
+namespace storage {
+
+/*
+ * Since we have to handle different type of statistic,
+ * here we use macro.
+ * TODO: do we have to update end_time_/last_value_ every time?
+ */
+#define TIME_STAT_UPDATE(time) \
+ do { \
+ if (UNLIKELY(count_ == 0)) { \
+ start_time_ = time; \
+ end_time_ = time; \
+ } else { \
+ if (UNLIKELY((time) < start_time_)) { \
+ start_time_ = (time); \
+ } \
+ if (LIKELY(time > end_time_)) { \
+ end_time_ = (time); \
+ } \
+ } \
+ } while (false)
+
+#define NUM_VALUE_STAT_UPDATE(value) \
+ do { \
+ if (UNLIKELY(count_ == 0)) { \
+ sum_value_ = (value); \
+ min_value_ = (value); \
+ max_value_ = (value); \
+ first_value_ = (value); \
+ last_value_ = (value); \
+ } else { \
+ if ((value) < min_value_) { \
+ min_value_ = (value); \
+ } \
+ if ((value) > max_value_) { \
+ max_value_ = (value); \
+ } \
+ sum_value_ += (value); \
+ last_value_ = (value); \
+ } \
+ } while (false)
+
+#define BOOL_VALUE_STAT_UPDATE(value) \
+ do { \
+ if (UNLIKELY(count_ == 0)) { \
+ sum_value_ = (value); \
+ first_value_ = (value); \
+ last_value_ = (value); \
+ } else { \
+ sum_value_ += (value); \
+ last_value_ = (value); \
+ } \
+ } while (false)
+
+#define NUM_STAT_UPDATE(time, value) \
+ do { \
+ /* update time */ \
+ TIME_STAT_UPDATE((time)); \
+ /* update value */ \
+ NUM_VALUE_STAT_UPDATE((value)); \
+ count_++; \
+ } while (false)
+
+#define BOOL_STAT_UPDATE(time, value) \
+ do { \
+ /* update time */ \
+ TIME_STAT_UPDATE((time)); \
+ /* update value */ \
+ BOOL_VALUE_STAT_UPDATE((value)); \
+ count_++; \
+ } while (false)
+
+// Base Statistic
+class Statistic {
+ public:
+ Statistic() : count_(0), start_time_(0), end_time_(0) {}
+ virtual void destroy() {}
+ FORCE_INLINE void reset() { count_ = 0; }
+
+ virtual FORCE_INLINE void update(int64_t time, bool value) {
+ ASSERT(false);
+ }
+ virtual FORCE_INLINE void update(int64_t time, int32_t value) {
+ ASSERT(false);
+ }
+ virtual FORCE_INLINE void update(int64_t time, int64_t value) {
+ ASSERT(false);
+ }
+ virtual FORCE_INLINE void update(int64_t time, float value) {
+ ASSERT(false);
+ }
+ virtual FORCE_INLINE void update(int64_t time, double value) {
+ ASSERT(false);
+ }
+
+ virtual int serialize_to(common::ByteStream &out) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::write_var_uint(count_, out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_ui64(start_time_,
+ out))) {
+ } else if (RET_FAIL(
+ common::SerializationUtil::write_ui64(end_time_, out))) {
+ } else if (RET_FAIL(serialize_typed_stat(out))) {
+ }
+ return ret;
+ }
+ virtual int serialize_typed_stat(common::ByteStream &out) {
+ ASSERT(false);
+ return 0;
+ }
+ virtual int deserialize_from(common::ByteStream &in) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::read_var_uint(
+ (uint32_t &)count_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui64(
+ (uint64_t &)start_time_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui64(
+ (uint64_t &)end_time_, in))) {
+ } else if (RET_FAIL(deserialize_typed_stat(in))) {
+ }
+ return ret;
+ }
+ virtual int deserialize_typed_stat(common::ByteStream &in) {
+ ASSERT(false);
+ return 0;
+ }
+ virtual int merge_with(Statistic *that) {
+ ASSERT(false);
+ return 0;
+ }
+ virtual int deep_copy_from(Statistic *stat) {
+ ASSERT(false);
+ return 0;
+ }
+ virtual common::TSDataType get_type() {
+ ASSERT(false);
+ return common::INVALID_DATATYPE;
+ }
+ virtual std::string to_string() const {
+ return std::string("UNTYPED_STATISTIC");
+ }
+
+ public:
+ int32_t count_;
+ int64_t start_time_;
+ int64_t end_time_;
+};
+
+#define MERGE_BOOL_STAT_FROM(StatType, untyped_stat) \
+ do { \
+ if (UNLIKELY(untyped_stat == nullptr)) { \
+ return common::E_INVALID_ARG; \
+ } \
+ StatType *typed_stat = (StatType *)(untyped_stat); \
+ if (UNLIKELY(typed_stat == nullptr)) { \
+ return common::E_TYPE_NOT_MATCH; \
+ } \
+ if (UNLIKELY(typed_stat->count_ == 0)) { \
+ return common::E_OK; \
+ } \
+ if (count_ == 0) { \
+ count_ = typed_stat->count_; \
+ start_time_ = typed_stat->start_time_; \
+ end_time_ = typed_stat->end_time_; \
+ sum_value_ = typed_stat->sum_value_; \
+ first_value_ = typed_stat->first_value_; \
+ last_value_ = typed_stat->last_value_; \
+ } else { \
+ count_ += typed_stat->count_; \
+ if (typed_stat->start_time_ < start_time_) { \
+ start_time_ = typed_stat->start_time_; \
+ first_value_ = typed_stat->first_value_; \
+ } \
+ if (typed_stat->end_time_ > end_time_) { \
+ end_time_ = typed_stat->end_time_; \
+ last_value_ = typed_stat->last_value_; \
+ } \
+ sum_value_ += typed_stat->sum_value_; \
+ } \
+ return common::E_OK; \
+ } while (false)
+
+#define MERGE_NUM_STAT_FROM(StatType, untyped_stat) \
+ do { \
+ if (UNLIKELY(untyped_stat == nullptr)) { \
+ return common::E_INVALID_ARG; \
+ } \
+ StatType *typed_stat = (StatType *)(untyped_stat); \
+ if (UNLIKELY(typed_stat == nullptr)) { \
+ return common::E_TYPE_NOT_MATCH; \
+ } \
+ if (UNLIKELY(typed_stat->count_ == 0)) { \
+ return common::E_OK; \
+ } \
+ if (count_ == 0) { \
+ count_ = typed_stat->count_; \
+ start_time_ = typed_stat->start_time_; \
+ end_time_ = typed_stat->end_time_; \
+ sum_value_ = typed_stat->sum_value_; \
+ first_value_ = typed_stat->first_value_; \
+ last_value_ = typed_stat->last_value_; \
+ min_value_ = typed_stat->min_value_; \
+ max_value_ = typed_stat->max_value_; \
+ } else { \
+ count_ += typed_stat->count_; \
+ if (typed_stat->start_time_ < start_time_) { \
+ start_time_ = typed_stat->start_time_; \
+ first_value_ = typed_stat->first_value_; \
+ } \
+ if (typed_stat->end_time_ > end_time_) { \
+ end_time_ = typed_stat->end_time_; \
+ last_value_ = typed_stat->last_value_; \
+ } \
+ sum_value_ += typed_stat->sum_value_; \
+ min_value_ = std::min(min_value_, typed_stat->min_value_); \
+ max_value_ = std::max(max_value_, typed_stat->max_value_); \
+ } \
+ return common::E_OK; \
+ } while (false)
+
+#define DEEP_COPY_BOOL_STAT_FROM(StatType, untyped_stat) \
+ do { \
+ if (UNLIKELY(untyped_stat == nullptr)) { \
+ return common::E_INVALID_ARG; \
+ } \
+ StatType *typed_stat = (StatType *)(untyped_stat); \
+ if (UNLIKELY(typed_stat == nullptr)) { \
+ return common::E_TYPE_NOT_MATCH; \
+ } \
+ count_ = typed_stat->count_; \
+ start_time_ = typed_stat->start_time_; \
+ end_time_ = typed_stat->end_time_; \
+ sum_value_ = typed_stat->sum_value_; \
+ first_value_ = typed_stat->first_value_; \
+ last_value_ = typed_stat->last_value_; \
+ return common::E_OK; \
+ } while (false)
+
+#define DEEP_COPY_NUM_STAT_FROM(StatType, untyped_stat) \
+ do { \
+ if (UNLIKELY(untyped_stat == nullptr)) { \
+ return common::E_INVALID_ARG; \
+ } \
+ StatType *typed_stat = (StatType *)(untyped_stat); \
+ if (UNLIKELY(typed_stat == nullptr)) { \
+ return common::E_TYPE_NOT_MATCH; \
+ } \
+ count_ = typed_stat->count_; \
+ start_time_ = typed_stat->start_time_; \
+ end_time_ = typed_stat->end_time_; \
+ sum_value_ = typed_stat->sum_value_; \
+ first_value_ = typed_stat->first_value_; \
+ last_value_ = typed_stat->last_value_; \
+ min_value_ = typed_stat->min_value_; \
+ max_value_ = typed_stat->max_value_; \
+ return common::E_OK; \
+ } while (false)
+
+/* ================ Typed Statistics ================*/
+class BooleanStatistic : public Statistic {
+ public:
+ int64_t sum_value_;
+ bool first_value_;
+ bool last_value_;
+
+ BooleanStatistic()
+ : sum_value_(0), first_value_(false), last_value_(false) {}
+
+ void clone_from(const BooleanStatistic &that) {
+ count_ = that.count_;
+ start_time_ = that.start_time_;
+ end_time_ = that.end_time_;
+
+ sum_value_ = that.sum_value_;
+ first_value_ = that.first_value_;
+ last_value_ = that.last_value_;
+ }
+
+ FORCE_INLINE void update(int64_t time, bool value) {
+ BOOL_STAT_UPDATE(time, value);
+ }
+ int serialize_typed_stat(common::ByteStream &out) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::write_ui8(first_value_ ? 1 : 0,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_ui8(
+ last_value_ ? 1 : 0, out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_ui64(sum_value_,
+ out))) {
+ }
+ return ret;
+ }
+ int deserialize_typed_stat(common::ByteStream &in) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::read_ui8(
+ (uint8_t &)first_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui8(
+ (uint8_t &)last_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui64(
+ (uint64_t &)sum_value_, in))) {
+ }
+ return ret;
+ }
+
+ FORCE_INLINE common::TSDataType get_type() { return common::BOOLEAN; }
+
+ int merge_with(Statistic *stat) {
+ MERGE_BOOL_STAT_FROM(BooleanStatistic, stat);
+ }
+
+ int deep_copy_from(Statistic *stat) {
+ DEEP_COPY_BOOL_STAT_FROM(BooleanStatistic, stat);
+ }
+};
+
+class Int32Statistic : public Statistic {
+ public:
+ int64_t sum_value_;
+ int32_t min_value_;
+ int32_t max_value_;
+ int32_t first_value_;
+ int32_t last_value_;
+
+ Int32Statistic()
+ : sum_value_(0),
+ min_value_(0),
+ max_value_(0),
+ first_value_(0),
+ last_value_(0) {}
+
+ void clone_from(const Int32Statistic &that) {
+ count_ = that.count_;
+ start_time_ = that.start_time_;
+ end_time_ = that.end_time_;
+
+ sum_value_ = that.sum_value_;
+ min_value_ = that.min_value_;
+ max_value_ = that.max_value_;
+ first_value_ = that.first_value_;
+ last_value_ = that.last_value_;
+ }
+
+ FORCE_INLINE void update(int64_t time, int32_t value) {
+ NUM_STAT_UPDATE(time, value);
+ }
+
+ FORCE_INLINE common::TSDataType get_type() { return common::INT32; }
+
+ int serialize_typed_stat(common::ByteStream &out) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::write_ui32(min_value_, out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_ui32(max_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_ui32(first_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_ui32(last_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_ui64(sum_value_,
+ out))) {
+ }
+ return ret;
+ }
+ int deserialize_typed_stat(common::ByteStream &in) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::read_ui32(
+ (uint32_t &)min_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui32(
+ (uint32_t &)max_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui32(
+ (uint32_t &)first_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui32(
+ (uint32_t &)last_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui64(
+ (uint64_t &)sum_value_, in))) {
+ }
+ // std::cout << "deserialize_typed_stat. ret=" << ret
+ // << ", min_value_= " << min_value_
+ // << ", max_value_=" << max_value_
+ // << ", first_value_=" << first_value_
+ // << ", last_value_=" << last_value_
+ // << ", sum_value_=" << sum_value_
+ // << std::endl;
+ return ret;
+ }
+ int merge_with(Statistic *stat) {
+ MERGE_NUM_STAT_FROM(Int32Statistic, stat);
+ }
+
+ int deep_copy_from(Statistic *stat) {
+ DEEP_COPY_NUM_STAT_FROM(Int32Statistic, stat);
+ }
+
+ std::string to_string() const {
+ const int buf_len = 256;
+ char buf[buf_len];
+ snprintf(buf, buf_len,
+ "{count=%d, start_time=%" PRId64 ", end_time=%" PRId64
+ ", first_val=%d, last_val=%d, sum_value=%" PRId64
+ ", min_value=%d, max_value=%d}",
+ count_, start_time_, end_time_, first_value_, last_value_,
+ sum_value_, min_value_, max_value_);
+ return std::string(buf);
+ }
+};
+
+class Int64Statistic : public Statistic {
+ public:
+ double sum_value_;
+ int64_t min_value_;
+ int64_t max_value_;
+ int64_t first_value_;
+ int64_t last_value_;
+
+ Int64Statistic()
+ : sum_value_(0),
+ min_value_(0),
+ max_value_(0),
+ first_value_(0),
+ last_value_(0) {}
+
+ void clone_from(const Int64Statistic &that) {
+ count_ = that.count_;
+ start_time_ = that.start_time_;
+ end_time_ = that.end_time_;
+
+ sum_value_ = that.sum_value_;
+ min_value_ = that.min_value_;
+ max_value_ = that.max_value_;
+ first_value_ = that.first_value_;
+ last_value_ = that.last_value_;
+ }
+
+ FORCE_INLINE void update(int64_t time, int64_t value) {
+ NUM_STAT_UPDATE(time, value);
+ }
+
+ FORCE_INLINE common::TSDataType get_type() { return common::INT64; }
+
+ int serialize_typed_stat(common::ByteStream &out) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::write_ui64(min_value_, out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_ui64(max_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_ui64(first_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_ui64(last_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_double(sum_value_,
+ out))) {
+ }
+ return ret;
+ }
+ int deserialize_typed_stat(common::ByteStream &in) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::read_ui64(
+ (uint64_t &)min_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui64(
+ (uint64_t &)max_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui64(
+ (uint64_t &)first_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_ui64(
+ (uint64_t &)last_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_double(sum_value_,
+ in))) {
+ }
+ return ret;
+ }
+ int merge_with(Statistic *stat) {
+ MERGE_NUM_STAT_FROM(Int64Statistic, stat);
+ }
+
+ int deep_copy_from(Statistic *stat) {
+ DEEP_COPY_NUM_STAT_FROM(Int64Statistic, stat);
+ }
+
+ std::string to_string() const {
+ const int buf_len = 256;
+ char buf[buf_len];
+ snprintf(buf, buf_len,
+ "{count=%d, start_time=%" PRId64 ", end_time=%" PRId64
+ ", first_val=%" PRId64 ", last_val=%" PRId64
+ ", sum_value=%lf, min_value=%" PRId64 ", max_value=%" PRId64
+ "}",
+ count_, start_time_, end_time_, first_value_, last_value_,
+ sum_value_, min_value_, max_value_);
+ return std::string(buf);
+ }
+};
+
+class FloatStatistic : public Statistic {
+ public:
+ double sum_value_;
+ float min_value_;
+ float max_value_;
+ float first_value_;
+ float last_value_;
+
+ FloatStatistic()
+ : sum_value_(0),
+ min_value_(0),
+ max_value_(0),
+ first_value_(0),
+ last_value_(0) {}
+
+ void clone_from(const FloatStatistic &that) {
+ count_ = that.count_;
+ start_time_ = that.start_time_;
+ end_time_ = that.end_time_;
+
+ sum_value_ = that.sum_value_;
+ min_value_ = that.min_value_;
+ max_value_ = that.max_value_;
+ first_value_ = that.first_value_;
+ last_value_ = that.last_value_;
+ }
+ FORCE_INLINE void update(int64_t time, float value) {
+ NUM_STAT_UPDATE(time, value);
+ }
+
+ FORCE_INLINE common::TSDataType get_type() { return common::FLOAT; }
+
+ int serialize_typed_stat(common::ByteStream &out) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::write_float(min_value_, out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_float(max_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_float(first_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_float(last_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_double(sum_value_,
+ out))) {
+ }
+ return ret;
+ }
+ int deserialize_typed_stat(common::ByteStream &in) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::read_float(min_value_, in))) {
+ } else if (RET_FAIL(
+ common::SerializationUtil::read_float(max_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_float(first_value_,
+ in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_float(last_value_,
+ in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_double(sum_value_,
+ in))) {
+ }
+ return ret;
+ }
+ int merge_with(Statistic *stat) {
+ MERGE_NUM_STAT_FROM(FloatStatistic, stat);
+ }
+ int deep_copy_from(Statistic *stat) {
+ DEEP_COPY_NUM_STAT_FROM(FloatStatistic, stat);
+ }
+};
+
+class DoubleStatistic : public Statistic {
+ public:
+ double sum_value_;
+ double min_value_;
+ double max_value_;
+ double first_value_;
+ double last_value_;
+
+ DoubleStatistic()
+ : sum_value_(0),
+ min_value_(0),
+ max_value_(0),
+ first_value_(0),
+ last_value_(0) {}
+
+ void clone_from(const DoubleStatistic &that) {
+ count_ = that.count_;
+ start_time_ = that.start_time_;
+ end_time_ = that.end_time_;
+
+ sum_value_ = that.sum_value_;
+ min_value_ = that.min_value_;
+ max_value_ = that.max_value_;
+ first_value_ = that.first_value_;
+ last_value_ = that.last_value_;
+ }
+ FORCE_INLINE void update(int64_t time, double value) {
+ NUM_STAT_UPDATE(time, value);
+ }
+
+ FORCE_INLINE common::TSDataType get_type() { return common::DOUBLE; }
+
+ int serialize_typed_stat(common::ByteStream &out) {
+ int ret = common::E_OK;
+ if (RET_FAIL(
+ common::SerializationUtil::write_double(min_value_, out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_double(max_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_double(
+ first_value_, out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_double(last_value_,
+ out))) {
+ } else if (RET_FAIL(common::SerializationUtil::write_double(sum_value_,
+ out))) {
+ }
+ return ret;
+ }
+ int deserialize_typed_stat(common::ByteStream &in) {
+ int ret = common::E_OK;
+ if (RET_FAIL(common::SerializationUtil::read_double(min_value_, in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_double(max_value_,
+ in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_double(first_value_,
+ in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_double(last_value_,
+ in))) {
+ } else if (RET_FAIL(common::SerializationUtil::read_double(sum_value_,
+ in))) {
+ }
+ return ret;
+ }
+ int merge_with(Statistic *stat) {
+ MERGE_NUM_STAT_FROM(DoubleStatistic, stat);
+ }
+ int deep_copy_from(Statistic *stat) {
+ DEEP_COPY_NUM_STAT_FROM(DoubleStatistic, stat);
+ }
+};
+
+#if 0
+class BinaryStatistic : public Statistic
+{
+ // TODO
+};
+
+class TimeStatistic : public Statistic
+{
+ // TODO
+};
+#endif
+
+FORCE_INLINE uint32_t get_typed_statistic_sizeof(common::TSDataType type) {
+ uint32_t ret_size = 0;
+ switch (type) {
+ case common::BOOLEAN:
+ ret_size = sizeof(BooleanStatistic);
+ break;
+ case common::INT32:
+ ret_size = sizeof(Int32Statistic);
+ break;
+ case common::INT64:
+ ret_size = sizeof(Int64Statistic);
+ break;
+ case common::FLOAT:
+ ret_size = sizeof(FloatStatistic);
+ break;
+ case common::DOUBLE:
+ ret_size = sizeof(DoubleStatistic);
+ break;
+ case common::TEXT:
+ ASSERT(false);
+ break;
+ default:
+ ASSERT(false);
+ break;
+ }
+ return ret_size;
+}
+
+FORCE_INLINE Statistic *placement_new_statistic(common::TSDataType type,
+ void *buf) {
+ Statistic *s = nullptr;
+ switch (type) {
+ case common::BOOLEAN:
+ s = new (buf) BooleanStatistic;
+ break;
+ case common::INT32:
+ s = new (buf) Int32Statistic;
+ break;
+ case common::INT64:
+ s = new (buf) Int64Statistic;
+ break;
+ case common::FLOAT:
+ s = new (buf) FloatStatistic;
+ break;
+ case common::DOUBLE:
+ s = new (buf) DoubleStatistic;
+ break;
+ case common::TEXT:
+ ASSERT(false);
+ break;
+ default:
+ ASSERT(false);
+ break;
+ }
+ return s;
+}
+
+#define TYPED_CLONE_STATISTIC(StatType) \
+ (static_cast<StatType *>(to)) \
+ ->clone_from(*(static_cast<const StatType *>(from)))
+
+FORCE_INLINE void clone_statistic(Statistic *from, Statistic *to,
+ common::TSDataType type) {
+ ASSERT(from != nullptr);
+ ASSERT(to != nullptr);
+
+ to->count_ = from->count_;
+ to->start_time_ = from->start_time_;
+ to->end_time_ = from->end_time_;
+
+ switch (type) {
+ case common::BOOLEAN:
+ TYPED_CLONE_STATISTIC(BooleanStatistic);
+ break;
+ case common::INT32:
+ TYPED_CLONE_STATISTIC(Int32Statistic);
+ break;
+ case common::INT64:
+ TYPED_CLONE_STATISTIC(Int64Statistic);
+ break;
+ case common::FLOAT:
+ TYPED_CLONE_STATISTIC(FloatStatistic);
+ break;
+ case common::DOUBLE:
+ TYPED_CLONE_STATISTIC(DoubleStatistic);
+ break;
+ case common::TEXT:
+ ASSERT(false);
+ break;
+ default:
+ ASSERT(false);
+ }
+}
+
+#define ALLOC_STATISTIC(StatType) \
+ do { \
+ buf = common::mem_alloc(sizeof(StatType), common::MOD_STATISTIC_OBJ); \
+ if (buf != nullptr) { \
+ stat = new (buf) StatType; \
+ } \
+ } while (false)
+
+#define ALLOC_STATISTIC_WITH_PA(StatType) \
+ do { \
+ buf = pa->alloc(sizeof(StatType)); \
+ if (buf != nullptr) { \
+ stat = new (buf) StatType; \
+ } \
+ } while (false);
+
+class StatisticFactory {
+ public:
+ static Statistic *alloc_statistic(common::TSDataType data_type) {
+ void *buf = nullptr;
+ Statistic *stat = nullptr;
+ switch (data_type) {
+ case common::BOOLEAN:
+ ALLOC_STATISTIC(BooleanStatistic);
+ break;
+ case common::INT32:
+ ALLOC_STATISTIC(Int32Statistic);
+ break;
+ case common::INT64:
+ ALLOC_STATISTIC(Int64Statistic);
+ break;
+ case common::FLOAT:
+ ALLOC_STATISTIC(FloatStatistic);
+ break;
+ case common::DOUBLE:
+ ALLOC_STATISTIC(DoubleStatistic);
+ break;
+ case common::TEXT:
+ ASSERT(false);
+ break;
+ default:
+ abort();
+ ASSERT(false);
+ }
+ return stat;
+ }
+ static Statistic *alloc_statistic_with_pa(common::TSDataType data_type,
+ common::PageArena *pa) {
+ void *buf = nullptr;
+ Statistic *stat = nullptr;
+ switch (data_type) {
+ case common::BOOLEAN:
+ ALLOC_STATISTIC_WITH_PA(BooleanStatistic);
+ break;
+ case common::INT32:
+ ALLOC_STATISTIC_WITH_PA(Int32Statistic);
+ break;
+ case common::INT64:
+ ALLOC_STATISTIC_WITH_PA(Int64Statistic);
+ break;
+ case common::FLOAT:
+ ALLOC_STATISTIC_WITH_PA(FloatStatistic);
+ break;
+ case common::DOUBLE:
+ ALLOC_STATISTIC_WITH_PA(DoubleStatistic);
+ break;
+ case common::TEXT:
+ ASSERT(false);
+ break;
+ default:
+ ASSERT(false);
+ }
+ return stat;
+ }
+ static void free(Statistic *stat) { common::mem_free(stat); }
+};
+
+} // end namespace storage
+#endif // COMMON_STATISTIC_H
diff --git a/cpp/src/common/tablet.cc b/cpp/src/common/tablet.cc
new file mode 100644
index 00000000..1ed92cf4
--- /dev/null
+++ b/cpp/src/common/tablet.cc
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "tablet.h"
+
+#include <stdlib.h>
+
+#include "utils/errno_define.h"
+
+using namespace common;
+
+namespace storage {
+
+int Tablet::init() {
+ ASSERT(timestamps_ == NULL);
+ timestamps_ = (int64_t *)malloc(sizeof(int64_t) * max_rows_);
+
+ size_t schema_count = schema_vec_->size();
+ std::pair<std::map<std::string, int>::iterator, bool> ins_res;
+ for (size_t c = 0; c < schema_count; c++) {
+ ins_res = schema_map_.insert(
+ std::make_pair(schema_vec_->at(c).measurement_name_, c));
+ if (!ins_res.second) {
+ ASSERT(false);
+ // maybe dup measurement_name
+ return E_INVALID_ARG;
+ }
+ }
+ ASSERT(schema_map_.size() == schema_count);
+
+ value_matrix_ = (void **)malloc(sizeof(void *) * schema_count);
+ for (size_t c = 0; c < schema_count; c++) {
+ const MeasurementSchema &schema = schema_vec_->at(c);
+ value_matrix_[c] =
+ malloc(get_data_type_size(schema.data_type_) * max_rows_);
+ }
+
+ bitmaps_ = new BitMap[schema_count];
+ for (size_t c = 0; c < schema_count; c++) {
+ bitmaps_[c].init(max_rows_, /*init_as_zero=*/true);
+ }
+ return E_OK;
+}
+
+void Tablet::destroy() {
+ if (timestamps_ != NULL) {
+ free(timestamps_);
+ timestamps_ = NULL;
+ }
+ if (value_matrix_ != NULL) {
+ for (size_t c = 0; c < schema_vec_->size(); c++) {
+ free(value_matrix_[c]);
+ }
+ free(value_matrix_);
+ value_matrix_ = NULL;
+ }
+ if (bitmaps_ != NULL) {
+ delete[] bitmaps_;
+ }
+}
+
+int Tablet::set_timestamp(int row_index, int64_t timestamp) {
+ ASSERT(timestamps_ != NULL);
+ if (UNLIKELY(row_index >= max_rows_)) {
+ ASSERT(false);
+ return E_OUT_OF_RANGE;
+ }
+ timestamps_[row_index] = timestamp;
+ return E_OK;
+}
+
+#define DO_SET_VALUE_BY_COL_NAME(row_index, measurement_name, val) \
+ do { \
+ SchemaMapIterator find_iter = schema_map_.find(measurement_name); \
+ if (LIKELY(find_iter == schema_map_.end())) { \
+ ASSERT(false); \
+ return E_INVALID_ARG; \
+ } \
+ return set_value(row_index, find_iter->second, val); \
+ } while (false)
+
+#define DO_SET_VALUE_BY_COL_INDEX(row_index, schema_index, CppType, val) \
+ do { \
+ if (LIKELY(schema_index >= schema_vec_->size())) { \
+ ASSERT(false); \
+ return E_OUT_OF_RANGE; \
+ } \
+ const MeasurementSchema &schema = schema_vec_->at(schema_index); \
+ if (LIKELY(GetDataTypeFromTemplateType<CppType>() != \
+ schema.data_type_)) { \
+ return E_TYPE_NOT_MATCH; \
+ } \
+ CppType *column_values = (CppType *)value_matrix_[schema_index]; \
+ column_values[row_index] = val; \
+ bitmaps_[schema_index].set(row_index); /* mark as non-null*/ \
+ } while (false)
+
+int Tablet::set_value(int row_index, const std::string &measurement_name,
+ bool val) {
+ DO_SET_VALUE_BY_COL_NAME(row_index, measurement_name, val);
+}
+
+int Tablet::set_value(int row_index, const std::string &measurement_name,
+ int32_t val) {
+ DO_SET_VALUE_BY_COL_NAME(row_index, measurement_name, val);
+}
+
+int Tablet::set_value(int row_index, const std::string &measurement_name,
+ int64_t val) {
+ DO_SET_VALUE_BY_COL_NAME(row_index, measurement_name, val);
+}
+
+int Tablet::set_value(int row_index, const std::string &measurement_name,
+ float val) {
+ DO_SET_VALUE_BY_COL_NAME(row_index, measurement_name, val);
+}
+
+int Tablet::set_value(int row_index, const std::string &measurement_name,
+ double val) {
+ DO_SET_VALUE_BY_COL_NAME(row_index, measurement_name, val);
+}
+
+int Tablet::set_value(int row_index, uint32_t schema_index, bool val) {
+ DO_SET_VALUE_BY_COL_INDEX(row_index, schema_index, bool, val);
+ return E_OK;
+}
+
+int Tablet::set_value(int row_index, uint32_t schema_index, int32_t val) {
+ DO_SET_VALUE_BY_COL_INDEX(row_index, schema_index, int32_t, val);
+ return E_OK;
+}
+
+int Tablet::set_value(int row_index, uint32_t schema_index, int64_t val) {
+ DO_SET_VALUE_BY_COL_INDEX(row_index, schema_index, int64_t, val);
+ return E_OK;
+}
+
+int Tablet::set_value(int row_index, uint32_t schema_index, float val) {
+ DO_SET_VALUE_BY_COL_INDEX(row_index, schema_index, float, val);
+ return E_OK;
+}
+
+int Tablet::set_value(int row_index, uint32_t schema_index, double val) {
+ DO_SET_VALUE_BY_COL_INDEX(row_index, schema_index, double, val);
+ return E_OK;
+}
+
+} // end namespace storage
\ No newline at end of file
diff --git a/cpp/src/common/tablet.h b/cpp/src/common/tablet.h
new file mode 100644
index 00000000..163f2ee8
--- /dev/null
+++ b/cpp/src/common/tablet.h
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#ifndef COMMON_TABLET_H
+#define COMMON_TABLET_H
+
+#include <vector>
+
+#include "common/container/bit_map.h"
+#include "schema.h"
+
+namespace storage {
+
... 48048 lines suppressed ...