You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/02/23 07:06:45 UTC
[incubator-iotdb] 01/01: Merge pull request #713 from
apache/new_series_reader
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 40c19d2b368901558a47233625febe123636ac61
Merge: 3ded47f f0e59f1
Author: Jialin Qiao <qj...@mails.tsinghua.edu.cn>
AuthorDate: Sun Feb 23 15:06:28 2020 +0800
Merge pull request #713 from apache/new_series_reader
[IOTDB-418] New series reader
.travis.yml | 6 +-
.../org/apache/iotdb/client/AbstractClient.java | 104 ++--
.../SystemDesign/1-TsFile/4-Read.md | 541 ++++++++++++++++++++-
.../SystemDesign/5-DataQuery/1-DataQuery.md | 52 +-
.../SystemDesign/5-DataQuery/2-SeriesReader.md | 414 ++++++++++++++++
.../SystemDesign/5-DataQuery/3-RawDataQuery.md | 301 ++++++++++++
.../SystemDesign/5-DataQuery/4-AggregationQuery.md | 118 +++++
.../SystemDesign/5-DataQuery/5-GroupByQuery.md | 192 ++++++++
.../2-DML (Data Manipulation Language).md | 4 +-
.../5-Operation Manual/4-SQL Reference.md | 58 +--
docs/Documentation/SystemDesign/0-Content.md | 4 +
docs/Documentation/SystemDesign/1-TsFile/4-Read.md | 537 ++++++++++++++++++++
.../2-DML (Data Manipulation Language).md | 20 +-
.../5-Operation Manual/4-SQL Reference.md | 58 +--
hadoop/src/test/resources/logback.xml | 2 +-
.../src/test/resources/logback.xml | 4 +-
.../resources/conf/iotdb-engine.properties | 18 +-
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 20 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 52 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 84 ++--
.../db/conf/adapter/ActiveTimeSeriesCounter.java | 19 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 26 +-
.../db/engine/cache/CacheHitRatioMonitor.java | 5 +
.../engine/cache/CacheHitRatioMonitorMXBean.java | 2 +
.../apache/iotdb/db/engine/cache/ChunkCache.java | 150 ++++++
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 13 +-
.../iotdb/db/engine/cache/RamUsageEstimator.java | 34 +-
.../apache/iotdb/db/engine/flush/FlushManager.java | 33 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 42 +-
.../apache/iotdb/db/engine/memtable/IMemTable.java | 8 +-
.../db/engine/memtable/IWritableMemChunk.java | 16 +-
.../db/engine/memtable/MemSeriesLazyMerger.java | 50 --
.../engine/memtable/TimeValuePairInMemTable.java | 45 --
.../db/engine/memtable/TimeValuePairSorter.java | 55 ---
.../iotdb/db/engine/memtable/WritableMemChunk.java | 66 +--
.../iotdb/db/engine/merge/manage/MergeManager.java | 40 ++
.../db/engine/merge/manage/MergeResource.java | 6 +-
.../iotdb/db/engine/merge/task/MergeFileTask.java | 6 +-
.../db/engine/merge/task/MergeMultiChunkTask.java | 17 +-
.../db/engine/querycontext/QueryDataSource.java | 13 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 131 ++---
.../engine/storagegroup/StorageGroupProcessor.java | 101 ++--
.../db/engine/storagegroup/TsFileProcessor.java | 274 +++++++----
.../db/engine/storagegroup/TsFileResource.java | 20 +-
.../db/exception/ConfigAdjusterException.java | 5 +-
.../exception/DiskSpaceInsufficientException.java | 4 +-
.../{ProcessException.java => IoTDBException.java} | 7 +-
.../apache/iotdb/db/exception/MergeException.java | 5 +-
.../exception/QueryInBatchStatementException.java | 6 +-
.../iotdb/db/exception/StartupException.java | 12 +-
.../iotdb/db/exception/StorageEngineException.java | 14 +-
.../db/exception/SyncConnectionException.java | 11 +-
.../SyncDeviceOwnerConflictException.java | 8 +-
.../iotdb/db/exception/SystemCheckException.java | 6 +-
.../db/exception/TsFileProcessorException.java | 8 +-
.../db/exception/metadata/MetadataException.java | 19 +-
.../metadata/TimeseriesAlreadyExistException.java | 4 +-
.../db/exception/path/MTreePathException.java | 1 -
.../exception/path/NotStorageGroupException.java | 4 +-
.../iotdb/db/exception/path/PathException.java | 10 +-
.../exception/query/LogicalOperatorException.java | 16 +-
.../exception/query/LogicalOptimizeException.java | 6 +-
.../db/exception/query/OutOfTTLException.java | 4 +-
.../db/exception/query/QueryProcessException.java | 16 +-
.../query/UnSupportedFillTypeException.java | 4 +-
.../JDBCServiceException.java} | 13 +-
.../storageGroup/StorageGroupException.java | 7 +-
.../StorageGroupProcessorException.java | 10 +-
.../org/apache/iotdb/db/metadata/MManager.java | 11 +
.../db/qp/{QueryProcessor.java => Planner.java} | 30 +-
.../apache/iotdb/db/qp/constant/SQLConstant.java | 3 +-
.../qp/executor/AbstractQueryProcessExecutor.java | 390 ---------------
...ueryProcessExecutor.java => IPlanExecutor.java} | 48 +-
...QueryProcessExecutor.java => PlanExecutor.java} | 478 ++++++++++++++----
.../db/qp/logical/crud/BasicFunctionOperator.java | 8 +-
.../iotdb/db/qp/logical/crud/FilterOperator.java | 17 +-
.../iotdb/db/qp/logical/crud/InOperator.java | 8 +-
.../iotdb/db/qp/logical/crud/QueryOperator.java | 20 +-
.../iotdb/db/qp/physical/crud/AggregationPlan.java | 2 +-
.../db/qp/physical/crud/AlignByDevicePlan.java | 176 +++++++
.../iotdb/db/qp/physical/crud/FillQueryPlan.java | 2 +-
.../iotdb/db/qp/physical/crud/GroupByPlan.java | 13 +-
.../iotdb/db/qp/physical/crud/QueryPlan.java | 188 +------
.../db/qp/physical/crud/RawDataQueryPlan.java | 76 +++
.../iotdb/db/qp/strategy/LogicalGenerator.java | 15 +-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 177 +++----
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 20 +-
.../db/qp/strategy/optimizer/IFilterOptimizer.java | 3 +-
.../db/query/aggregation/AggreResultData.java | 192 --------
.../db/query/aggregation/AggregateFunction.java | 122 -----
.../db/query/aggregation/AggregateResult.java | 212 ++++++++
.../db/query/aggregation/impl/AvgAggrFunc.java | 173 -------
.../db/query/aggregation/impl/AvgAggrResult.java | 115 +++++
.../db/query/aggregation/impl/CountAggrFunc.java | 156 ------
.../db/query/aggregation/impl/CountAggrResult.java | 89 ++++
.../query/aggregation/impl/FirstValueAggrFunc.java | 154 ------
.../aggregation/impl/FirstValueAggrResult.java | 97 ++++
.../query/aggregation/impl/LastValueAggrFunc.java | 151 ------
.../aggregation/impl/LastValueAggrResult.java | 90 ++++
.../db/query/aggregation/impl/MaxTimeAggrFunc.java | 135 -----
.../query/aggregation/impl/MaxTimeAggrResult.java | 93 ++++
.../query/aggregation/impl/MaxValueAggrFunc.java | 170 -------
.../query/aggregation/impl/MaxValueAggrResult.java | 94 ++++
.../db/query/aggregation/impl/MinTimeAggrFunc.java | 156 ------
.../query/aggregation/impl/MinTimeAggrResult.java | 85 ++++
.../query/aggregation/impl/MinValueAggrFunc.java | 166 -------
.../query/aggregation/impl/MinValueAggrResult.java | 90 ++++
.../db/query/aggregation/impl/SumAggrResult.java | 107 ++++
.../iotdb/db/query/control/FileReaderManager.java | 21 +-
.../db/query/control/QueryResourceManager.java | 29 +-
.../query/dataset/AggreResultDataPointReader.java | 56 ---
...erateDataSet.java => AlignByDeviceDataSet.java} | 139 +++---
.../db/query/dataset/NonAlignEngineDataSet.java | 45 +-
.../OldEngineDataSetWithoutValueFilter.java | 153 ------
...er.java => RawQueryDataSetWithValueFilter.java} | 28 +-
...java => RawQueryDataSetWithoutValueFilter.java} | 103 ++--
.../iotdb/db/query/dataset/SingleDataSet.java | 3 +-
.../dataset/groupby/GroupByEngineDataSet.java | 86 +---
.../groupby/GroupByWithValueFilterDataSet.java | 71 +--
.../groupby/GroupByWithoutValueFilterDataSet.java | 385 +++++++--------
.../db/query/executor/AggregateEngineExecutor.java | 336 -------------
.../db/query/executor/AggregationExecutor.java | 289 +++++++++++
...lEngineExecutor.java => FillQueryExecutor.java} | 57 ++-
.../{IEngineQueryRouter.java => IQueryRouter.java} | 6 +-
.../{EngineQueryRouter.java => QueryRouter.java} | 123 +++--
...gineExecutor.java => RawDataQueryExecutor.java} | 98 ++--
.../db/query/externalsort/ExternalSortJob.java | 2 +-
.../query/externalsort/ExternalSortJobEngine.java | 7 +-
.../db/query/externalsort/ExternalSortJobPart.java | 2 +-
.../iotdb/db/query/externalsort/LineMerger.java | 6 +-
.../MultiSourceExternalSortJobPart.java | 2 +-
.../externalsort/SimpleExternalSortEngine.java | 15 +-
.../SingleSourceExternalSortJobPart.java | 28 +-
.../adapter/ByTimestampReaderAdapter.java | 67 ++-
.../serialize/IExternalSortFileDeserializer.java | 6 +-
.../serialize/IExternalSortFileSerializer.java | 2 +-
.../FixLengthIExternalSortFileDeserializer.java | 6 +-
.../impl/FixLengthTimeValuePairSerializer.java | 2 +-
...greFuncFactory.java => AggreResultFactory.java} | 44 +-
.../java/org/apache/iotdb/db/query/fill/IFill.java | 71 +--
.../org/apache/iotdb/db/query/fill/LinearFill.java | 51 +-
.../apache/iotdb/db/query/fill/PreviousFill.java | 39 +-
.../ChunkDataIterator.java} | 44 +-
.../{chunkRelated => chunk}/ChunkReaderWrap.java | 14 +-
.../db/query/reader/chunk/DiskChunkLoader.java | 36 +-
.../DiskChunkReaderByTimestamp.java | 64 +--
.../db/query/reader/chunk/MemChunkLoader.java | 41 +-
.../{chunkRelated => chunk}/MemChunkReader.java | 58 +--
.../chunk/MemPageReader.java} | 31 +-
.../reader/chunkRelated/CachedDiskChunkReader.java | 88 ----
.../chunkRelated/MemChunkReaderByTimestamp.java | 78 ---
.../FileSeriesReaderByTimestampAdapter.java | 49 --
.../fileRelated/UnSealedTsFileIterateReader.java | 130 -----
.../UnSealedTsFileReaderByTimestamp.java | 96 ----
.../CachedUnseqResourceMergeReader.java | 11 +-
.../NewUnseqResourceMergeReader.java | 199 --------
.../OldUnseqResourceMergeReader.java | 140 ------
.../resourceRelated/ResourceRelatedUtil.java | 48 --
.../resourceRelated/SeqResourceIterateReader.java | 157 ------
.../SeqResourceReaderByTimestamp.java | 190 --------
.../UnseqResourceReaderByTimestamp.java | 101 ----
.../db/query/reader/series}/IAggregateReader.java | 36 +-
.../reader/{ => series}/IReaderByTimestamp.java | 9 +-
.../reader/{ => series}/ManagedSeriesReader.java | 8 +-
.../query/reader/series/SeriesAggregateReader.java | 102 ++++
.../reader/series/SeriesRawDataBatchReader.java | 132 +++++
.../reader/series/SeriesRawDataPointReader.java | 100 ++++
.../iotdb/db/query/reader/series/SeriesReader.java | 471 ++++++++++++++++++
.../reader/series/SeriesReaderByTimestamp.java | 93 ++++
.../seriesRelated/SeriesReaderByTimestamp.java | 57 ---
.../seriesRelated/SeriesReaderWithValueFilter.java | 115 -----
.../SeriesReaderWithoutValueFilter.java | 270 ----------
.../universal/CachedPriorityMergeReader.java | 8 +-
.../db/query/reader/universal/IterateReader.java | 90 ----
.../reader/universal/PriorityMergeReader.java | 48 +-
.../universal/PriorityMergeReaderByTimestamp.java | 75 ---
.../timegenerator/AbstractNodeConstructor.java | 75 ---
.../query/timegenerator/EngineNodeConstructor.java | 69 ---
.../query/timegenerator/EngineTimeGenerator.java | 67 ---
.../query/timegenerator/ServerTimeGenerator.java | 76 +++
.../java/org/apache/iotdb/db/service/IService.java | 2 +
.../org/apache/iotdb/db/service/JDBCService.java | 7 +-
.../org/apache/iotdb/db/service/JMXService.java | 10 +-
.../apache/iotdb/db/service/MetricsService.java | 6 +-
.../apache/iotdb/db/service/RegisterManager.java | 2 +-
.../org/apache/iotdb/db/service/StartupChecks.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 321 ++++++------
.../java/org/apache/iotdb/db/utils/FileUtils.java | 8 +-
.../java/org/apache/iotdb/db/utils/MergeUtils.java | 1 +
.../apache/iotdb/db/utils/TimeValuePairUtils.java | 66 +--
.../iotdb/db/utils/datastructure/BinaryTVList.java | 26 +-
.../db/utils/datastructure/BooleanTVList.java | 27 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 31 +-
.../iotdb/db/utils/datastructure/FloatTVList.java | 31 +-
.../iotdb/db/utils/datastructure/IntTVList.java | 27 +-
.../iotdb/db/utils/datastructure/LongTVList.java | 26 +-
.../iotdb/db/utils/datastructure/TVList.java | 122 ++++-
.../db/engine/cache/DeviceMetaDataCacheTest.java | 6 +-
.../db/engine/memtable/PrimitiveMemTableTest.java | 80 +--
.../iotdb/db/engine/merge/MergeOverLapTest.java | 39 +-
.../iotdb/db/engine/merge/MergeTaskTest.java | 190 ++++----
.../engine/modification/DeletionFileNodeTest.java | 68 +--
.../db/engine/modification/DeletionQueryTest.java | 27 +-
.../storagegroup/StorageGroupProcessorTest.java | 44 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 96 ++--
.../engine/storagegroup/TsFileProcessorTest.java | 144 +++---
.../iotdb/db/integration/IOTDBGroupByIT.java | 43 +-
.../integration/IOTDBGroupByInnerIntervalIT.java | 26 +-
.../iotdb/db/integration/IoTDBAggregationIT.java | 170 +++++--
.../integration/IoTDBAggregationSmallDataIT.java | 129 +++--
...upbyDeviceIT.java => IoTDBAlignByDeviceIT.java} | 67 ++-
.../iotdb/db/integration/IoTDBDisableAlignIT.java | 4 +-
.../db/integration/IoTDBEngineTimeGeneratorIT.java | 10 +-
.../apache/iotdb/db/integration/IoTDBFillIT.java | 2 +-
.../iotdb/db/integration/IoTDBMergeTest.java | 3 +
.../iotdb/db/integration/IoTDBRecoverIT.java | 2 +
.../db/integration/IoTDBSequenceDataQueryIT.java | 23 +-
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 53 +-
.../{QueryProcessorTest.java => PlannerTest.java} | 7 +-
.../iotdb/db/qp/bench/QueryParseBenchmark.java | 11 +-
.../iotdb/db/qp/other/TSPlanContextAuthorTest.java | 9 +-
.../db/qp/other/TSPlanContextPropertyTest.java | 9 +-
.../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 38 +-
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 288 +++++------
.../org/apache/iotdb/db/qp/plan/QPUpdateTest.java | 206 --------
.../iotdb/db/qp/plan/TestConcatOptimizer.java | 103 ++--
.../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 196 --------
.../dataset/EngineDataSetWithValueFilterTest.java | 14 +-
.../query/executor/GroupByEngineDataSetTest.java | 27 +-
.../query/externalsort/ExternalSortEngineTest.java | 40 +-
.../db/query/externalsort/FakeChunkReaderWrap.java | 6 +-
...ExternalSortFileSerializerDeserializerTest.java | 6 +-
.../iotdb/db/query/reader/ReaderTestHelper.java | 2 +
.../fileRelated/UnSealedTsFileReaderTest.java | 112 -----
.../NewUnseqResourceMergeReaderTest.java | 109 -----
.../resourceRelated/SeqResourceReaderTest.java | 116 -----
.../resourceRelated/UnseqResourceReaderTest.java | 146 ------
.../reader/series/SeriesAggregateReaderTest.java | 116 +++++
.../reader/series/SeriesReaderByTimestampTest.java | 84 ++++
.../db/query/reader/series/SeriesReaderTest.java | 127 +++++
.../query/reader/series/SeriesReaderTestUtil.java | 177 +++++++
.../reader/seriesRelated/FakedIBatchPoint.java | 110 -----
.../reader/seriesRelated/FakedIPointReader.java | 79 ---
.../SeriesReaderWithValueFilterTest.java | 59 ---
.../SeriesReaderWithoutValueFilterTest.java | 74 ---
.../query/reader/universal/FakedSeriesReader.java | 10 +-
.../universal/FakedSeriesReaderByTimestamp.java | 79 ---
.../PriorityMergeReaderByTimestampTest.java | 172 -------
.../reader/universal/PriorityMergeReaderTest.java | 8 +-
.../reader/universal/PriorityMergeReaderTest2.java | 13 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 69 ++-
.../db/utils/datastructure/LongTVListTest.java | 2 +-
.../iotdb/db/writelog/recover/LogReplayerTest.java | 21 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 9 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 25 +-
server/src/test/resources/logback.xml | 13 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 +
.../org/apache/iotdb/session/IoTDBSessionIT.java | 60 ++-
.../apache/iotdb/tsfile/common/cache/LRUCache.java | 8 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 11 +
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 5 +-
.../iotdb/tsfile/file/header/ChunkHeader.java | 8 +-
.../iotdb/tsfile/file/metadata/ChunkMetaData.java | 412 ++++++++--------
.../apache/iotdb/tsfile/read/ReadOnlyTsFile.java | 4 +-
.../apache/iotdb/tsfile/read}/TimeValuePair.java | 2 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 11 +-
.../apache/iotdb/tsfile/read/common/BatchData.java | 52 +-
.../org/apache/iotdb/tsfile/read/common/Field.java | 40 +-
.../org/apache/iotdb/tsfile/read/common/Path.java | 3 +-
.../apache/iotdb/tsfile/read/common/RowRecord.java | 5 +
.../iotdb/tsfile/read/common/SignalBatchData.java | 5 +
.../common/{RowRecord.java => TimeColumn.java} | 67 ++-
...kLoaderImpl.java => CachedChunkLoaderImpl.java} | 16 +-
.../iotdb/tsfile/read/controller/IChunkLoader.java | 8 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 22 +-
.../tsfile/read/filter/basic/UnaryFilter.java | 4 +
.../query/dataset/DataSetWithTimeGenerator.java | 4 +-
.../tsfile/read/query/dataset/QueryDataSet.java | 34 --
.../query/executor/ExecutorWithTimeGenerator.java | 4 +-
.../read/query/timegenerator/TimeGenerator.java | 99 +++-
.../query/timegenerator/TimeGeneratorImpl.java | 130 -----
.../query/timegenerator/TsFileTimeGenerator.java | 51 ++
.../read/query/timegenerator/node/AndNode.java | 86 ++--
.../read/query/timegenerator/node/LeafNode.java | 54 +-
.../tsfile/read/query/timegenerator/node/Node.java | 5 +-
.../read/query/timegenerator/node/OrNode.java | 121 +++--
.../tsfile/read/reader/BatchDataIterator.java | 44 +-
.../iotdb/tsfile/read/reader/IChunkReader.java | 17 +-
.../{IDataReader.java => reader/IPageReader.java} | 13 +-
.../iotdb/tsfile/read}/reader/IPointReader.java | 12 +-
.../tsfile/read/reader/chunk/ChunkReader.java | 87 ++--
.../read/reader/chunk/ChunkReaderByTimestamp.java | 4 +-
.../iotdb/tsfile/read/reader/page/PageReader.java | 42 +-
.../reader/series/AbstractFileSeriesReader.java | 37 +-
.../read/reader/series/EmptyFileSeriesReader.java | 5 -
.../org/apache/iotdb/tsfile/utils/PublicBAOS.java | 13 -
.../apache/iotdb/tsfile/write/TsFileWriter.java | 23 +-
.../write/writer/ForceAppendTsFileWriter.java | 8 +-
.../write/writer/RestorableTsFileIOWriter.java | 5 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 9 +-
.../decoder/delta/DeltaBinaryEncoderLongTest.java | 18 +-
.../apache/iotdb/tsfile/read/ExpressionTest.java | 60 +++
.../org/apache/iotdb/tsfile/read/ReadTest.java | 18 +-
.../iotdb/tsfile/read/common/BatchDataTest.java | 82 ++++
.../tsfile/read/controller/ChunkLoaderTest.java | 2 +-
.../read/filter/MinTimeMaxTimeFilterTest.java | 6 +
.../read/query/executor/QueryExecutorTest.java | 4 +-
.../tsfile/read/query/timegenerator/NodeTest.java | 38 +-
.../query/timegenerator/ReaderByTimestampTest.java | 4 +-
.../query/timegenerator/TimeGeneratorTest.java | 9 +-
.../iotdb/tsfile/read/reader/ReaderTest.java | 6 +-
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 56 +--
.../iotdb/tsfile/utils/TsPrimitiveTypeTest.java | 60 +++
.../iotdb/tsfile/write/TsFileWriterTest.java | 26 +-
.../iotdb/tsfile/write/writer/PageWriterTest.java | 18 +-
315 files changed, 10368 insertions(+), 10471 deletions(-)