You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/07/19 13:56:30 UTC
[incubator-iotdb] branch master updated (99e2596 -> f1adbb4)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from 99e2596 [IOTDB-799]remove log visualizer code and doc (#1516)
add 72169dc feat:add vm
add 1771ef0 fix chunkmeta bugs
add ea534cf update hot merge
add de6130e Merge branch 'hot_compaction' into add_vm_merge
add 873ad3b Merge pull request #1401 from zhanglingzhe0820/add_vm_merge
add 6483032 query adapter
add a10b404 update StorageGroupProcessor
add d08b086 Merge pull request #1410 from zhanglingzhe0820/add_vm_merge
add 8466408 resolve conflicts
add 6e0b8bd some changes in flush
add f88863b fix ChunkHeader merge
add c682ab3 fix small issues
add 5711c27 fix some small issues
add c2216eb clear get num
add bdb67d8 Fix hot compaction (#1412)
add 96f72bf fix recover folder
add 27a579d Fix hot compaction (#1415)
add ae637b6 add comment
add 6510a20 fix restart
add a50f46f fix recover vm
add b68bc19 delete unusable serialize
add a64002a query lock
add 96d2909 remove unused config
add acd8039 Merge branch 'hot_compaction' of https://github.com/apache/incubator-iotdb into hot_compaction
add 9aa0bc2 Merge branch 'hot_compaction' of https://github.com/apache/incubator-iotdb into hot_compaction
add 0b81e0b update mergeio
add c6abcb9 Merge branch 'hot_compaction' of https://github.com/apache/incubator-iotdb into hot_compaction
add 7596294 Merge branch 'hot_compaction' of https://github.com/apache/incubator-iotdb into hot_compaction
add 0ed3f9f Merge branch 'hot_compaction' of https://github.com/apache/incubator-iotdb into hot_compaction
add 39235aa Merge branch 'hot_compaction' of https://github.com/apache/incubator-iotdb into hot_compaction
add d3f80d2 update schema get
add 4cc7103 Merge pull request #1419 from zhanglingzhe0820/hot_compaction
add d2bd803 format memtable flush task
add 343b7e8 fix path
add cf089c6 Merge branch 'hot_compaction' of https://github.com/apache/incubator-iotdb into hot_compaction
add 2d4b394 Merge remote-tracking branch 'origin/hot_compaction' into hot_compaction
add 2ac8ce3 fix failed tests
add abee581 add recover support
add 62f4095 no need to serialize vmfile
add 7767fac update merge recover (#1421)
add 99cbcf4 add unseq tsfile processor recover
add 888daa4 fix resource readlock lack origin readlock
add 1e05957 fix sequence in TsFileReoverPerformer and MemtableFlushTask
add 820c377 Merge branch 'hot_compaction' of https://github.com/apache/incubator-iotdb into hot_compaction
add 5c4f304 only serialize closed file
add 7fae051 and file exists judgment
add 886084e fix close bug
add 5b70863 and file exists judgment (#1424)
add 71b0c45 update startTime and endTime in vmfile tsfile resource
add f2057b3 add licenses
add aba6218 Merge branch 'hot_compaction' into fix_hot_compaction_recover
add 50b4054 Merge pull request #1426 from zhanglingzhe0820/fix_hot_compaction_recover
add 2f71fb2 fix query deadlock
add 564d42d recover bug
add 105c019 fix recover bug
add cf15a27 add some comments
add 09f0bcb add merge version
add 592612c Merge commit '2f71fb218d32646396f34e5fd2819968923b125a' into fix_hot_compaction_recover
add 3471ff9 Merge branch 'hot_compaction' into fix_hot_compaction_recover
add 451d422 add tmp remove logic
add a622a26 Merge pull request #1427 from zhanglingzhe0820/fix_hot_compaction_recover
add f3e21bf fix recover
add 85ba4ee update version
add 731a526 Merge branch 'hot_compaction' into fix_hot_compaction_recover
add e93b1dc add merge
add cd7481f Merge pull request #1428 from zhanglingzhe0820/fix_hot_compaction_recover
add 4f68137 fix SeriesReaderIT
add 9934b36 fix vm file merge bug
add 5b94dc2 Merge pull request #1432 from zhanglingzhe0820/fix_vm_merge_bug
add b44f46a add recover processing for crashing while memtable is not full
add 83f3d9b fix recover
add b01f724 Merge branch 'master' into hot_compaction
add 9fcb20d Merge branch 'hot_compaction' of https://github.com/apache/incubator-iotdb into hot_compaction
add 871fce0 add test
add 524c9fe resolve conflict
add c493621 add properties
add cff8f1d change vm mergeTimes to flushTimes
add b263bf0 change flush vm times to 11 and add log
add c038c03 fix test
add b59aad1 merge master
add c1472cb Merge branch 'master' of https://github.com/apache/incubator-iotdb into hot_compaction
add 71b4711 fix review
add a9b2e87 fix test
add acfdee2 fix test
add d0d5815 split vm with flush (#1468)
add a7e9030 fix shouldClose
add 11f747a fix bug
add 336c034 Fix sonar problems in hot_compaction branch (#1469)
add 4b6fe39 Fix more sonar issues
add c20cdb7 Fix typo
add 732c112 fix conflicts
add 80355cd Fix tests
add 4dcd1d3 fix conflicts
add 42ce3f6 Fix conflicts
add 95de1f8 Merge remote-tracking branch 'origin/master' into hot_compaction2
add 6b8e118 Fix sonar problems resulted by merging master
add caae141 change back the delete way
add dd88ab1 change name
add d4ff7f9 make query right
add 25c40da add javadoc for RestorableTsFileIOWriter
add 938e80a add log
add 07cc9dc fix windows ci bug
add 703014b add log close
add cb4bcf0 fix
add 782518d fix
add a100b24 fix
add e52e0e4 Merge branch 'hot_compaction' into fix_windows_ci_bug
add 9bf7da2 fix
add a436db2 just end file
add 9e30f0f Merge branch 'fix_windows_ci_bug' into add_vm_logs
add 2cefde3 update
add 4d7b1cf remove endFile
add f6d6d1d Merge pull request #1477 from zhanglingzhe0820/add_vm_logs
add 5e97e01 fix tests
add a4922ca Merge branch 'master' into hot_compaction
add 4b023b7 update log and add some reader close (#1480)
add 619172f fix sonar
add 92546c4 fix sonar
add e71e710 rename upgrade to tmp suffix
add 6d6ae7b reduce duplication
add f53b94a fix vm recover bug
add c2675fa fix
add 69ae505 update vm logger
add 7f35d50 fix
add 8f2fc3b add truncate
add 1e9297c add logger close
add f39b9e9 update vm merge utils
add f620159 log
add 2943f48 update log
add c979feb fix vm pack bug
add 3b3ac1a update filename compare
add 326898e add log
add f1f0307 fix sonar
add 97a9937 change back memtable size in IoTDBConfig
add 4d2cc32 Merge remote-tracking branch 'origin/master' into hot_compaction
new f1adbb4 Merge pull request #1411 from apache/hot_compaction
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../resources/conf/iotdb-engine.properties | 11 +
.../org/apache/iotdb/db/concurrent/ThreadName.java | 1 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 57 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 16 +
.../org/apache/iotdb/db/engine/StorageEngine.java | 2 +-
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 2 +-
.../apache/iotdb/db/engine/flush/FlushManager.java | 14 +-
.../iotdb/db/engine/flush/FlushManagerMBean.java | 1 -
.../iotdb/db/engine/flush/MemTableFlushTask.java | 56 +-
.../iotdb/db/engine/flush/TsFileFlushPolicy.java | 4 +-
.../iotdb/db/engine/flush/VmLogAnalyzer.java | 95 +++
.../org/apache/iotdb/db/engine/flush/VmLogger.java | 76 +++
.../apache/iotdb/db/engine/flush/VmMergeUtils.java | 225 +++++++
.../db/engine/flush/pool/AbstractPoolManager.java | 7 +-
.../engine/flush/pool/FlushSubTaskPoolManager.java | 5 +-
.../db/engine/flush/pool/FlushTaskPoolManager.java | 5 +-
...oolManager.java => VmMergeTaskPoolManager.java} | 27 +-
.../db/engine/merge/manage/MergeResource.java | 4 +-
.../iotdb/db/engine/merge/recover/LogAnalyzer.java | 8 +-
.../iotdb/db/engine/merge/recover/MergeLogger.java | 4 +-
.../merge/selector/MaxFileMergeFileSelector.java | 6 +-
.../iotdb/db/engine/merge/task/MergeFileTask.java | 30 +-
.../iotdb/db/engine/merge/task/MergeTask.java | 2 +-
.../db/engine/merge/task/RecoverMergeTask.java | 8 +-
.../engine/storagegroup/StorageGroupProcessor.java | 509 ++++++++-------
.../iotdb/db/engine/storagegroup/TsFileLock.java | 4 +-
.../db/engine/storagegroup/TsFileProcessor.java | 683 +++++++++++++++++----
.../db/engine/storagegroup/TsFileResource.java | 102 +--
.../iotdb/db/engine/upgrade/UpgradeTask.java | 14 +-
.../iotdb/db/query/control/FileReaderManager.java | 16 +-
.../iotdb/db/query/control/QueryFileManager.java | 3 +-
.../db/query/reader/chunk/DiskChunkLoader.java | 2 +-
.../chunk/metadata/DiskChunkMetadataLoader.java | 2 +-
.../db/sync/receiver/load/FileLoaderManager.java | 2 +-
.../db/sync/sender/manage/SyncFileManager.java | 2 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 6 +-
.../org/apache/iotdb/db/utils/FilePathUtils.java | 2 +-
.../java/org/apache/iotdb/db/utils/MergeUtils.java | 37 +-
.../org/apache/iotdb/db/utils/UpgradeUtils.java | 8 +-
.../iotdb/db/writelog/recover/LogReplayer.java | 10 +-
.../writelog/recover/TsFileRecoverPerformer.java | 248 ++++++--
.../db/engine/cache/ChunkMetadataCacheTest.java | 12 +-
.../db/engine/memtable/MemTableFlushTaskTest.java | 10 +-
.../iotdb/db/engine/merge/MergeOverLapTest.java | 2 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 2 +-
.../iotdb/db/engine/merge/MergeUpgradeTest.java | 6 +-
.../storagegroup/StorageGroupProcessorTest.java | 37 +-
.../storagegroup/TsFileProcessorEnableVmTest.java | 194 ++++++
.../engine/storagegroup/TsFileProcessorTest.java | 152 +++--
.../iotdb/db/integration/IoTDBLargeDataIT.java | 2 +-
.../db/integration/IoTDBLoadExternalTsfileIT.java | 40 +-
.../iotdb/db/integration/IoTDBRecoverIT.java | 72 ++-
.../db/query/control/FileReaderManagerTest.java | 4 +-
.../query/reader/series/SeriesReaderTestUtil.java | 2 +-
.../db/sync/receiver/load/FileLoaderTest.java | 6 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 19 +-
.../iotdb/db/writelog/recover/LogReplayerTest.java | 32 +-
.../recover/RecoverResourceFromReaderTest.java | 6 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 8 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 4 +-
.../tsfile/common/constant/TsFileConstant.java | 5 +-
.../iotdb/tsfile/file/header/ChunkHeader.java | 32 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 16 +-
.../org/apache/iotdb/tsfile/read/common/Chunk.java | 9 +
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 1 +
.../iotdb/tsfile/write/chunk/IChunkWriter.java | 6 +
.../write/writer/RestorableTsFileIOWriter.java | 12 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 5 +
68 files changed, 2270 insertions(+), 742 deletions(-)
create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/flush/VmLogAnalyzer.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/flush/VmLogger.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/flush/VmMergeUtils.java
copy server/src/main/java/org/apache/iotdb/db/engine/flush/pool/{FlushSubTaskPoolManager.java => VmMergeTaskPoolManager.java} (68%)
create mode 100644 server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorEnableVmTest.java
[incubator-iotdb] 01/01: Merge pull request #1411 from
apache/hot_compaction
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit f1adbb41664bf08b74c6e1f5352a9fab28f8db47
Merge: 99e2596 4d2cc32
Author: Jialin Qiao <qj...@mails.tsinghua.edu.cn>
AuthorDate: Sun Jul 19 21:56:13 2020 +0800
Merge pull request #1411 from apache/hot_compaction
[IOTDB-706] Introduce virtual memtable for larger Chunk
.../resources/conf/iotdb-engine.properties | 11 +
.../org/apache/iotdb/db/concurrent/ThreadName.java | 1 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 57 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 16 +
.../org/apache/iotdb/db/engine/StorageEngine.java | 2 +-
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 2 +-
.../apache/iotdb/db/engine/flush/FlushManager.java | 14 +-
.../iotdb/db/engine/flush/FlushManagerMBean.java | 1 -
.../iotdb/db/engine/flush/MemTableFlushTask.java | 56 +-
.../iotdb/db/engine/flush/TsFileFlushPolicy.java | 4 +-
.../iotdb/db/engine/flush/VmLogAnalyzer.java | 95 +++
.../org/apache/iotdb/db/engine/flush/VmLogger.java | 76 +++
.../apache/iotdb/db/engine/flush/VmMergeUtils.java | 225 +++++++
.../db/engine/flush/pool/AbstractPoolManager.java | 7 +-
.../engine/flush/pool/FlushSubTaskPoolManager.java | 5 +-
.../db/engine/flush/pool/FlushTaskPoolManager.java | 5 +-
...oolManager.java => VmMergeTaskPoolManager.java} | 27 +-
.../db/engine/merge/manage/MergeResource.java | 4 +-
.../iotdb/db/engine/merge/recover/LogAnalyzer.java | 8 +-
.../iotdb/db/engine/merge/recover/MergeLogger.java | 4 +-
.../merge/selector/MaxFileMergeFileSelector.java | 6 +-
.../iotdb/db/engine/merge/task/MergeFileTask.java | 30 +-
.../iotdb/db/engine/merge/task/MergeTask.java | 2 +-
.../db/engine/merge/task/RecoverMergeTask.java | 8 +-
.../engine/storagegroup/StorageGroupProcessor.java | 509 ++++++++-------
.../iotdb/db/engine/storagegroup/TsFileLock.java | 4 +-
.../db/engine/storagegroup/TsFileProcessor.java | 683 +++++++++++++++++----
.../db/engine/storagegroup/TsFileResource.java | 102 +--
.../iotdb/db/engine/upgrade/UpgradeTask.java | 14 +-
.../iotdb/db/query/control/FileReaderManager.java | 16 +-
.../iotdb/db/query/control/QueryFileManager.java | 3 +-
.../db/query/reader/chunk/DiskChunkLoader.java | 2 +-
.../chunk/metadata/DiskChunkMetadataLoader.java | 2 +-
.../db/sync/receiver/load/FileLoaderManager.java | 2 +-
.../db/sync/sender/manage/SyncFileManager.java | 2 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 6 +-
.../org/apache/iotdb/db/utils/FilePathUtils.java | 2 +-
.../java/org/apache/iotdb/db/utils/MergeUtils.java | 37 +-
.../org/apache/iotdb/db/utils/UpgradeUtils.java | 8 +-
.../iotdb/db/writelog/recover/LogReplayer.java | 10 +-
.../writelog/recover/TsFileRecoverPerformer.java | 248 ++++++--
.../db/engine/cache/ChunkMetadataCacheTest.java | 12 +-
.../db/engine/memtable/MemTableFlushTaskTest.java | 10 +-
.../iotdb/db/engine/merge/MergeOverLapTest.java | 2 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 2 +-
.../iotdb/db/engine/merge/MergeUpgradeTest.java | 6 +-
.../storagegroup/StorageGroupProcessorTest.java | 37 +-
.../storagegroup/TsFileProcessorEnableVmTest.java | 194 ++++++
.../engine/storagegroup/TsFileProcessorTest.java | 152 +++--
.../iotdb/db/integration/IoTDBLargeDataIT.java | 2 +-
.../db/integration/IoTDBLoadExternalTsfileIT.java | 40 +-
.../iotdb/db/integration/IoTDBRecoverIT.java | 72 ++-
.../db/query/control/FileReaderManagerTest.java | 4 +-
.../query/reader/series/SeriesReaderTestUtil.java | 2 +-
.../db/sync/receiver/load/FileLoaderTest.java | 6 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 19 +-
.../iotdb/db/writelog/recover/LogReplayerTest.java | 32 +-
.../recover/RecoverResourceFromReaderTest.java | 6 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 8 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 4 +-
.../tsfile/common/constant/TsFileConstant.java | 5 +-
.../iotdb/tsfile/file/header/ChunkHeader.java | 32 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 16 +-
.../org/apache/iotdb/tsfile/read/common/Chunk.java | 9 +
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 1 +
.../iotdb/tsfile/write/chunk/IChunkWriter.java | 6 +
.../write/writer/RestorableTsFileIOWriter.java | 12 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 5 +
68 files changed, 2270 insertions(+), 742 deletions(-)