You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/07/17 03:44:02 UTC
[incubator-iotdb] branch dev_merge updated (b5b7400 -> 06eef43)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch dev_merge
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from b5b7400 refactor RecoverMergeTask
add efe7db7 Fix links of nav content in documents
add 9a1f8a1 Merge pull request #229 from samperson1997/document_link
add 8d2f4ac add a function to support appending data at the end of a completed TsFile
add b36a120 remove unused class import which cause an error in UT
add d24cb03 Merge pull request #235 from apache/feature_append_add_on_complete_tsfile
add a300868 [IOTDB-133] Fix mistaken links in documents
add 73f4936 Merge pull request #237 from samperson1997/new_document_link
add a7815e3 [IOTDB-133] Fix mistaken links in documents
add 52c8744 Merge pull request #238 from Genius-pig/master
add 16f0b3b fix bugs in performacne stats module; add new functions in its MBean
add 75fad74 add stopCalculatingStats in MeasurementMBean
add 2f0f4f0 modify log info in performance stats
add fc05734 Merge pull request #239 from apache/fix_performance_stats
add 57c49f6 add pathExist() check after obtain lock when trying to add path to metadata tree
add 8826fda fix typo in comments
add fd815e1 fix WriteLogNodeManagerTest failure (Travis #2557)
add 4c67e79 refactor readonly mode check
add 2bf13d2 add read-only log
add e734368 fix getting timeseries list error in grafana connector
add b018288 [IOTDB-134]Add default parameters for client (#244)
add 2678988 [IOTDB-130]Dynamic parameters adapter (#232)
add 4fb2818 fix conflict after merging master
add 7e04cd6 Merge pull request #245 from apache/fix_create_timeseries_check
add 3e95348 [IOTDB-73]Add new encoding method for regular timestamp column
add 78f0ac5 Delete TSRecordBatch.java
add 43271bb Update documentation
add d1522e7 Code reformat
add 19c56aa Merge pull request #231 from jack870131/missing_point_encoding
add 0c37235 [release preparation] Updated scm information in pom
add 1884dd2 [maven-release-plugin] prepare branch rel/0.8
add 88f8e5b [maven-release-plugin] prepare for next development iteration
new 06eef43 Merge branch 'master' into dev_merge
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/Community-Powered By.md | 8 +-
docs/Development.md | 55 ++--
docs/Documentation-CHN/QuickStart.md | 17 +
.../3-Encoding.md | 10 +-
.../4-Deployment and Management/2-Configuration.md | 12 +-
.../1-IoTDB Query Statement.md | 2 +-
.../UserGuideV0.7.0/7-Tools-Cli.md | 8 +
.../UserGuideV0.7.0/7-Tools-Grafana.md | 17 +
docs/Documentation/Frequently asked questions.md | 20 +-
.../OtherMaterial-ReleaseNotesV0.7.0.md | 16 +-
docs/Documentation/OtherMaterial-Sample Data.md | 8 +-
docs/Documentation/QuickStart.md | 22 +-
.../3-Encoding.md | 10 +-
.../4-Deployment and Management/2-Configuration.md | 14 +-
.../1-IoTDB Query Statement.md | 2 +-
docs/Documentation/UserGuideV0.7.0/7-Tools-Cli.md | 8 +-
.../UserGuideV0.7.0/7-Tools-Grafana.md | 25 +-
.../UserGuideV0.7.0/7-Tools-Hadoop.md | 3 +-
docs/Documentation/UserGuideV0.7.0/7-Tools-Sync.md | 20 +-
.../Documentation/UserGuideV0.7.0/7-Tools-spark.md | 57 ++--
example/kafka/pom.xml | 2 +-
example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
grafana/pom.xml | 6 +-
.../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 3 +-
iotdb-cli/cli/bin/start-client.bat | 6 +-
iotdb-cli/cli/bin/start-client.sh | 9 +-
iotdb-cli/pom.xml | 4 +-
iotdb/iotdb/conf/iotdb-engine.properties | 23 +-
iotdb/pom.xml | 2 +-
.../iotdb/db/auth/role/LocalFileRoleAccessor.java | 10 +-
.../iotdb/db/auth/user/LocalFileUserAccessor.java | 7 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 80 ++++-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 1 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 51 ++-
.../iotdb/db/conf/adapter/CompressionRatio.java | 184 +++++++++++
.../adapter/IDynamicAdapter.java} | 34 +-
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 275 ++++++++++++++++
.../db/conf/directories/DirectoryManager.java | 3 +
.../iotdb/db/cost/statistic/Measurement.java | 111 ++++---
.../iotdb/db/cost/statistic/MeasurementMBean.java | 14 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 84 +----
.../iotdb/db/engine/memtable/ChunkBufferPool.java | 4 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 2 +-
.../iotdb/db/engine/storagegroup/FlushManager.java | 2 +-
.../engine/storagegroup/StorageGroupProcessor.java | 44 +--
.../db/engine/storagegroup/TsFileProcessor.java | 42 ++-
...Exception.java => ConfigAdjusterException.java} | 11 +-
.../org/apache/iotdb/db/metadata/MManager.java | 96 ++++--
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 62 ++--
.../org/apache/iotdb/db/rescon/MemTablePool.java | 23 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 17 +-
.../org/apache/iotdb/db/service/JDBCService.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 73 ++---
.../writelog/manager/MultiFileLogNodeManager.java | 3 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 12 +-
.../db/conf/adapter/CompressionRatioTest.java | 115 +++++++
.../adapter/IoTDBConfigDynamicAdapterTest.java | 148 +++++++++
.../db/cost/statistic/PerformanceStatTest.java | 52 ++-
.../iotdb/db/engine/MetadataManagerHelper.java | 1 +
.../iotdb/db/engine/memtable/MemTablePoolTest.java | 20 +-
.../engine/modification/DeletionFileNodeTest.java | 1 +
.../db/engine/modification/DeletionQueryTest.java | 3 +-
.../engine/modification/ModificationFileTest.java | 1 +
.../integration/IoTDBAggregationSmallDataIT.java | 2 +-
.../db/integration/IoTDBEngineTimeGeneratorIT.java | 3 +
.../iotdb/db/integration/IoTDBLargeDataIT.java | 3 +
.../iotdb/db/integration/IoTDBMultiSeriesIT.java | 3 +
.../db/integration/IoTDBSequenceDataQueryIT.java | 3 +
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 3 +
.../org/apache/iotdb/db/metadata/MGraphTest.java | 1 +
.../iotdb/db/metadata/MManagerAdvancedTest.java | 2 +
.../iotdb/db/metadata/MManagerBasicTest.java | 1 +
.../iotdb/db/metadata/MManagerImproveTest.java | 1 +
.../org/apache/iotdb/db/metadata/MTreeTest.java | 1 +
.../org/apache/iotdb/db/metadata/MetadataTest.java | 1 +
.../org/apache/iotdb/db/qp/QueryProcessorTest.java | 4 +
.../dataset/EngineDataSetWithValueFilterTest.java | 5 +
.../iotdb/db/query/reader/ReaderTestHelper.java | 7 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 19 ++
.../iotdb/db/writelog/IoTDBLogFileSizeTest.java | 3 +
.../iotdb/db/writelog/WriteLogNodeManagerTest.java | 6 +-
jdbc/pom.xml | 8 +-
pom.xml | 14 +-
service-rpc/pom.xml | 4 +-
spark/pom.xml | 2 +-
tsfile/pom.xml | 2 +-
.../iotdb/tsfile/encoding/decoder/Decoder.java | 10 +
...aBinaryDecoder.java => RegularDataDecoder.java} | 160 +++++----
.../encoding/encoder/RegularDataEncoder.java | 361 +++++++++++++++++++++
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 27 +-
.../tsfile/file/metadata/enums/TSEncoding.java | 6 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../write/writer/RestorableTsFileIOWriter.java | 42 ++-
.../decoder/delta/DeltaBinaryEncoderLongTest.java | 32 +-
.../regular/RegularDataEncoderIntegerTest.java | 145 +++++++++
.../regular/RegularDataEncoderLongTest.java | 223 +++++++++++++
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 24 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 22 ++
99 files changed, 2535 insertions(+), 598 deletions(-)
create mode 100644 iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java
copy iotdb/src/main/java/org/apache/iotdb/db/{writelog/manager/WriteLogNodeManager.java => conf/adapter/IDynamicAdapter.java} (51%)
create mode 100644 iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
copy iotdb/src/main/java/org/apache/iotdb/db/exception/{FlushRunTimeException.java => ConfigAdjusterException.java} (77%)
create mode 100644 iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/CompressionRatioTest.java
create mode 100644 iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java
copy tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/{DeltaBinaryDecoder.java => RegularDataDecoder.java} (60%)
create mode 100644 tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/RegularDataEncoder.java
create mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderIntegerTest.java
create mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/decoder/regular/RegularDataEncoderLongTest.java
[incubator-iotdb] 01/01: Merge branch 'master' into dev_merge
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev_merge
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 06eef43fe34337983b1f4b0ea5c37f67ffab7399
Merge: b5b7400 88f8e5b
Author: 江天 <jt...@163.com>
AuthorDate: Wed Jul 17 11:41:37 2019 +0800
Merge branch 'master' into dev_merge
# Conflicts:
# iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
# iotdb/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
# iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/FlushManager.java
# iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
# tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
docs/Community-Powered By.md | 8 +-
docs/Development.md | 55 ++--
docs/Documentation-CHN/QuickStart.md | 17 +
.../3-Encoding.md | 10 +-
.../4-Deployment and Management/2-Configuration.md | 12 +-
.../1-IoTDB Query Statement.md | 2 +-
.../UserGuideV0.7.0/7-Tools-Cli.md | 8 +
.../UserGuideV0.7.0/7-Tools-Grafana.md | 17 +
docs/Documentation/Frequently asked questions.md | 20 +-
.../OtherMaterial-ReleaseNotesV0.7.0.md | 16 +-
docs/Documentation/OtherMaterial-Sample Data.md | 8 +-
docs/Documentation/QuickStart.md | 22 +-
.../3-Encoding.md | 10 +-
.../4-Deployment and Management/2-Configuration.md | 14 +-
.../1-IoTDB Query Statement.md | 2 +-
docs/Documentation/UserGuideV0.7.0/7-Tools-Cli.md | 8 +-
.../UserGuideV0.7.0/7-Tools-Grafana.md | 25 +-
.../UserGuideV0.7.0/7-Tools-Hadoop.md | 3 +-
docs/Documentation/UserGuideV0.7.0/7-Tools-Sync.md | 20 +-
.../Documentation/UserGuideV0.7.0/7-Tools-spark.md | 57 ++--
example/kafka/pom.xml | 2 +-
example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
grafana/pom.xml | 6 +-
.../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 3 +-
iotdb-cli/cli/bin/start-client.bat | 6 +-
iotdb-cli/cli/bin/start-client.sh | 9 +-
iotdb-cli/pom.xml | 4 +-
iotdb/iotdb/conf/iotdb-engine.properties | 23 +-
iotdb/pom.xml | 2 +-
.../iotdb/db/auth/role/LocalFileRoleAccessor.java | 10 +-
.../iotdb/db/auth/user/LocalFileUserAccessor.java | 7 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 80 ++++-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 1 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 51 ++-
.../iotdb/db/conf/adapter/CompressionRatio.java | 184 +++++++++++
.../iotdb/db/conf/adapter/IDynamicAdapter.java | 47 +++
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 275 ++++++++++++++++
.../db/conf/directories/DirectoryManager.java | 3 +
.../iotdb/db/cost/statistic/Measurement.java | 111 ++++---
.../iotdb/db/cost/statistic/MeasurementMBean.java | 14 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 84 +----
.../iotdb/db/engine/memtable/ChunkBufferPool.java | 4 +-
.../db/engine/querycontext/ReadOnlyMemChunk.java | 2 +-
.../iotdb/db/engine/storagegroup/FlushManager.java | 2 +-
.../engine/storagegroup/StorageGroupProcessor.java | 44 +--
.../db/engine/storagegroup/TsFileProcessor.java | 42 ++-
.../db/exception/ConfigAdjusterException.java | 37 +++
.../org/apache/iotdb/db/metadata/MManager.java | 96 ++++--
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 62 ++--
.../org/apache/iotdb/db/rescon/MemTablePool.java | 23 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 17 +-
.../org/apache/iotdb/db/service/JDBCService.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 73 ++---
.../writelog/manager/MultiFileLogNodeManager.java | 3 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 12 +-
.../db/conf/adapter/CompressionRatioTest.java | 115 +++++++
.../adapter/IoTDBConfigDynamicAdapterTest.java | 148 +++++++++
.../db/cost/statistic/PerformanceStatTest.java | 52 ++-
.../iotdb/db/engine/MetadataManagerHelper.java | 1 +
.../iotdb/db/engine/memtable/MemTablePoolTest.java | 20 +-
.../engine/modification/DeletionFileNodeTest.java | 1 +
.../db/engine/modification/DeletionQueryTest.java | 3 +-
.../engine/modification/ModificationFileTest.java | 1 +
.../integration/IoTDBAggregationSmallDataIT.java | 2 +-
.../db/integration/IoTDBEngineTimeGeneratorIT.java | 3 +
.../iotdb/db/integration/IoTDBLargeDataIT.java | 3 +
.../iotdb/db/integration/IoTDBMultiSeriesIT.java | 3 +
.../db/integration/IoTDBSequenceDataQueryIT.java | 3 +
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 3 +
.../org/apache/iotdb/db/metadata/MGraphTest.java | 1 +
.../iotdb/db/metadata/MManagerAdvancedTest.java | 2 +
.../iotdb/db/metadata/MManagerBasicTest.java | 1 +
.../iotdb/db/metadata/MManagerImproveTest.java | 1 +
.../org/apache/iotdb/db/metadata/MTreeTest.java | 1 +
.../org/apache/iotdb/db/metadata/MetadataTest.java | 1 +
.../org/apache/iotdb/db/qp/QueryProcessorTest.java | 4 +
.../dataset/EngineDataSetWithValueFilterTest.java | 5 +
.../iotdb/db/query/reader/ReaderTestHelper.java | 7 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 19 ++
.../iotdb/db/writelog/IoTDBLogFileSizeTest.java | 3 +
.../iotdb/db/writelog/WriteLogNodeManagerTest.java | 6 +-
jdbc/pom.xml | 8 +-
pom.xml | 14 +-
service-rpc/pom.xml | 4 +-
spark/pom.xml | 2 +-
tsfile/pom.xml | 2 +-
.../iotdb/tsfile/encoding/decoder/Decoder.java | 10 +
.../encoding/decoder/RegularDataDecoder.java | 289 +++++++++++++++++
.../encoding/encoder/RegularDataEncoder.java | 361 +++++++++++++++++++++
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 27 +-
.../tsfile/file/metadata/enums/TSEncoding.java | 6 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../write/writer/RestorableTsFileIOWriter.java | 42 ++-
.../decoder/delta/DeltaBinaryEncoderLongTest.java | 32 +-
.../regular/RegularDataEncoderIntegerTest.java | 145 +++++++++
.../regular/RegularDataEncoderLongTest.java | 223 +++++++++++++
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 24 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 22 ++
99 files changed, 2791 insertions(+), 510 deletions(-)
diff --cc iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 150848b,2eaa02a..4b71971
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@@ -539,39 -535,30 +572,62 @@@ public class IoTDBConfig
this.chunkBufferPoolEnable = chunkBufferPoolEnable;
}
-
+ public long getMergeMemoryBudget() {
+ return mergeMemoryBudget;
+ }
+
+ public void setMergeMemoryBudget(long mergeMemoryBudget) {
+ this.mergeMemoryBudget = mergeMemoryBudget;
+ }
+
+ public int getMergeThreadNum() {
+ return mergeThreadNum;
+ }
+
+ public void setMergeThreadNum(int mergeThreadNum) {
+ this.mergeThreadNum = mergeThreadNum;
+ }
+
+ public boolean isContinueMergeAfterReboot() {
+ return continueMergeAfterReboot;
+ }
+
+ public void setContinueMergeAfterReboot(boolean continueMergeAfterReboot) {
+ this.continueMergeAfterReboot = continueMergeAfterReboot;
+ }
+
+ public long getMergeIntervalSec() {
+ return mergeIntervalSec;
+ }
+
+ public void setMergeIntervalSec(long mergeIntervalSec) {
+ this.mergeIntervalSec = mergeIntervalSec;
+ }
+
+ public boolean isEnableParameterAdapter() {
+ return enableParameterAdapter;
+ }
+
+ public void setEnableParameterAdapter(boolean enableParameterAdapter) {
+ this.enableParameterAdapter = enableParameterAdapter;
+ }
+
+ public long getAllocateMemoryForWrite() {
+ return allocateMemoryForWrite;
+ }
+
+ public void setAllocateMemoryForWrite(long allocateMemoryForWrite) {
+ this.allocateMemoryForWrite = allocateMemoryForWrite;
+ }
+
+ public long getAllocateMemoryForRead() {
+ return allocateMemoryForRead;
+ }
+
+ public void setAllocateMemoryForRead(long allocateMemoryForRead) {
+ this.allocateMemoryForRead = allocateMemoryForRead;
+ }
+
public boolean isEnablePerformanceStat() {
return enablePerformanceStat;
}
@@@ -596,19 -583,11 +652,27 @@@
this.performance_stat_memory_in_kb = performance_stat_memory_in_kb;
}
+ public boolean isForceFullMerge() {
+ return forceFullMerge;
+ }
+
+ public void setForceFullMerge(boolean forceFullMerge) {
+ this.forceFullMerge = forceFullMerge;
+ }
+
+ public int getChunkMergePointThreshold() {
+ return chunkMergePointThreshold;
+ }
+
+ public void setChunkMergePointThreshold(int chunkMergePointThreshold) {
+ this.chunkMergePointThreshold = chunkMergePointThreshold;
+ }
++
+ public long getMemtableSizeThreshold() {
+ return memtableSizeThreshold;
+ }
+
+ public void setMemtableSizeThreshold(long memtableSizeThreshold) {
+ this.memtableSizeThreshold = memtableSizeThreshold;
+ }
}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 762cb64,49b3692..95c7d78
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@@ -230,27 -208,27 +208,6 @@@ public class StorageEngine implements I
}
}
--
-- /**
-- * begin a query on a given deviceId. Any TsFile contains such device should not be deleted at
-- * once after merge.
-- *
-- * @param deviceId queried deviceId
-- * @return a token for the query.
-- */
-- public int beginQuery(String deviceId) throws StorageEngineException {
-- // TODO implement it when developing the merge function
-- return -1;
-- }
--
-- /**
- * end query on a given deviceId. If some TsFile has been merged and this query is the
- * last query using it, the TsFile can be deleted safely.
- * end query on a given deviceId. If some TsFile has been merged and this query is the last query
- * using it, the TsFile can be deleted safely.
-- */
-- public void endQuery(String deviceId, int token) throws StorageEngineException {
-- // TODO implement it when developing the merge function
-- }
--
/**
* query data.
*/
@@@ -303,13 -274,8 +253,13 @@@
*
* @throws StorageEngineException StorageEngineException
*/
- public void mergeAll() throws StorageEngineException {
- // TODO
+ public void mergeAll(boolean fullMerge) throws StorageEngineException {
- if (readOnly) {
++ if (IoTDBDescriptor.getInstance().getConfig().isReadOnly()) {
+ throw new StorageEngineException("Current system mode is read only, does not support merge");
+ }
+ for (StorageGroupProcessor storageGroupProcessor : processorMap.values()) {
+ storageGroupProcessor.merge(fullMerge);
+ }
}
/**
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/FlushManager.java
index ffbd5fe,6141ab5..3d2ab19
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/FlushManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/FlushManager.java
@@@ -43,7 -43,7 +43,7 @@@ public class FlushManager
}
/**
- * Add TsFileProcessor to asyncFlush manager
- * Add BufferWriteProcessor to asyncTryToFlush manager
++ * Add TsFileProcessor to asyncTryToFlush manager
*/
@SuppressWarnings("squid:S2445")
void registerTsFileProcessor(TsFileProcessor tsFileProcessor) {
diff --cc iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index a9ae74d,f04469e..2aeae50
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@@ -37,13 -35,8 +37,12 @@@ import java.util.concurrent.locks.Reent
import org.apache.commons.io.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
--import org.apache.iotdb.db.engine.StorageEngine;
- import org.apache.iotdb.db.engine.merge.selector.MergeFileSelector;
+import org.apache.iotdb.db.engine.merge.manage.MergeManager;
++import org.apache.iotdb.db.engine.merge.selector.MergeFileSelector;
+import org.apache.iotdb.db.engine.merge.task.MergeTask;
+import org.apache.iotdb.db.engine.merge.task.RecoverMergeTask;
import org.apache.iotdb.db.engine.modification.Deletion;
+import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
@@@ -136,11 -127,9 +135,11 @@@ public class StorageGroupProcessor
*/
private Map<String, Long> latestFlushedTimeForEachDevice = new HashMap<>();
private String storageGroupName;
+ private File storageGroupSysDir;
+
/**
- * versionController assigns a version for each MemTable and deletion/update such that after
- * they are persisted, the order of insertions, deletions and updates can be re-determined.
+ * versionController assigns a version for each MemTable and deletion/update such that after they
+ * are persisted, the order of insertions, deletions and updates can be re-determined.
*/
private VersionController versionController;
@@@ -151,11 -140,10 +150,11 @@@
*/
@SuppressWarnings("unused") // to be used in merge
private ReentrantLock mergeDeleteLock = new ReentrantLock();
+ private ReentrantReadWriteLock mergeLock = new ReentrantReadWriteLock();
/**
- * This is the modification file of the result of the current merge. Because the merged file
- * may be invisible at this moment, without this, deletion/update during merge could be lost.
+ * This is the modification file of the result of the current merge. Because the merged file may
+ * be invisible at this moment, without this, deletion/update during merge could be lost.
*/
private ModificationFile mergingModification;
diff --cc iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 582b4ed,d8e7ead..39ff2be
--- a/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@@ -21,14 -21,15 +21,16 @@@ package org.apache.iotdb.db.service
import org.apache.iotdb.db.concurrent.IoTDBDefaultThreadExceptionHandler;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+ import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
import org.apache.iotdb.db.cost.statistic.Measurement;
import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.exception.builder.ExceptionBuilder;
+ import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.StatMonitor;
- import org.apache.iotdb.db.sync.receiver.SyncServerManager;
import org.apache.iotdb.db.rescon.TVListAllocator;
+ import org.apache.iotdb.db.sync.receiver.SyncServerManager;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --cc tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
index c08404d,9af6b6e..d218369
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/RestorableTsFileIOWriter.java
@@@ -182,4 -183,40 +187,39 @@@ public class RestorableTsFileIOWriter e
return append;
}
+ /**
+ * Given a TsFile, generate a writable RestorableTsFileIOWriter. That is, for a complete TsFile,
+ * the function erases all FileMetadata and supports writing new data; For a incomplete TsFile,
+ * the function supports writing new data directly. However, it is more efficient using the
+ * construction function of RestorableTsFileIOWriter, if the tsfile is incomplete.
+ *
+ * @param file a TsFile
+ * @return a writable RestorableTsFileIOWriter
+ */
+ public static RestorableTsFileIOWriter getWriterForAppendingDataOnCompletedTsFile(File file)
+ throws IOException {
+ long position = file.length();
+
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), false)) {
+ // this tsfile is complete
+ if (reader.isComplete()) {
+ reader.loadMetadataSize();
+ TsFileMetaData metaData = reader.readFileMetadata();
+ for (TsDeviceMetadataIndex deviceMetadata : metaData.getDeviceMap().values()) {
+ if (position > deviceMetadata.getOffset()) {
+ position = deviceMetadata.getOffset();
+ }
+ }
+ }
+ }
+
+ if (position != file.length()) {
+ // if the file is complete, we will remove all file metadatas
+ try (FileChannel channel = FileChannel
+ .open(Paths.get(file.getAbsolutePath()), StandardOpenOption.WRITE)) {
+ channel.truncate(position - 1);//remove the last marker.
+ }
+ }
+ return new RestorableTsFileIOWriter(file);
+ }
-
}