You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/06/25 04:14:57 UTC
[incubator-iotdb] branch feature_async_close_tsfile updated
(39f6d01 -> 023959e)
This is an automated email from the ASF dual-hosted git repository.
lta pushed a change to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from 39f6d01 Merge branch 'feature_async_close_tsfile' of github.com:apache/incubator-iotdb into feature_async_close_tsfile
new 71386ef modify filenode processor test
new f73d593 Merge branch 'feature_async_close_tsfile' of github.com:apache/incubator-iotdb into feature_async_close_tsfile
new dcd6feb Merge branch 'feature_async_close_tsfile' of github.com:apache/incubator-iotdb into feature_async_close_tsfile
new 023959e memtable pool clear
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../db/engine/filenodeV2/FileNodeProcessorV2.java | 60 +++++++++++-----------
.../iotdb/db/engine/memtable/AbstractMemTable.java | 2 +
.../filenodeV2/UnsealedTsFileProcessorV2Test.java | 2 +-
3 files changed, 32 insertions(+), 32 deletions(-)
[incubator-iotdb] 04/04: memtable pool clear
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 023959e71b48943caae027abb08be18d97171f59
Author: lta <li...@163.com>
AuthorDate: Tue Jun 25 12:14:44 2019 +0800
memtable pool clear
---
.../main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index 1beafc5..b90570d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -130,6 +130,8 @@ public abstract class AbstractMemTable implements IMemTable {
@Override
public void clear() {
memTableMap.clear();
+ modifications.clear();
+ memSize = 0;
}
@Override
[incubator-iotdb] 01/04: modify filenode processor test
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 71386ef5fefbe895cc02444ea33753fe07c82703
Author: lta <li...@163.com>
AuthorDate: Tue Jun 25 11:45:07 2019 +0800
modify filenode processor test
---
.../db/engine/filenodeV2/FileNodeProcessorV2.java | 60 +++++++++++-----------
.../filenodeV2/UnsealedTsFileProcessorV2Test.java | 2 +-
2 files changed, 30 insertions(+), 32 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
index d7f6f6b..365860e 100755
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
@@ -90,7 +90,7 @@ public class FileNodeProcessorV2 {
private final ReadWriteLock lock = new ReentrantReadWriteLock();
- private Object closeFileNodeCondition = new Object();
+ private final Object closeFileNodeCondition = new Object();
/**
* Mark whether to close file node
@@ -233,6 +233,7 @@ public class FileNodeProcessorV2 {
* @return -1: failed, 1: Overflow, 2:Bufferwrite
*/
public boolean insert(InsertPlan insertPlan) {
+ lock.writeLock().lock();
try {
if (toBeClosed) {
throw new FileNodeProcessorException(
@@ -251,47 +252,44 @@ public class FileNodeProcessorV2 {
} catch (FileNodeProcessorException | IOException e) {
LOGGER.error("insert tsRecord to unsealed data file failed, because {}", e.getMessage(), e);
return false;
+ } finally {
+ lock.writeLock().unlock();
}
}
private boolean insertUnsealedDataFile(InsertPlan insertPlan, boolean sequence)
throws IOException {
- lock.writeLock().lock();
UnsealedTsFileProcessorV2 unsealedTsFileProcessor;
- try {
- boolean result;
- // create a new BufferWriteProcessor
- if (sequence) {
- if (workSequenceTsFileProcessor == null) {
- workSequenceTsFileProcessor = createTsFileProcessor(true);
- sequenceFileList.add(workSequenceTsFileProcessor.getTsFileResource());
- }
- unsealedTsFileProcessor = workSequenceTsFileProcessor;
- } else {
- if (workUnSequenceTsFileProcessor == null) {
- workUnSequenceTsFileProcessor = createTsFileProcessor(false);
- unSequenceFileList.add(workUnSequenceTsFileProcessor.getTsFileResource());
- }
- unsealedTsFileProcessor = workUnSequenceTsFileProcessor;
+ boolean result;
+ // create a new BufferWriteProcessor
+ if (sequence) {
+ if (workSequenceTsFileProcessor == null) {
+ workSequenceTsFileProcessor = createTsFileProcessor(true);
+ sequenceFileList.add(workSequenceTsFileProcessor.getTsFileResource());
}
-
- // insert BufferWrite
- result = unsealedTsFileProcessor.insert(insertPlan);
-
- // try to update the latest time of the device of this tsRecord
- if (result && latestTimeForEachDevice.get(insertPlan.getDeviceId()) < insertPlan.getTime()) {
- latestTimeForEachDevice.put(insertPlan.getDeviceId(), insertPlan.getTime());
+ unsealedTsFileProcessor = workSequenceTsFileProcessor;
+ } else {
+ if (workUnSequenceTsFileProcessor == null) {
+ workUnSequenceTsFileProcessor = createTsFileProcessor(false);
+ unSequenceFileList.add(workUnSequenceTsFileProcessor.getTsFileResource());
}
+ unsealedTsFileProcessor = workUnSequenceTsFileProcessor;
+ }
- // check memtable size and may asyncFlush the workMemtable
- if (unsealedTsFileProcessor.shouldFlush()) {
- flushAndCheckShouldClose(unsealedTsFileProcessor, sequence);
- }
+ // insert BufferWrite
+ result = unsealedTsFileProcessor.insert(insertPlan);
- return result;
- } finally {
- lock.writeLock().unlock();
+ // try to update the latest time of the device of this tsRecord
+ if (result && latestTimeForEachDevice.get(insertPlan.getDeviceId()) < insertPlan.getTime()) {
+ latestTimeForEachDevice.put(insertPlan.getDeviceId(), insertPlan.getTime());
}
+
+ // check memtable size and may asyncFlush the workMemtable
+ if (unsealedTsFileProcessor.shouldFlush()) {
+ flushAndCheckShouldClose(unsealedTsFileProcessor, sequence);
+ }
+
+ return result;
}
private UnsealedTsFileProcessorV2 createTsFileProcessor(boolean sequence) throws IOException {
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2Test.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2Test.java
index 7d08217..ce00700 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2Test.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2Test.java
@@ -50,7 +50,7 @@ public class UnsealedTsFileProcessorV2Test {
private UnsealedTsFileProcessorV2 processor;
private String storageGroup = "storage_group1";
- private String filePath = "testUnsealedTsFileProcessor.tsfile";
+ private String filePath = "data/testUnsealedTsFileProcessor.tsfile";
private String deviceId = "root.vehicle.d0";
private String measurementId = "s0";
private TSDataType dataType = TSDataType.INT32;
[incubator-iotdb] 02/04: Merge branch 'feature_async_close_tsfile'
of github.com:apache/incubator-iotdb into feature_async_close_tsfile
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit f73d5930b1becb646eeee928cbe8fbbdde9404bf
Merge: 71386ef c3a36d1
Author: lta <li...@163.com>
AuthorDate: Tue Jun 25 11:45:20 2019 +0800
Merge branch 'feature_async_close_tsfile' of github.com:apache/incubator-iotdb into feature_async_close_tsfile
.../db/engine/memtable/MemTableFlushTaskV2.java | 50 +++++++++++++++-------
1 file changed, 35 insertions(+), 15 deletions(-)
[incubator-iotdb] 03/04: Merge branch 'feature_async_close_tsfile'
of github.com:apache/incubator-iotdb into feature_async_close_tsfile
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit dcd6feb7c23a0673626b39040019b5689817e852
Merge: f73d593 39f6d01
Author: lta <li...@163.com>
AuthorDate: Tue Jun 25 12:09:39 2019 +0800
Merge branch 'feature_async_close_tsfile' of github.com:apache/incubator-iotdb into feature_async_close_tsfile
.../engine/bufferwrite/BufferWriteProcessor.java | 4 +-
.../db/engine/filenode/FileNodeProcessor.java | 2 +-
.../db/engine/filenodeV2/FileNodeProcessorV2.java | 4 +-
.../db/engine/memtable/MemTableFlushTaskV2.java | 38 ++-
.../db/qp/executor/IQueryProcessExecutor.java | 8 +-
.../groupby/GroupByWithValueFilterDataSet.java | 2 +-
.../db/query/executor/AggregateEngineExecutor.java | 6 +-
...rWithTimeGenerator.java => EngineExecutor.java} | 63 ++++-
.../EngineExecutorWithoutTimeGenerator.java | 103 -------
.../iotdb/db/query/executor/EngineQueryRouter.java | 17 +-
.../db/query/executor/FillEngineExecutor.java | 2 +-
.../db/query/factory/ISeriesReaderFactory.java | 12 +-
.../db/query/factory/SeriesReaderFactory.java | 312 ---------------------
.../db/query/factory/SeriesReaderFactoryImpl.java | 8 +-
.../java/org/apache/iotdb/db/query/fill/IFill.java | 2 +-
.../query/reader/sequence/SealedTsFilesReader.java | 183 ------------
.../sequence/SealedTsFilesReaderByTimestamp.java | 136 ---------
.../query/reader/sequence/SequenceDataReader.java | 148 ----------
.../sequence/SequenceDataReaderByTimestamp.java | 108 -------
.../reader/sequence/UnSealedTsFileReader.java | 101 -------
.../sequence/UnSealedTsFilesReaderByTimestamp.java | 64 -----
.../query/timegenerator/EngineNodeConstructor.java | 3 +-
.../org/apache/iotdb/db/service/StartupChecks.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 4 +-
.../iotdb/db/engine/memtable/MemTablePoolTest.java | 68 +++++
.../iotdb/db/integration/IoTDBAuthorizationIT.java | 2 +-
.../iotdb/db/integration/IoTDBLargeDataIT.java | 4 +-
.../iotdb/db/integration/IoTDBMultiSeriesIT.java | 4 +-
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 4 +-
.../iotdb/db/integration/IoTDBTimeZoneIT.java | 2 +-
.../org/apache/iotdb/db/qp/plan/QPUpdateTest.java | 2 +-
.../SequenceDataReaderByTimestampTest.java | 91 ------
...reateByTimestampReadersOfSelectedPathsTest.java | 2 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 6 +
34 files changed, 195 insertions(+), 1322 deletions(-)