You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/06/28 03:48:14 UTC
[incubator-iotdb] branch feature_async_close_tsfile updated
(308fbc3 -> 5d5e5bf)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a change to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from 308fbc3 Merge branch 'feature_async_close_tsfile' of github.com:apache/incubator-iotdb into feature_async_close_tsfile
new b30eae4 add storage group name in log
new 8190de7 add log in filenodeprocessor to show the size of memtable
new 5d5e5bf Merge remote-tracking branch 'origin/feature_async_close_tsfile' into feature_async_close_tsfile
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../iotdb/db/engine/filenodeV2/FileNodeManagerV2.java | 12 ++++--------
.../iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java | 10 ----------
.../org/apache/iotdb/db/engine/filenodeV2/FlushManager.java | 9 ++++++---
.../db/engine/filenodeV2/UnsealedTsFileProcessorV2.java | 13 +++++++++----
4 files changed, 19 insertions(+), 25 deletions(-)
[incubator-iotdb] 02/03: add log in filenodeprocessor to show the
size of memtable
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 8190de72d19712d93773a4a707dfaa9260ef011e
Author: qiaojialin <64...@qq.com>
AuthorDate: Fri Jun 28 11:47:45 2019 +0800
add log in filenodeprocessor to show the size of memtable
---
.../apache/iotdb/db/engine/filenodeV2/FileNodeManagerV2.java | 12 ++++--------
.../db/engine/filenodeV2/UnsealedTsFileProcessorV2.java | 3 ++-
2 files changed, 6 insertions(+), 9 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeManagerV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeManagerV2.java
index 178a54a..514f40c 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeManagerV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeManagerV2.java
@@ -117,11 +117,7 @@ public class FileNodeManagerV2 implements IService {
@Override
public void stop() {
- try {
- syncCloseAllProcessor();
- } catch (FileNodeManagerException e) {
- LOGGER.error("Failed to close file node manager because .", e);
- }
+ syncCloseAllProcessor();
}
@Override
@@ -349,7 +345,7 @@ public class FileNodeManagerV2 implements IService {
*/
public void addTimeSeries(Path path, TSDataType dataType, TSEncoding encoding,
CompressionType compressor,
- Map<String, String> props) throws FileNodeManagerException, ProcessorException {
+ Map<String, String> props) throws FileNodeManagerException {
FileNodeProcessorV2 fileNodeProcessor = getProcessor(path.getFullPath());
fileNodeProcessor.addTimeSeries(path.getMeasurement(), dataType, encoding, compressor, props);
}
@@ -358,7 +354,7 @@ public class FileNodeManagerV2 implements IService {
/**
* delete all filenode.
*/
- public synchronized boolean deleteAll() throws FileNodeManagerException {
+ public synchronized boolean deleteAll() {
LOGGER.info("Start deleting all filenode");
// TODO
return true;
@@ -368,7 +364,7 @@ public class FileNodeManagerV2 implements IService {
* flush command
* Sync asyncCloseOneProcessor all file node processors.
*/
- public void syncCloseAllProcessor() throws FileNodeManagerException {
+ public void syncCloseAllProcessor() {
LOGGER.info("Start closing all filenode processor");
synchronized (processorMap){
for(FileNodeProcessorV2 processor: processorMap.values()){
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java
index 54d3181..b87fb7d 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java
@@ -145,7 +145,8 @@ public class UnsealedTsFileProcessorV2 {
workMemTable.insert(insertPlan);
start2 = System.currentTimeMillis() - start2;
if (start2 > 1000) {
- LOGGER.info("UFP {} insert into memtable cost: {}, insertPlan: {}", storageGroupName, start2, insertPlan);
+ LOGGER.info("UFP {} insert into memtable cost: {}, insertPlan: {}, current data points in memtable",
+ storageGroupName, start2, insertPlan, workMemTable.size());
}
return true;
[incubator-iotdb] 03/03: Merge remote-tracking branch
'origin/feature_async_close_tsfile' into feature_async_close_tsfile
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 5d5e5bf77d56633d4da9a9c5d651682e24b05837
Merge: 8190de7 308fbc3
Author: qiaojialin <64...@qq.com>
AuthorDate: Fri Jun 28 11:47:56 2019 +0800
Merge remote-tracking branch 'origin/feature_async_close_tsfile' into feature_async_close_tsfile
.../iotdb/db/engine/memtable/FlushTaskPool.java | 117 +++++++++++++++++++++
.../db/engine/memtable/MemTableFlushTaskV2.java | 5 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 38 ++++---
.../iotdb/db/utils/datastructure/LongTVList.java | 32 +++---
.../iotdb/db/utils/datastructure/TVList.java | 40 ++++++-
.../iotdb/tsfile/write/chunk/ChunkBuffer.java | 18 +++-
6 files changed, 217 insertions(+), 33 deletions(-)
[incubator-iotdb] 01/03: add storage group name in log
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit b30eae409b23f5a402c15712d536cf3fd3fd52d1
Author: qiaojialin <64...@qq.com>
AuthorDate: Fri Jun 28 10:57:37 2019 +0800
add storage group name in log
---
.../apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java | 10 ----------
.../org/apache/iotdb/db/engine/filenodeV2/FlushManager.java | 9 ++++++---
.../iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java | 10 +++++++---
3 files changed, 13 insertions(+), 16 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
index 61b077d..cca0151 100755
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
@@ -234,13 +234,7 @@ public class FileNodeProcessorV2 {
* @return -1: failed, 1: Overflow, 2:Bufferwrite
*/
public boolean insert(InsertPlan insertPlan) {
- long lockStartTime = System.currentTimeMillis();
writeLock();
- long lockElapsed = System.currentTimeMillis() - lockStartTime;
- if (lockElapsed > 1000) {
- LOGGER.info("FNP {} insert waiting for lock costs {} ms", storageGroupName, lockElapsed);
- }
- long start = System.currentTimeMillis();
try {
if (toBeClosed) {
throw new FileNodeProcessorException(
@@ -260,10 +254,6 @@ public class FileNodeProcessorV2 {
LOGGER.error("insert tsRecord to unsealed data file failed, because {}", e.getMessage(), e);
return false;
} finally {
- long elapse = System.currentTimeMillis() - start;
- if (elapse > 2000) {
- LOGGER.info("long-tail insertion, cost: {}", elapse);
- }
writeUnlock();
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FlushManager.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FlushManager.java
index b971b75..002f809 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FlushManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FlushManager.java
@@ -43,11 +43,13 @@ public class FlushManager {
try {
unsealedTsFileProcessor.flushOneMemTable();
} catch (IOException e) {
- LOGGER.error("flush one memtable meet error", e);
+ LOGGER.error("storage group {} flush one memtable meet error",
+ unsealedTsFileProcessor.getStorageGroupName(), e);
// TODO do sth
}
unsealedTsFileProcessor.setManagedByFlushManager(false);
- LOGGER.info("flush process consume {} ms", System.currentTimeMillis() - startTime);
+ LOGGER.info("storage group {} flush process consume {} ms",
+ unsealedTsFileProcessor.getStorageGroupName(), System.currentTimeMillis() - startTime);
registerUnsealedTsFileProcessor(unsealedTsFileProcessor);
}
}
@@ -58,7 +60,8 @@ public class FlushManager {
public Future registerUnsealedTsFileProcessor(UnsealedTsFileProcessorV2 unsealedTsFileProcessor) {
synchronized (unsealedTsFileProcessor) {
if (!unsealedTsFileProcessor.isManagedByFlushManager() && unsealedTsFileProcessor.getFlushingMemTableSize() > 0) {
- LOGGER.info("begin to submit a flush thread, flushing memtable size: {}", unsealedTsFileProcessor.getFlushingMemTableSize());
+ LOGGER.info("storage group {} begin to submit a flush thread, flushing memtable size: {}",
+ unsealedTsFileProcessor.getStorageGroupName(), unsealedTsFileProcessor.getFlushingMemTableSize());
unsealedTsFileProcessorQueue.add(unsealedTsFileProcessor);
unsealedTsFileProcessor.setManagedByFlushManager(true);
return flushPool.submit(new FlushThread());
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java
index e708a44..54d3181 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/UnsealedTsFileProcessorV2.java
@@ -215,9 +215,9 @@ public class UnsealedTsFileProcessorV2 {
try {
IMemTable tmpMemTable = workMemTable == null ? new EmptyMemTable() : workMemTable;
if (!tmpMemTable.isManagedByMemPool()) {
- LOGGER.info("add an empty memtable into flushing memtable list when async close");
+ LOGGER.info("storage group {} add an empty memtable into flushing memtable list when async close", storageGroupName);
} else {
- LOGGER.info("async flush a memtable when async close");
+ LOGGER.info("storage group {} async flush a memtable when async close", storageGroupName);
}
flushingMemTables.add(tmpMemTable);
shouldClose = true;
@@ -307,7 +307,7 @@ public class UnsealedTsFileProcessorV2 {
IMemTable memTableToFlush;
memTableToFlush = flushingMemTables.getFirst();
- LOGGER.info("start to flush a memtable in a flush thread");
+ LOGGER.info("storage group {} start to flush a memtable in a flush thread", storageGroupName);
// null memtable only appears when calling asyncClose()
if (memTableToFlush.isManagedByMemPool()) {
@@ -409,6 +409,10 @@ public class UnsealedTsFileProcessorV2 {
return versionController;
}
+ public String getStorageGroupName() {
+ return storageGroupName;
+ }
+
/**
* get the chunk(s) in the memtable ( one from work memtable and the other ones in flushing status
* and then compact them into one TimeValuePairSorter). Then get the related ChunkMetadatas of