You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/06/27 08:22:37 UTC
[incubator-iotdb] branch feature_async_close_tsfile updated
(616362b -> f2fc478)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a change to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from 616362b Merge branch 'feature_async_close_tsfile' of github.com:apache/incubator-iotdb into feature_async_close_tsfile
new 02922e9 add TVListTest
new b382fe3 Merge remote-tracking branch 'origin/feature_async_close_tsfile' into feature_async_close_tsfile
new 8f824c7 Merge remote-tracking branch 'origin/feature_async_close_tsfile' into feature_async_close_tsfile
new f2fc478 reuse getSortedTimeValuePair, because it is fast...
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../db/engine/memtable/MemTableFlushTaskV2.java | 10 ++---
.../db/utils/datastructure/LongTVListTest.java | 48 +++++++++++++++++++++-
2 files changed, 52 insertions(+), 6 deletions(-)
[incubator-iotdb] 02/04: Merge remote-tracking branch
'origin/feature_async_close_tsfile' into feature_async_close_tsfile
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit b382fe3404d2284d56c1d2f4a4f3a025d3915e8e
Merge: 02922e9 3b384b7
Author: qiaojialin <64...@qq.com>
AuthorDate: Thu Jun 27 16:04:56 2019 +0800
Merge remote-tracking branch 'origin/feature_async_close_tsfile' into feature_async_close_tsfile
iotdb/iotdb/conf/iotdb-engine.properties | 4 ++--
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 ++--
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 4 ++--
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 ++--
.../java/org/apache/iotdb/db/engine/Processor.java | 10 ++++-----
.../engine/bufferwrite/BufferWriteProcessor.java | 12 +++++-----
.../bufferwrite/RestorableTsFileIOWriter.java | 8 +++----
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 2 +-
.../iotdb/db/engine/cache/TsFileMetadataUtils.java | 2 +-
.../iotdb/db/engine/filenode/FileNodeManager.java | 26 +++++++++++-----------
.../db/engine/filenode/FileNodeProcessor.java | 14 ++++++------
.../db/engine/filenode/FileNodeProcessorStore.java | 4 ++--
.../iotdb/db/engine/filenode/TsFileResource.java | 4 ++--
.../db/engine/filenodeV2/FileNodeManagerV2.java | 6 ++---
.../db/engine/filenodeV2/FileNodeProcessorV2.java | 2 +-
.../filenodeV2/UnsealedTsFileProcessorV2.java | 2 +-
.../db/engine/memcontrol/BasicMemController.java | 2 +-
.../db/engine/overflow/io/OverflowResource.java | 2 +-
.../iotdb/db/engine/pool/MergePoolManager.java | 2 +-
.../org/apache/iotdb/db/metadata/MManager.java | 6 ++---
.../org/apache/iotdb/db/monitor/StatMonitor.java | 2 +-
.../iotdb/db/qp/executor/OverflowQPExecutor.java | 2 +-
.../db/qp/physical/transfer/CodecInstances.java | 20 ++++++++---------
.../db/query/aggregation/AggregateFunction.java | 8 +++----
.../iotdb/db/query/context/QueryContext.java | 2 +-
.../iotdb/db/query/control/FileReaderManager.java | 2 +-
.../query/dataset/AggreResultDataPointReader.java | 2 +-
.../java/org/apache/iotdb/db/query/fill/IFill.java | 2 +-
.../apache/iotdb/db/service/CloseMergeService.java | 20 ++++++++---------
.../org/apache/iotdb/db/service/JDBCService.java | 6 ++---
.../org/apache/iotdb/db/service/JMXService.java | 2 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 6 ++---
.../iotdb/db/sync/conf/SyncSenderDescriptor.java | 4 ++--
.../iotdb/db/sync/receiver/SyncServerManager.java | 2 +-
.../iotdb/db/sync/receiver/SyncServiceImpl.java | 10 ++++-----
.../iotdb/db/sync/sender/SyncSenderImpl.java | 2 +-
.../java/org/apache/iotdb/db/utils/IOUtils.java | 8 +++----
.../org/apache/iotdb/db/utils/LoadDataUtils.java | 4 ++--
.../writelog/recover/TsFileRecoverPerformer.java | 2 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 4 ++--
.../apache/iotdb/db/utils/OpenFileNumUtilTest.java | 4 ++--
41 files changed, 117 insertions(+), 117 deletions(-)
[incubator-iotdb] 03/04: Merge remote-tracking branch
'origin/feature_async_close_tsfile' into feature_async_close_tsfile
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 8f824c7e7cc021bfc10c6f3228811ed5b0d6cedc
Merge: b382fe3 616362b
Author: qiaojialin <64...@qq.com>
AuthorDate: Thu Jun 27 16:07:08 2019 +0800
Merge remote-tracking branch 'origin/feature_async_close_tsfile' into feature_async_close_tsfile
.../db/engine/memtable/WritableMemChunkV2.java | 151 +++++++++++++++++++++
.../iotdb/db/utils/datastructure/LongTVList.java | 8 +-
.../iotdb/db/utils/datastructure/TVList.java | 10 +-
3 files changed, 157 insertions(+), 12 deletions(-)
[incubator-iotdb] 01/04: add TVListTest
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 02922e9a193f053da59db7a9fca27ae2e9255894
Author: qiaojialin <64...@qq.com>
AuthorDate: Thu Jun 27 16:04:41 2019 +0800
add TVListTest
---
.../db/utils/datastructure/LongTVListTest.java | 48 +++++++++++++++++++++-
1 file changed, 47 insertions(+), 1 deletion(-)
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/utils/datastructure/LongTVListTest.java b/iotdb/src/test/java/org/apache/iotdb/db/utils/datastructure/LongTVListTest.java
index 4d9e0fa..973054b 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/utils/datastructure/LongTVListTest.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/utils/datastructure/LongTVListTest.java
@@ -1,13 +1,17 @@
package org.apache.iotdb.db.utils.datastructure;
+import java.sql.Time;
+import java.util.List;
+import org.apache.iotdb.db.engine.memtable.DeduplicatedSortedData;
import org.apache.iotdb.db.engine.memtable.WritableMemChunk;
+import org.apache.iotdb.db.utils.TimeValuePair;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.junit.Test;
public class LongTVListTest {
@Test
- public void testLongTVListInsertTime() {
+ public void compareLongTVListInsertTime() {
long start = System.currentTimeMillis();
LongTVList tvList = new LongTVList();
@@ -27,4 +31,46 @@ public class LongTVListTest {
System.out.println("writable memchunk insert time: " + time);
}
+ @Test
+ public void compareLongTVListSortTime() {
+
+ LongTVList tvList = new LongTVList();
+ for (long i = 0; i < 1000000; i ++) {
+ tvList.putLong(i, i);
+ }
+
+ WritableMemChunk writableMemChunk = new WritableMemChunk(TSDataType.INT64);
+ for (long i = 0; i < 1000000; i ++) {
+ writableMemChunk.putLong(i, i);
+ }
+
+ long start = System.currentTimeMillis();
+ tvList.sort();
+ for (int i = 0; i < tvList.size; i ++) {
+ tvList.getLong(i);
+ tvList.getTime(i);
+ }
+ start = System.currentTimeMillis() - start;
+ System.out.println("tvList sort time: " + start);
+
+
+ long time1 = System.currentTimeMillis();
+ List<TimeValuePair> timeValuePairs = writableMemChunk.getSortedTimeValuePairList();
+ for (int i = 0; i < timeValuePairs.size(); i++) {
+ timeValuePairs.get(i);
+ }
+ time1 = System.currentTimeMillis() - time1;
+ System.out.println("writable memchunk getSortedTimeValuePairList time: " + time1);
+
+
+ long time2 = System.currentTimeMillis();
+ DeduplicatedSortedData deduplicatedSortedData = writableMemChunk.getDeduplicatedSortedData();
+ while(deduplicatedSortedData.hasNext()) {
+ deduplicatedSortedData.next();
+ }
+ time2 = System.currentTimeMillis() - time2;
+ System.out.println("writable memchunk getDeduplicatedSortedData time: " + time2);
+ }
+
+
}
\ No newline at end of file
[incubator-iotdb] 04/04: reuse getSortedTimeValuePair,
because it is fast...
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit f2fc4785f9c5a6b52ae02e5924d2ab5d0fa0a02b
Author: qiaojialin <64...@qq.com>
AuthorDate: Thu Jun 27 16:22:21 2019 +0800
reuse getSortedTimeValuePair, because it is fast...
---
.../apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
index f933939..3271041 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/memtable/MemTableFlushTaskV2.java
@@ -15,6 +15,7 @@
package org.apache.iotdb.db.engine.memtable;
import java.io.IOException;
+import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -80,7 +81,7 @@ public class MemTableFlushTaskV2 {
// TODO if we can not use TSFileIO writer, then we have to redesign the class of TSFileIO.
IWritableMemChunk series = memTable.getMemTableMap().get(deviceId).get(measurementId);
MeasurementSchema desc = fileSchema.getMeasurementSchema(measurementId);
- DeduplicatedSortedData sortedTimeValuePairs = series.getDeduplicatedSortedData();
+ List<TimeValuePair> sortedTimeValuePairs = series.getSortedTimeValuePairList();
sortTime += System.currentTimeMillis() - startTime;
encodingTaskQueue.add(new Pair<>(sortedTimeValuePairs, desc));
}
@@ -132,7 +133,7 @@ public class MemTableFlushTaskV2 {
ioTaskQueue.add(task);
} else {
long starTime = System.currentTimeMillis();
- Pair<DeduplicatedSortedData, MeasurementSchema> encodingMessage = (Pair<DeduplicatedSortedData, MeasurementSchema>) task;
+ Pair<List<TimeValuePair>, MeasurementSchema> encodingMessage = (Pair<List<TimeValuePair>, MeasurementSchema>) task;
ChunkBuffer chunkBuffer = new ChunkBuffer(encodingMessage.right);
IChunkWriter seriesWriter = new ChunkWriterImpl(encodingMessage.right, chunkBuffer,
PAGE_SIZE_THRESHOLD);
@@ -213,11 +214,10 @@ public class MemTableFlushTaskV2 {
};
- private void writeOneSeries(DeduplicatedSortedData tvPairs, IChunkWriter seriesWriterImpl,
+ private void writeOneSeries(List<TimeValuePair> tvPairs, IChunkWriter seriesWriterImpl,
TSDataType dataType)
throws IOException {
- while (tvPairs.hasNext()) {
- TimeValuePair timeValuePair = tvPairs.next();
+ for (TimeValuePair timeValuePair: tvPairs) {
switch (dataType) {
case BOOLEAN:
seriesWriterImpl