You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/06/21 01:35:59 UTC
[incubator-iotdb] branch feature_async_close_tsfile updated
(e56e464 -> 0fa0095)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a change to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from e56e464 merge
new eb775f8 add filenodeprocessor test half
new c512192 Merge remote-tracking branch 'origin/feature_async_close_tsfile' into feature_async_close_tsfile
new 0fa0095 resolve query conflict
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../db/engine/filenodeV2/TsFileResourceV2.java | 25 ++++++--
.../db/engine/querycontext/QueryDataSourceV2.java | 18 +++---
.../db/engine/querycontext/UnsealedTsFileV2.java | 70 ----------------------
.../iotdb/db/query/control/JobFileManager.java | 7 +--
.../reader/sequence/SequenceDataReaderV2.java | 6 +-
.../reader/sequence/UnSealedTsFileReaderV2.java | 8 +--
.../engine/filenodeV2/FileNodeProcessorV2Test.java | 36 ++++++++++-
7 files changed, 71 insertions(+), 99 deletions(-)
delete mode 100644 iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/UnsealedTsFileV2.java
[incubator-iotdb] 02/03: Merge remote-tracking branch
'origin/feature_async_close_tsfile' into feature_async_close_tsfile
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit c512192d6127b86f641d9a00472e2bbf06329394
Merge: eb775f8 e56e464
Author: qiaojialin <64...@qq.com>
AuthorDate: Fri Jun 21 09:23:56 2019 +0800
Merge remote-tracking branch 'origin/feature_async_close_tsfile' into feature_async_close_tsfile
.../db/engine/filenodeV2/FileNodeManagerV2.java | 10 +-
.../db/engine/filenodeV2/FileNodeProcessorV2.java | 0
.../db/engine/querycontext/UnsealedTsFileV2.java | 70 +++++++++++
.../iotdb/db/query/control/JobFileManager.java | 31 +++++
.../db/query/control/QueryResourceManager.java | 15 +++
.../AbstractExecutorWithoutTimeGeneratorV2.java | 85 +++++++++++++
.../EngineExecutorWithoutTimeGenerator.java | 2 +-
.../sequence/FileSeriesIAggregateReader.java | 59 +++++++++
.../db/query/reader/sequence/IterateReader.java | 82 +++++++++++++
.../reader/sequence/SequenceDataReaderV2.java | 132 +++++++++++++++++++++
.../reader/sequence/UnSealedTsFileReaderV2.java | 82 +++++++++++++
11 files changed, 563 insertions(+), 5 deletions(-)
[incubator-iotdb] 01/03: add filenodeprocessor test half
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit eb775f862b71989e511aca69e51952fbdd9ca30b
Author: qiaojialin <64...@qq.com>
AuthorDate: Fri Jun 21 09:22:13 2019 +0800
add filenodeprocessor test half
---
.../db/engine/filenodeV2/FileNodeProcessorV2.java | 14 +--------
.../db/engine/querycontext/QueryDataSourceV2.java | 18 +++++------
.../engine/filenodeV2/FileNodeProcessorV2Test.java | 36 ++++++++++++++++++++--
3 files changed, 44 insertions(+), 24 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
index b3ef56f..02cea35 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2.java
@@ -109,25 +109,13 @@ public class FileNodeProcessorV2 {
"directory {}", storageGroupName, storageGroupDir.getAbsolutePath());
}
- /**
- * restore
- */
- File restoreFolder = new File(absoluteBaseDir + storageGroupName);
- if (!restoreFolder.exists()) {
- restoreFolder.mkdirs();
- LOGGER.info("The restore directory of the filenode processor {} doesn't exist. Create new " +
- "directory {}", storageGroupName, restoreFolder.getAbsolutePath());
- }
-
- String absoluteFileNodeRestoreFilePath = new File(restoreFolder, storageGroupName + RESTORE_FILE_SUFFIX).getAbsolutePath();
-
recovery();
/**
* version controller
*/
try {
- versionController = new SimpleFileVersionController(restoreFolder.getPath());
+ versionController = new SimpleFileVersionController(storageGroupDir.getAbsolutePath());
} catch (IOException e) {
throw new FileNodeProcessorException(e);
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSourceV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSourceV2.java
index b39e3e1..fb12950 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSourceV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSourceV2.java
@@ -21,22 +21,22 @@ package org.apache.iotdb.db.engine.querycontext;
public class QueryDataSourceV2 {
// sequence data source
- private GlobalSortedSeriesDataSourceV2 seriesDataSource;
+ private GlobalSortedSeriesDataSourceV2 sequenceDataSource;
// unSequence data source
- private GlobalSortedSeriesDataSourceV2 overflowSeriesDataSource;
+ private GlobalSortedSeriesDataSourceV2 unSequenceDataSource;
- public QueryDataSourceV2(GlobalSortedSeriesDataSourceV2 seriesDataSource,
- GlobalSortedSeriesDataSourceV2 overflowSeriesDataSource) {
- this.seriesDataSource = seriesDataSource;
- this.overflowSeriesDataSource = overflowSeriesDataSource;
+ public QueryDataSourceV2(GlobalSortedSeriesDataSourceV2 sequenceDataSource,
+ GlobalSortedSeriesDataSourceV2 unSequenceDataSource) {
+ this.sequenceDataSource = sequenceDataSource;
+ this.unSequenceDataSource = unSequenceDataSource;
}
public GlobalSortedSeriesDataSourceV2 getSeqDataSource() {
- return seriesDataSource;
+ return sequenceDataSource;
}
- public GlobalSortedSeriesDataSourceV2 getOverflowSeriesDataSource() {
- return overflowSeriesDataSource;
+ public GlobalSortedSeriesDataSourceV2 getUnSequenceDataSource() {
+ return unSequenceDataSource;
}
}
diff --git a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2Test.java b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2Test.java
index 3810b56..385b86e 100644
--- a/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2Test.java
+++ b/iotdb/src/test/java/org/apache/iotdb/db/engine/filenodeV2/FileNodeProcessorV2Test.java
@@ -20,27 +20,38 @@ package org.apache.iotdb.db.engine.filenodeV2;
import java.util.Collections;
import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.MetadataManagerHelper;
+import org.apache.iotdb.db.engine.bufferwrite.BufferWriteProcessor;
+import org.apache.iotdb.db.engine.filenode.FileNodeProcessor;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSourceV2;
+import org.apache.iotdb.db.exception.BufferWriteProcessorException;
+import org.apache.iotdb.db.exception.FileNodeProcessorException;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
+import org.junit.Test;
public class FileNodeProcessorV2Test {
- private UnsealedTsFileProcessorV2 processor;
private String storageGroup = "storage_group1";
private String baseDir = "data";
private String deviceId = "root.vehicle.d0";
private String measurementId = "s0";
private TSDataType dataType = TSDataType.INT32;
private Map<String, String> props = Collections.emptyMap();
-
+ private FileNodeProcessorV2 processor;
@Before
public void setUp() throws Exception {
MetadataManagerHelper.initMetadata();
EnvironmentUtils.envSetUp();
+ processor = new FileNodeProcessorV2(baseDir, storageGroup);
}
@After
@@ -50,4 +61,25 @@ public class FileNodeProcessorV2Test {
}
+ @Test
+ public void testAsyncClose()
+ throws FileNodeProcessorException, BufferWriteProcessorException, ExecutionException, InterruptedException {
+
+
+ for (int j = 1; j <= 10; j++) {
+ TSRecord record = new TSRecord(j, deviceId);
+ for (int i = 0; i < 10; i++) {
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(i)));
+ }
+ processor.insert(record);
+ processor.asyncForceClose();
+ }
+
+ QueryDataSourceV2 queryDataSource = processor.query(deviceId, measurementId);
+
+ Assert.assertEquals(queryDataSource.getSeqDataSource().getQueryTsFiles().size(), 10);
+
+ }
+
+
}
\ No newline at end of file
[incubator-iotdb] 03/03: resolve query conflict
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch feature_async_close_tsfile
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 0fa0095713b4e452e716ccef8ce9e3d9ec88ea66
Author: qiaojialin <64...@qq.com>
AuthorDate: Fri Jun 21 09:35:47 2019 +0800
resolve query conflict
---
.../db/engine/filenodeV2/TsFileResourceV2.java | 25 ++++++--
.../db/engine/querycontext/UnsealedTsFileV2.java | 70 ----------------------
.../iotdb/db/query/control/JobFileManager.java | 7 +--
.../reader/sequence/SequenceDataReaderV2.java | 6 +-
.../reader/sequence/UnSealedTsFileReaderV2.java | 8 +--
5 files changed, 28 insertions(+), 88 deletions(-)
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/TsFileResourceV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/TsFileResourceV2.java
index a732f0d..b9dac47 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/TsFileResourceV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/engine/filenodeV2/TsFileResourceV2.java
@@ -50,12 +50,13 @@ public class TsFileResourceV2 {
/**
* Chunk metadata list of unsealed tsfile
*/
- private List<ChunkMetaData> timeSeriesChunkMetaDatas;
+ private List<ChunkMetaData> chunkMetaDatas;
+
/**
* Mem chunk data
*/
- private ReadOnlyMemChunk readableChunk;
+ private ReadOnlyMemChunk readOnlyMemChunk;
public TsFileResourceV2(File file) {
this.file = file;
@@ -70,11 +71,11 @@ public class TsFileResourceV2 {
}
public TsFileResourceV2(File file,
- ReadOnlyMemChunk readableChunk,
- List<ChunkMetaData> timeSeriesChunkMetaDatas) {
+ ReadOnlyMemChunk readOnlyMemChunk,
+ List<ChunkMetaData> chunkMetaDatas) {
this.file = file;
- this.timeSeriesChunkMetaDatas = timeSeriesChunkMetaDatas;
- this.readableChunk = readableChunk;
+ this.chunkMetaDatas = chunkMetaDatas;
+ this.readOnlyMemChunk = readOnlyMemChunk;
}
public void updateStartTime(String device, long time) {
@@ -85,6 +86,18 @@ public class TsFileResourceV2 {
}
}
+ public List<ChunkMetaData> getChunkMetaDatas() {
+ return chunkMetaDatas;
+ }
+
+ public ReadOnlyMemChunk getReadOnlyMemChunk() {
+ return readOnlyMemChunk;
+ }
+
+ public ModificationFile getModFile() {
+ return modFile;
+ }
+
public File getFile() {
return file;
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/UnsealedTsFileV2.java b/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/UnsealedTsFileV2.java
deleted file mode 100644
index 58e988d..0000000
--- a/iotdb/src/main/java/org/apache/iotdb/db/engine/querycontext/UnsealedTsFileV2.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.engine.querycontext;
-
-import java.io.File;
-import java.util.List;
-import java.util.Map;
-import org.apache.iotdb.db.engine.filenodeV2.TsFileResourceV2;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
-
-public class UnsealedTsFileV2 extends TsFileResourceV2 {
-
- private List<ChunkMetaData> timeSeriesChunkMetaDatas;
-
- // seq mem-table
- private ReadOnlyMemChunk readableChunk;
-
- public UnsealedTsFileV2(File file) {
- super(file);
- }
-
- public UnsealedTsFileV2(File file, Map<String, Long> startTimeMap,
- Map<String, Long> endTimeMap) {
- super(file, startTimeMap, endTimeMap);
- }
-
- public UnsealedTsFileV2(File file, ReadOnlyMemChunk readableChunk,
- List<ChunkMetaData> timeSeriesChunkMetaDatas) {
- super(file);
- this.timeSeriesChunkMetaDatas = timeSeriesChunkMetaDatas;
- this.readableChunk = readableChunk;
- }
-
- @Override
- public TSFILE_TYPE getTsFileType() {
- return TSFILE_TYPE.UNSEALED;
- }
-
- public List<ChunkMetaData> getChunkMetaDataList() {
- return timeSeriesChunkMetaDatas;
- }
-
- public void setTimeSeriesChunkMetaDatas(List<ChunkMetaData> timeSeriesChunkMetaDatas) {
- this.timeSeriesChunkMetaDatas = timeSeriesChunkMetaDatas;
- }
-
- public ReadOnlyMemChunk getReadableChunk() {
- return readableChunk;
- }
-
- public void setReadableChunk(ReadOnlyMemChunk readableChunk) {
- this.readableChunk = readableChunk;
- }
-}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
index 0b44c01..256c9fc 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/control/JobFileManager.java
@@ -23,7 +23,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.iotdb.db.engine.filenode.TsFileResource;
import org.apache.iotdb.db.engine.filenodeV2.TsFileResourceV2;
-import org.apache.iotdb.db.engine.filenodeV2.TsFileResourceV2.TSFILE_TYPE;
import org.apache.iotdb.db.engine.querycontext.OverflowInsertFile;
import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.querycontext.QueryDataSourceV2;
@@ -85,7 +84,7 @@ public class JobFileManager {
//sequence data
for(TsFileResourceV2 tsFileResource : dataSource.getSeqDataSource().getQueryTsFiles()){
String path = tsFileResource.getFile().getPath();
- if(tsFileResource.getTsFileType().equals(TSFILE_TYPE.SEALED)){
+ if(tsFileResource.isClosed()){
addFilePathToMap(jobId, path, true);
}
else {
@@ -94,9 +93,9 @@ public class JobFileManager {
}
//overflow data
- for(TsFileResourceV2 tsFileResource : dataSource.getOverflowSeriesDataSource().getQueryTsFiles()){
+ for(TsFileResourceV2 tsFileResource : dataSource.getUnSequenceDataSource().getQueryTsFiles()){
String path = tsFileResource.getFile().getPath();
- if(tsFileResource.getTsFileType().equals(TSFILE_TYPE.SEALED)){
+ if(tsFileResource.isClosed()){
addFilePathToMap(jobId, path, true);
}
else {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderV2.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderV2.java
index 25e235d..c4e2652 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/SequenceDataReaderV2.java
@@ -22,10 +22,8 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
import org.apache.iotdb.db.engine.filenodeV2.TsFileResourceV2;
-import org.apache.iotdb.db.engine.filenodeV2.TsFileResourceV2.TSFILE_TYPE;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.querycontext.GlobalSortedSeriesDataSourceV2;
-import org.apache.iotdb.db.engine.querycontext.UnsealedTsFileV2;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.IAggregateReader;
@@ -62,11 +60,11 @@ public class SequenceDataReaderV2 extends IterateReader {
this.enableReverse = isReverse;
for (TsFileResourceV2 tsFileResource : sources.getQueryTsFiles()) {
- if (tsFileResource.getTsFileType().equals(TSFILE_TYPE.SEALED)) {
+ if (tsFileResource.isClosed()) {
constructSealedTsFileReader(tsFileResource, filter, context, seriesReaders);
} else {
seriesReaders.add(
- new UnSealedTsFileReaderV2((UnsealedTsFileV2) tsFileResource, filter, enableReverse));
+ new UnSealedTsFileReaderV2(tsFileResource, filter, enableReverse));
}
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFileReaderV2.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFileReaderV2.java
index 97dfce1..d9fdadb 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFileReaderV2.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/sequence/UnSealedTsFileReaderV2.java
@@ -22,7 +22,7 @@ package org.apache.iotdb.db.query.reader.sequence;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
-import org.apache.iotdb.db.engine.querycontext.UnsealedTsFileV2;
+import org.apache.iotdb.db.engine.filenodeV2.TsFileResourceV2;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.reader.mem.MemChunkReader;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
@@ -48,13 +48,13 @@ public class UnSealedTsFileReaderV2 extends IterateReader {
* @param isReverse true-traverse chunks from behind forward; false-traverse chunks from front to
* back;
*/
- public UnSealedTsFileReaderV2(UnsealedTsFileV2 unsealedTsFile, Filter filter, boolean isReverse)
+ public UnSealedTsFileReaderV2(TsFileResourceV2 unsealedTsFile, Filter filter, boolean isReverse)
throws IOException {
TsFileSequenceReader unClosedTsFileReader = FileReaderManager.getInstance()
.get(unsealedTsFile.getFile().getPath(), false);
ChunkLoader chunkLoader = new ChunkLoaderImpl(unClosedTsFileReader);
- List<ChunkMetaData> metaDataList = unsealedTsFile.getChunkMetaDataList();
+ List<ChunkMetaData> metaDataList = unsealedTsFile.getChunkMetaDatas();
// reverse chunk metadata list if traversing chunks from behind forward
if (isReverse && metaDataList != null && !metaDataList.isEmpty()) {
Collections.reverse(metaDataList);
@@ -69,7 +69,7 @@ public class UnSealedTsFileReaderV2 extends IterateReader {
}
// data in flushing memtable
- MemChunkReader memChunkReader = new MemChunkReader(unsealedTsFile.getReadableChunk(), filter);
+ MemChunkReader memChunkReader = new MemChunkReader(unsealedTsFile.getReadOnlyMemChunk(), filter);
if (isReverse) {
seriesReaders.add(memChunkReader);
seriesReaders.add(new FileSeriesIAggregateReader(unSealedReader));