You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ea...@apache.org on 2020/06/02 05:33:03 UTC
[incubator-iotdb] branch nvmlogging updated: update
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch nvmlogging
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/nvmlogging by this push:
new 672bb97 update
672bb97 is described below
commit 672bb978b58d6f698a37e66925dd2b02c4209be6
Author: mdf369 <95...@qq.com>
AuthorDate: Tue Jun 2 11:39:17 2020 +0800
update
---
pom.xml | 1 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 --
.../org/apache/iotdb/db/engine/StorageEngine.java | 6 ---
.../iotdb/db/engine/flush/MemTableFlushTask.java | 2 -
.../engine/storagegroup/StorageGroupProcessor.java | 6 +--
.../db/engine/storagegroup/TsFileProcessor.java | 6 +--
.../java/org/apache/iotdb/db/nvm/PerfMonitor.java | 45 ----------------------
.../nvm/recover/NVMMemtableRecoverPerformer.java | 3 --
.../iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java | 10 ++---
.../org/apache/iotdb/db/nvm/space/NVMSpace.java | 5 +++
.../apache/iotdb/db/nvm/space/NVMSpaceManager.java | 1 +
.../apache/iotdb/db/rescon/PrimitiveArrayPool.java | 5 +--
.../java/org/apache/iotdb/db/service/IoTDB.java | 6 ---
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 3 --
.../iotdb/db/utils/datastructure/FloatTVList.java | 9 -----
.../db/utils/datastructure/NVMDoubleTVList.java | 12 +++++-
.../db/utils/datastructure/NVMFloatTVList.java | 3 --
.../iotdb/db/utils/datastructure/NVMTVList.java | 16 ++------
.../iotdb/db/utils/datastructure/TVList.java | 3 --
.../iotdb/db/writelog/io/MultiFileLogReader.java | 7 ----
.../db/writelog/node/ExclusiveWriteLogNode.java | 1 +
.../iotdb/db/writelog/recover/LogReplayer.java | 4 +-
.../writelog/recover/TsFileRecoverPerformer.java | 20 ----------
.../iotdb/db/integration/IoTDBAggregationIT.java | 11 ++++++
24 files changed, 42 insertions(+), 146 deletions(-)
diff --git a/pom.xml b/pom.xml
index 99cae40..ce39a4f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,7 +57,6 @@
<module>hadoop</module>
<module>spark-iotdb-connector</module>
<module>distribution</module>
- <module>hive-connector</module>
</modules>
<!-- Properties Management -->
<properties>
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index d677c35..7069c2f 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -355,9 +355,6 @@ public class IoTDBDescriptor {
Boolean.toString(conf.isEnableNVM()))));
conf.setNvmDir(properties.getProperty("nvm_dir", conf.getNvmDir()));
-
- conf.setArraySize(Integer.parseInt(properties.getProperty("array_size",
- Integer.toString(conf.getArraySize()))));
}
private void loadAutoCreateSchemaProps(Properties properties){
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 7017a78..dcea62e 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -51,7 +51,6 @@ import org.apache.iotdb.db.exception.storageGroup.StorageGroupException;
import org.apache.iotdb.db.exception.storageGroup.StorageGroupProcessorException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.metadata.MNode;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
import org.apache.iotdb.db.query.context.QueryContext;
@@ -111,7 +110,6 @@ public class StorageEngine implements IService {
/*
* recover all storage group processors.
*/
-
List<MNode> sgNodes = MManager.getInstance().getAllStorageGroups();
List<Future> futures = new ArrayList<>();
for (MNode storageGroup : sgNodes) {
@@ -213,7 +211,6 @@ public class StorageEngine implements IService {
*/
public void insert(InsertPlan insertPlan)
throws StorageEngineException, QueryProcessException {
- long time = System.currentTimeMillis();
StorageGroupProcessor storageGroupProcessor;
try {
storageGroupProcessor = getProcessor(insertPlan.getDeviceId());
@@ -226,7 +223,6 @@ public class StorageEngine implements IService {
// TODO monitor: update statistics
try {
storageGroupProcessor.insert(insertPlan);
- PerfMonitor.add("StorageEngine.insert", System.currentTimeMillis() - time);
} catch (QueryProcessException e) {
throw new QueryProcessException(e);
}
@@ -238,7 +234,6 @@ public class StorageEngine implements IService {
* @return result of each row
*/
public Integer[] insertBatch(BatchInsertPlan batchInsertPlan) throws StorageEngineException {
- long time = System.currentTimeMillis();
StorageGroupProcessor storageGroupProcessor;
try {
storageGroupProcessor = getProcessor(batchInsertPlan.getDeviceId());
@@ -252,7 +247,6 @@ public class StorageEngine implements IService {
// TODO monitor: update statistics
try {
Integer[] res = storageGroupProcessor.insertBatch(batchInsertPlan);
- PerfMonitor.add("StorageEngine.insertBatch", System.currentTimeMillis() - time);
return res;
} catch (QueryProcessException e) {
throw new StorageEngineException(e);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
index cb5615c..8873dc4 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
@@ -27,7 +27,6 @@ import org.apache.iotdb.db.engine.flush.pool.FlushSubTaskPoolManager;
import org.apache.iotdb.db.engine.memtable.IMemTable;
import org.apache.iotdb.db.engine.memtable.IWritableMemChunk;
import org.apache.iotdb.db.exception.runtime.FlushRunTimeException;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.utils.datastructure.AbstractTVList;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -94,7 +93,6 @@ public class MemTableFlushTask {
logger.debug(
"Storage group {} memtable {}, flushing into disk: data sort time cost {} ms.",
storageGroup, memTable.getVersion(), sortTime);
- PerfMonitor.add("sort", sortTime);
ioTaskFuture.get();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 49da491..900e145 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -65,7 +65,6 @@ import org.apache.iotdb.db.exception.query.OutOfTTLException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.exception.storageGroup.StorageGroupProcessorException;
import org.apache.iotdb.db.metadata.MManager;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
@@ -208,9 +207,8 @@ public class StorageGroupProcessor {
throw new StorageGroupProcessorException(e);
}
- long time = System.currentTimeMillis();
recover();
- PerfMonitor.add("StoragaGroupProcessor.recocer", System.currentTimeMillis() - time);
+ System.out.println("recover:" + storageGroupName);
}
private void recover() throws StorageGroupProcessorException {
@@ -224,6 +222,7 @@ public class StorageGroupProcessor {
getAllFiles(DirectoryManager.getInstance().getAllUnSequenceFileFolders());
recoverSeqFiles(seqTsFiles);
+
recoverUnseqFiles(unseqTsFiles);
String taskName = storageGroupName + "-" + System.currentTimeMillis();
@@ -733,6 +732,7 @@ public class StorageGroupProcessor {
deviceId, measurementId, context);
QueryDataSource dataSource = new QueryDataSource(new Path(deviceId, measurementId),
seqResources, unseqResources);
+
// used files should be added before mergeLock is unlocked, or they may be deleted by
// running merge
// is null only in tests
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 93759d7..85793f5 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -48,7 +48,6 @@ import org.apache.iotdb.db.engine.storagegroup.StorageGroupProcessor.CloseTsFile
import org.apache.iotdb.db.engine.version.VersionController;
import org.apache.iotdb.db.exception.TsFileProcessorException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.qp.constant.DatetimeUtils;
import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
@@ -178,12 +177,9 @@ public class TsFileProcessor {
}
// insert insertPlan to the work memtable
- long time = System.currentTimeMillis();
workMemTable.insertBatch(batchInsertPlan, indexes);
- PerfMonitor.add("TsFileProcessor.insertBatch", System.currentTimeMillis() - time);
if (!useNVM && IoTDBDescriptor.getInstance().getConfig().isEnableWal()) {
- time = System.currentTimeMillis();
try {
batchInsertPlan.setIndex(new HashSet<>(indexes));
getLogNode().write(batchInsertPlan);
@@ -194,7 +190,6 @@ public class TsFileProcessor {
}
return false;
}
- PerfMonitor.add("TsFileProcessor.WAL", System.currentTimeMillis() - time);
}
tsFileResource.updateStartTime(batchInsertPlan.getDeviceId(), batchInsertPlan.getMinTime());
@@ -236,6 +231,7 @@ public class TsFileProcessor {
boolean shouldFlush() {
+// return false;
return workMemTable != null
&& workMemTable.memSize() > getMemtableSizeThresholdBasedOnSeriesNum();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java b/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java
deleted file mode 100644
index ec9aa06..0000000
--- a/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.apache.iotdb.db.nvm;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class PerfMonitor {
-
- private static Map<String, Perf> perfMap = new HashMap<>();
-
- public synchronized static void add(String name, long timeLen) {
- Perf perf;
- if (!perfMap.containsKey(name)) {
- perf = new Perf(name);
- perfMap.put(name, perf);
- } else {
- perf = perfMap.get(name);
- }
-
- perf.add(timeLen);
- }
-
- public static void output() {
- perfMap.values().forEach(perf -> System.out.println(perf));
- }
-
- private static class Perf {
- String name;
- long timeLen = 0;
- int count = 0;
-
- public Perf(String name) {
- this.name = name;
- }
-
- public void add(long tl) {
- timeLen += tl;
- count++;
- }
-
- @Override
- public String toString() {
- return name + "\t" + timeLen + "\t" + count;
- }
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java
index 7d54bc7..bbb3cd5 100644
--- a/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java
@@ -8,7 +8,6 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StartupException;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.nvm.memtable.NVMPrimitiveMemTable;
import org.apache.iotdb.db.nvm.space.NVMDataSpace;
import org.apache.iotdb.db.nvm.space.NVMSpaceManager;
@@ -31,9 +30,7 @@ public class NVMMemtableRecoverPerformer {
public void init() throws StartupException {
try {
- long time = System.currentTimeMillis();
dataMap = recoverDataInNVM();
- PerfMonitor.add("NVMMemtableRecoverPerformer.init", System.currentTimeMillis() - time);
} catch (IOException e) {
throw new StartupException(e);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java b/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java
index 3f1c264..1ea4f6e 100644
--- a/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java
@@ -3,7 +3,6 @@ package org.apache.iotdb.db.nvm.rescon;
import java.util.ArrayDeque;
import java.util.EnumMap;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.nvm.exception.NVMSpaceManagerException;
import org.apache.iotdb.db.nvm.space.NVMDataSpace;
import org.apache.iotdb.db.nvm.space.NVMSpaceManager;
@@ -16,7 +15,7 @@ public class NVMPrimitiveArrayPool {
*/
private static final EnumMap<TSDataType, ArrayDeque<NVMDataSpace>> primitiveArraysMap = new EnumMap<>(TSDataType.class);
- public static final int ARRAY_SIZE = IoTDBDescriptor.getInstance().getConfig().getArraySize();
+ public static final int ARRAY_SIZE = 128;
static {
primitiveArraysMap.put(TSDataType.BOOLEAN, new ArrayDeque());
@@ -36,20 +35,19 @@ public class NVMPrimitiveArrayPool {
private NVMPrimitiveArrayPool() {}
public synchronized NVMDataSpace getPrimitiveDataListByType(TSDataType dataType, boolean isTime) {
- long time = System.currentTimeMillis();
ArrayDeque<NVMDataSpace> dataListQueue = primitiveArraysMap.computeIfAbsent(dataType, k ->new ArrayDeque<>());
NVMDataSpace nvmSpace = dataListQueue.poll();
- long size = NVMSpaceManager.getPrimitiveTypeByteSize(dataType);
if (nvmSpace == null) {
try {
+ long size = NVMSpaceManager.getPrimitiveTypeByteSize(dataType);
nvmSpace = NVMSpaceManager.getInstance().allocateDataSpace(size * ARRAY_SIZE, dataType, isTime);
} catch (NVMSpaceManagerException e) {
+ e.printStackTrace();
+ System.exit(0);
// TODO
}
}
-
- PerfMonitor.add("NVM.getDataList" + (isTime ? "Time" : "Value"), System.currentTimeMillis() - time);
return nvmSpace;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpace.java b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpace.java
index 1f4c224..5d27e95 100644
--- a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpace.java
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpace.java
@@ -1,6 +1,7 @@
package org.apache.iotdb.db.nvm.space;
import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
public class NVMSpace {
@@ -61,4 +62,8 @@ public class NVMSpace {
public void get(byte[] src) {
byteBuffer.get(src);
}
+
+ public void force() {
+ ((MappedByteBuffer) byteBuffer).force();
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java
index 7ae0f4e..b51c073 100644
--- a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java
@@ -82,6 +82,7 @@ public class NVMSpaceManager {
index, dataType, isTime);
}
curOffset += size;
+
return nvmSpace;
} catch (IOException e) {
logger.error("Fail to allocate {} nvm space at {}.", size, curOffset);
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java b/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java
index be9b697..73633ba 100644
--- a/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java
+++ b/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java
@@ -20,8 +20,6 @@ package org.apache.iotdb.db.rescon;
import java.util.ArrayDeque;
import java.util.EnumMap;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
@@ -36,7 +34,7 @@ public class PrimitiveArrayPool {
*/
private static final EnumMap<TSDataType, ArrayDeque> primitiveArraysMap = new EnumMap<>(TSDataType.class);
- public static final int ARRAY_SIZE = IoTDBDescriptor.getInstance().getConfig().getArraySize();
+ public static final int ARRAY_SIZE = 128;
static {
primitiveArraysMap.put(TSDataType.BOOLEAN, new ArrayDeque());
@@ -95,7 +93,6 @@ public class PrimitiveArrayPool {
throw new UnSupportedDataTypeException("DataType: " + dataType);
}
- PerfMonitor.add("Normal.getDataList", System.currentTimeMillis() - time);
return dataArray;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index c2611b5..2628ba1 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -32,7 +32,6 @@ import org.apache.iotdb.db.engine.flush.FlushManager;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.StatMonitor;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.nvm.recover.NVMMemtableRecoverPerformer;
import org.apache.iotdb.db.nvm.space.NVMSpaceManager;
import org.apache.iotdb.db.rescon.TVListAllocator;
@@ -86,13 +85,8 @@ public class IoTDB implements IoTDBMBean {
setUncaughtExceptionHandler();
if (IoTDBDescriptor.getInstance().getConfig().isEnableNVM()) {
- long time = System.currentTimeMillis();
NVMSpaceManager.getInstance().init();
- PerfMonitor.add("NVMSpaceManager.init", System.currentTimeMillis() - time);
-
- time = System.currentTimeMillis();
NVMMemtableRecoverPerformer.getInstance().init();
- PerfMonitor.add("NVMMemtableRecoverPerformer.init", System.currentTimeMillis() - time);
}
initMManager();
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java
index b230b8b..caa0439 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDBShutdownHook.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.service;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.utils.MemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,8 +28,6 @@ public class IoTDBShutdownHook extends Thread{
@Override
public void run() {
- PerfMonitor.output();
-
if(logger.isInfoEnabled()) {
logger.info("IoTDB exits. Jvm memory usage: {}", MemUtils.bytesCntToStr(
Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()));
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java
index 73fac89..38b8034 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java
@@ -22,7 +22,6 @@ import static org.apache.iotdb.db.rescon.PrimitiveArrayPool.ARRAY_SIZE;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -91,7 +90,6 @@ public class FloatTVList extends TVList {
@Override
public void sort() {
- long time = System.currentTimeMillis();
if (sortedTimestamps == null || sortedTimestamps.length < size) {
sortedTimestamps = (long[][]) PrimitiveArrayPool
.getInstance().getDataListsByType(TSDataType.INT64, size);
@@ -100,16 +98,11 @@ public class FloatTVList extends TVList {
sortedValues = (float[][]) PrimitiveArrayPool
.getInstance().getDataListsByType(TSDataType.FLOAT, size);
}
- PerfMonitor.add("sort-initarr", System.currentTimeMillis() - time);
- time = System.currentTimeMillis();
sort(0, size);
- PerfMonitor.add("sort-sort", System.currentTimeMillis() - time);
- time = System.currentTimeMillis();
clearSortedValue();
clearSortedTime();
- PerfMonitor.add("sort-cleararr", System.currentTimeMillis() - time);
sorted = true;
}
@@ -197,7 +190,6 @@ public class FloatTVList extends TVList {
updateMinTimeAndSorted(time);
- long startTime = System.currentTimeMillis();
while (idx < length) {
int inputRemaining = length - idx;
int arrayIdx = size / ARRAY_SIZE;
@@ -219,7 +211,6 @@ public class FloatTVList extends TVList {
checkExpansion();
}
}
- PerfMonitor.add("TVList.puts", System.currentTimeMillis() - startTime);
}
void addBatchValue(float[] batch) {
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMDoubleTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMDoubleTVList.java
index 0ebdc11..7455d18 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMDoubleTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMDoubleTVList.java
@@ -20,6 +20,7 @@ public class NVMDoubleTVList extends NVMTVList {
@Override
public void putDouble(long timestamp, double value) {
checkExpansion();
+
int arrayIndex = size / ARRAY_SIZE;
int elementIndex = size % ARRAY_SIZE;
minTime = minTime <= timestamp ? minTime : timestamp;
@@ -156,7 +157,7 @@ public class NVMDoubleTVList extends NVMTVList {
@Override
public void putDoubles(long[] time, double[] value) {
- checkExpansion();
+// checkExpansion();
int idx = 0;
int length = time.length;
@@ -164,6 +165,15 @@ public class NVMDoubleTVList extends NVMTVList {
putDouble(time[i], value[i]);
}
+// long startTime = System.currentTimeMillis();
+// for (NVMDataSpace timeSpace : timestamps) {
+// timeSpace.force();
+// }
+// for (NVMDataSpace valueSpace : values) {
+// valueSpace.force();
+// }
+// PerfMonitor.add("NVMTVList.force", System.currentTimeMillis() - startTime);
+
// updateMinTimeAndSorted(time);
//
// while (idx < length) {
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java
index 2882aea..b9814e8 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java
@@ -2,7 +2,6 @@ package org.apache.iotdb.db.utils.datastructure;
import static org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool.ARRAY_SIZE;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.nvm.space.NVMDataSpace;
import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -161,11 +160,9 @@ public class NVMFloatTVList extends NVMTVList {
int idx = 0;
int length = time.length;
- long startTime = System.currentTimeMillis();
for (int i = 0; i < length; i++) {
putFloat(time[i], value[i]);
}
- PerfMonitor.add("NVMTVList.puts", System.currentTimeMillis() - startTime);
// updateMinTimeAndSorted(time);
//
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java
index 7bf854a..eda0cb6 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java
@@ -4,7 +4,6 @@ import static org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool.ARRAY_SIZE;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool;
import org.apache.iotdb.db.nvm.space.NVMDataSpace;
import org.apache.iotdb.db.nvm.space.NVMSpaceMetadataManager;
@@ -87,6 +86,7 @@ public abstract class NVMTVList extends AbstractTVList {
protected NVMDataSpace expandValues() {
NVMDataSpace dataSpace = NVMPrimitiveArrayPool
.getInstance().getPrimitiveDataListByType(dataType, false);
+
values.add(dataSpace);
return dataSpace;
}
@@ -165,12 +165,12 @@ public abstract class NVMTVList extends AbstractTVList {
@Override
protected void checkExpansion() {
if ((size % ARRAY_SIZE) == 0) {
- long time = System.currentTimeMillis();
NVMDataSpace valueSpace = expandValues();
+
NVMDataSpace timeSpace = NVMPrimitiveArrayPool.getInstance().getPrimitiveDataListByType(TSDataType.INT64, true);
timestamps.add(timeSpace);
+
NVMSpaceMetadataManager.getInstance().registerTVSpace(timeSpace, valueSpace, sgId, deviceId, measurementId);
- PerfMonitor.add("NVMTVList.expand", System.currentTimeMillis() - time);
}
}
@@ -236,26 +236,16 @@ public abstract class NVMTVList extends AbstractTVList {
@Override
public void sort() {
- long time = System.currentTimeMillis();
initTempArrays();
- PerfMonitor.add("sort-initarr", System.currentTimeMillis() - time);
- time = System.currentTimeMillis();
copyTVToTempArrays();
- PerfMonitor.add("sort-copytoarr", System.currentTimeMillis() - time);
- time = System.currentTimeMillis();
sort(0, size);
- PerfMonitor.add("sort-sort", System.currentTimeMillis() - time);
- time = System.currentTimeMillis();
copyTVFromTempArrays();
- PerfMonitor.add("sort-copyfromarr", System.currentTimeMillis() - time);
- time = System.currentTimeMillis();
clearSortedValue();
clearSortedTime();
- PerfMonitor.add("sort-cleararr", System.currentTimeMillis() - time);
sorted = true;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index ea07fec..a189e24 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -23,7 +23,6 @@ import static org.apache.iotdb.db.rescon.PrimitiveArrayPool.ARRAY_SIZE;
import java.util.ArrayList;
import java.util.List;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -115,11 +114,9 @@ public abstract class TVList extends AbstractTVList {
@Override
protected void checkExpansion() {
if ((size % ARRAY_SIZE) == 0) {
- long time = System.currentTimeMillis();
expandValues();
timestamps.add((long[]) PrimitiveArrayPool.getInstance().getPrimitiveDataListByType(
TSDataType.INT64));
- PerfMonitor.add("TVList.expand", System.currentTimeMillis() - time);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java b/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java
index 75fb57b..b8d6cca 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.writelog.io;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.NoSuchElementException;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
/**
@@ -48,7 +47,6 @@ public class MultiFileLogReader implements ILogReader {
@Override
public boolean hasNext() throws FileNotFoundException {
- long time = System.currentTimeMillis();
if (files == null || files.length == 0) {
return false;
}
@@ -56,17 +54,14 @@ public class MultiFileLogReader implements ILogReader {
currentReader = new SingleFileLogReader(files[fileIdx++]);
}
if (currentReader.hasNext()) {
- PerfMonitor.add("LogReader.hasNext", System.currentTimeMillis() - time);
return true;
}
while (fileIdx < files.length) {
currentReader.open(files[fileIdx++]);
if (currentReader.hasNext()) {
- PerfMonitor.add("LogReader.hasNext", System.currentTimeMillis() - time);
return true;
}
}
- PerfMonitor.add("LogReader.hasNext", System.currentTimeMillis() - time);
return false;
}
@@ -75,9 +70,7 @@ public class MultiFileLogReader implements ILogReader {
if (!hasNext()) {
throw new NoSuchElementException();
}
- long time = System.currentTimeMillis();
PhysicalPlan plan = currentReader.next();
- PerfMonitor.add("LogReader.next", System.currentTimeMillis() - time);
return plan;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
index b9b5e0a..bf4f360 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
@@ -111,6 +111,7 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
public void close() {
sync();
forceWal();
+
lock.writeLock().lock();
try {
if (this.currentFileWriter != null) {
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
index 52dc646..a63140a 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
@@ -31,7 +31,6 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.version.VersionController;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.exception.storageGroup.StorageGroupProcessorException;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
@@ -94,8 +93,8 @@ public class LogReplayer {
ILogReader logReader = logNode.getLogReader();
try {
while (logReader.hasNext()) {
- long time = System.currentTimeMillis();
PhysicalPlan plan = logReader.next();
+
if (plan instanceof InsertPlan) {
replayInsert((InsertPlan) plan);
} else if (plan instanceof DeletePlan) {
@@ -105,7 +104,6 @@ public class LogReplayer {
} else if (plan instanceof BatchInsertPlan) {
replayBatchInsert((BatchInsertPlan) plan);
}
- PerfMonitor.add("LogReplayer.replay", System.currentTimeMillis() - time);
}
} catch (IOException e) {
throw new StorageGroupProcessorException("Cannot replay logs" + e.getMessage());
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index 8fbdb64..0f18bed 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -29,13 +29,11 @@ import java.util.concurrent.ExecutionException;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.engine.flush.MemTableFlushTask;
-import org.apache.iotdb.db.engine.memtable.AbstractMemTable;
import org.apache.iotdb.db.engine.memtable.IMemTable;
import org.apache.iotdb.db.engine.memtable.PrimitiveMemTable;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.version.VersionController;
import org.apache.iotdb.db.exception.storageGroup.StorageGroupProcessorException;
-import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.nvm.memtable.NVMPrimitiveMemTable;
import org.apache.iotdb.db.nvm.recover.NVMMemtableRecoverPerformer;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
@@ -146,18 +144,12 @@ public class TsFileRecoverPerformer {
}
// recover data in memory
- long time;
if (IoTDBDescriptor.getInstance().getConfig().isEnableNVM()) {
- time = System.currentTimeMillis();
reloadNVMData(restorableTsFileIOWriter);
- PerfMonitor.add("TsFileRecoverPerformer.reloadNVMData", System.currentTimeMillis() - time);
} else {
- time = System.currentTimeMillis();
redoLogs(restorableTsFileIOWriter);
- PerfMonitor.add("TsFileRecoverPerformer.redoLogs", System.currentTimeMillis() - time);
// clean logs
- time = System.currentTimeMillis();
try {
MultiFileLogNodeManager.getInstance()
.deleteNode(
@@ -165,7 +157,6 @@ public class TsFileRecoverPerformer {
} catch (IOException e) {
throw new StorageGroupProcessorException(e);
}
- PerfMonitor.add("TsFileRecoverPerformer.redoAndCleanLogs", System.currentTimeMillis() - time);
}
}
@@ -221,10 +212,7 @@ public class TsFileRecoverPerformer {
versionController,
tsFileResource, schema, recoverMemTable, acceptUnseq);
- long time = System.currentTimeMillis();
logReplayer.replayLogs();
- PerfMonitor.add("LogReplayer.replayLogs-ALL", System.currentTimeMillis() - time);
-
flushRecoverdMemtable(recoverMemTable, restorableTsFileIOWriter);
}
@@ -232,30 +220,22 @@ public class TsFileRecoverPerformer {
throws StorageGroupProcessorException {
NVMPrimitiveMemTable recoverMemTable = new NVMPrimitiveMemTable(storageGroupId);
- long time = System.currentTimeMillis();
NVMMemtableRecoverPerformer.getInstance().reconstructMemtable(recoverMemTable, tsFileResource);
- PerfMonitor.add("NVMMemtableRecoverPerformer.reconstructMemtable", System.currentTimeMillis() - time);
-
flushRecoverdMemtable(recoverMemTable, restorableTsFileIOWriter);
}
private void flushRecoverdMemtable(IMemTable recoverMemTable, RestorableTsFileIOWriter restorableTsFileIOWriter)
throws StorageGroupProcessorException {
- long time;
try {
if (!recoverMemTable.isEmpty()) {
MemTableFlushTask tableFlushTask = new MemTableFlushTask(recoverMemTable, schema,
restorableTsFileIOWriter, tsFileResource.getFile().getParentFile().getName());
- time = System.currentTimeMillis();
tableFlushTask.syncFlushMemTable();
- PerfMonitor.add("MemTableFlushTask.syncFlushMemTable", System.currentTimeMillis() - time);
}
// close file
- time = System.currentTimeMillis();
restorableTsFileIOWriter.endFile(schema);
tsFileResource.serialize();
- PerfMonitor.add("TsFileRecoverPerformer.closeFile", System.currentTimeMillis() - time);
} catch (IOException | InterruptedException | ExecutionException e) {
throw new StorageGroupProcessorException(e);
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java
index 6db444c..550be31 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBAggregationIT.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.jdbc.Config;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import java.sql.*;
@@ -77,24 +78,29 @@ public class IoTDBAggregationIT {
@Before
public void setUp() throws Exception {
+ System.out.println("set up start");
EnvironmentUtils.closeStatMonitor();
daemon = IoTDB.getInstance();
daemon.active();
EnvironmentUtils.envSetUp();
Class.forName(Config.JDBC_DRIVER_NAME);
prepareData();
+ System.out.println("set up end");
}
@After
public void tearDown() throws Exception {
+ System.out.println("tear down start");
daemon.stop();
EnvironmentUtils.cleanEnv();
+ System.out.println("tear down end");
}
//add test for part of points in page don't satisfy filter
//details in: https://issues.apache.org/jira/projects/IOTDB/issues/IOTDB-54
@Test
public void test() throws SQLException {
+ System.out.println("test start");
String[] retArray = new String[]{
"0,2",
"0,4",
@@ -156,6 +162,7 @@ public class IoTDBAggregationIT {
@Test
public void countTest() throws SQLException {
+ System.out.println("countTest start");
String[] retArray = new String[]{
"0,2001,2001,2001,2001",
"0,7500,7500,7500,7500"
@@ -202,6 +209,7 @@ public class IoTDBAggregationIT {
@Test
public void firstTest() {
+ System.out.println("firstTest start");
String[] retArray = new String[]{
"0,2000,2000,2000.0,2000",
"0,500,500,500.0,500"
@@ -250,6 +258,7 @@ public class IoTDBAggregationIT {
@Test
public void lastTest() throws SQLException {
+ System.out.println("lastTest start");
String[] retArray = new String[]{
"0,8499,8499.0",
"0,1499,1499.0",
@@ -310,6 +319,7 @@ public class IoTDBAggregationIT {
@Test
public void maxminTimeTest() throws SQLException {
+ System.out.println("maxminTimeTest start");
String[] retArray = new String[]{
"0,8499,500",
"0,2499,2000"
@@ -404,6 +414,7 @@ public class IoTDBAggregationIT {
@Test
public void avgSumTest() {
+ System.out.println("avgSumTest start");
String[] retArray = new String[]{
"0,1.4508E7,7250.374812593703",
"0,626750.0,1250.998003992016"