You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ea...@apache.org on 2020/04/15 02:38:46 UTC
[incubator-iotdb] 03/03: add perf monitor
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch nvmlogging
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 35bf4b5d67b3a0b70dc64694a3f65924dc5af38a
Author: mdf369 <95...@qq.com>
AuthorDate: Wed Apr 15 10:38:15 2020 +0800
add perf monitor
---
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 ++
.../java/org/apache/iotdb/db/nvm/PerfMonitor.java | 2 +-
.../nvm/recover/NVMMemtableRecoverPerformer.java | 3 ++
.../iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java | 3 +-
.../apache/iotdb/db/rescon/PrimitiveArrayPool.java | 3 +-
.../iotdb/db/utils/datastructure/FloatTVList.java | 5 +--
.../db/utils/datastructure/NVMBinaryTVList.java | 44 ++++++++++++++++++----
.../db/utils/datastructure/NVMFloatTVList.java | 3 ++
.../iotdb/db/utils/datastructure/NVMTVList.java | 7 +---
.../iotdb/db/utils/datastructure/TVList.java | 3 ++
.../iotdb/db/writelog/io/MultiFileLogReader.java | 10 ++++-
.../iotdb/db/writelog/recover/LogReplayer.java | 3 ++
.../writelog/recover/TsFileRecoverPerformer.java | 2 +-
13 files changed, 70 insertions(+), 21 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 7069c2f..d677c35 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -355,6 +355,9 @@ public class IoTDBDescriptor {
Boolean.toString(conf.isEnableNVM()))));
conf.setNvmDir(properties.getProperty("nvm_dir", conf.getNvmDir()));
+
+ conf.setArraySize(Integer.parseInt(properties.getProperty("array_size",
+ Integer.toString(conf.getArraySize()))));
}
private void loadAutoCreateSchemaProps(Properties properties){
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java b/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java
index 6784e31..ec9aa06 100644
--- a/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/PerfMonitor.java
@@ -39,7 +39,7 @@ public class PerfMonitor {
@Override
public String toString() {
- return name + ":\t\t\t" + timeLen + "\t\t\t" + count;
+ return name + "\t" + timeLen + "\t" + count;
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java
index bbb3cd5..7d54bc7 100644
--- a/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/recover/NVMMemtableRecoverPerformer.java
@@ -8,6 +8,7 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StartupException;
+import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.nvm.memtable.NVMPrimitiveMemTable;
import org.apache.iotdb.db.nvm.space.NVMDataSpace;
import org.apache.iotdb.db.nvm.space.NVMSpaceManager;
@@ -30,7 +31,9 @@ public class NVMMemtableRecoverPerformer {
public void init() throws StartupException {
try {
+ long time = System.currentTimeMillis();
dataMap = recoverDataInNVM();
+ PerfMonitor.add("NVMMemtableRecoverPerformer.init", System.currentTimeMillis() - time);
} catch (IOException e) {
throw new StartupException(e);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java b/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java
index 019bb27..3f1c264 100644
--- a/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java
@@ -2,6 +2,7 @@ package org.apache.iotdb.db.nvm.rescon;
import java.util.ArrayDeque;
import java.util.EnumMap;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.nvm.exception.NVMSpaceManagerException;
import org.apache.iotdb.db.nvm.space.NVMDataSpace;
@@ -15,7 +16,7 @@ public class NVMPrimitiveArrayPool {
*/
private static final EnumMap<TSDataType, ArrayDeque<NVMDataSpace>> primitiveArraysMap = new EnumMap<>(TSDataType.class);
- public static final int ARRAY_SIZE = 128;
+ public static final int ARRAY_SIZE = IoTDBDescriptor.getInstance().getConfig().getArraySize();
static {
primitiveArraysMap.put(TSDataType.BOOLEAN, new ArrayDeque());
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java b/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java
index dc8b72e..be9b697 100644
--- a/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java
+++ b/server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayPool.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.rescon;
import java.util.ArrayDeque;
import java.util.EnumMap;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -35,7 +36,7 @@ public class PrimitiveArrayPool {
*/
private static final EnumMap<TSDataType, ArrayDeque> primitiveArraysMap = new EnumMap<>(TSDataType.class);
- public static final int ARRAY_SIZE = 128;
+ public static final int ARRAY_SIZE = IoTDBDescriptor.getInstance().getConfig().getArraySize();
static {
primitiveArraysMap.put(TSDataType.BOOLEAN, new ArrayDeque());
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java
index 578fc13..73fac89 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java
@@ -100,18 +100,15 @@ public class FloatTVList extends TVList {
sortedValues = (float[][]) PrimitiveArrayPool
.getInstance().getDataListsByType(TSDataType.FLOAT, size);
}
- System.out.println("init arr:" + (System.currentTimeMillis() - time));
PerfMonitor.add("sort-initarr", System.currentTimeMillis() - time);
time = System.currentTimeMillis();
sort(0, size);
- System.out.println("sort:" + (System.currentTimeMillis() - time));
PerfMonitor.add("sort-sort", System.currentTimeMillis() - time);
time = System.currentTimeMillis();
clearSortedValue();
clearSortedTime();
- System.out.println("clear arr:" + (System.currentTimeMillis() - time));
PerfMonitor.add("sort-cleararr", System.currentTimeMillis() - time);
sorted = true;
@@ -200,6 +197,7 @@ public class FloatTVList extends TVList {
updateMinTimeAndSorted(time);
+ long startTime = System.currentTimeMillis();
while (idx < length) {
int inputRemaining = length - idx;
int arrayIdx = size / ARRAY_SIZE;
@@ -221,6 +219,7 @@ public class FloatTVList extends TVList {
checkExpansion();
}
}
+ PerfMonitor.add("TVList.puts", System.currentTimeMillis() - startTime);
}
void addBatchValue(float[] batch) {
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java
index a5b270e..0b8ce51 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java
@@ -2,11 +2,16 @@ package org.apache.iotdb.db.utils.datastructure;
import static org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool.ARRAY_SIZE;
+import org.apache.iotdb.db.exception.StartupException;
+import org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool;
import org.apache.iotdb.db.nvm.space.NVMDataSpace;
+import org.apache.iotdb.db.nvm.space.NVMSpaceManager;
+import org.apache.iotdb.db.nvm.space.NVMSpaceMetadataManager;
import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
+// TODO how to organize data
public class NVMBinaryTVList extends NVMTVList {
// TODO
@@ -21,8 +26,8 @@ public class NVMBinaryTVList extends NVMTVList {
@Override
public void putBinary(long timestamp, Binary value) {
checkExpansion();
- int arrayIndex = size / ARRAY_SIZE;
- int elementIndex = size % ARRAY_SIZE;
+ int arrayIndex = size;
+ int elementIndex = 0;
minTime = minTime <= timestamp ? minTime : timestamp;
timestamps.get(arrayIndex).setData(elementIndex, timestamp);
values.get(arrayIndex).setData(elementIndex, value);
@@ -33,12 +38,22 @@ public class NVMBinaryTVList extends NVMTVList {
}
@Override
+ protected void checkExpansion() {
+ NVMDataSpace valueSpace = expandValues();
+ NVMDataSpace timeSpace = NVMPrimitiveArrayPool
+ .getInstance().getPrimitiveDataListByType(TSDataType.INT64, true);
+ timestamps.add(timeSpace);
+ NVMSpaceMetadataManager
+ .getInstance().registerTVSpace(timeSpace, valueSpace, sgId, deviceId, measurementId);
+ }
+
+ @Override
public Binary getBinary(int index) {
if (index >= size) {
throw new ArrayIndexOutOfBoundsException(index);
}
- int arrayIndex = index / ARRAY_SIZE;
- int elementIndex = index % ARRAY_SIZE;
+ int arrayIndex = index;
+ int elementIndex = 0;
return (Binary) values.get(arrayIndex).getData(elementIndex);
}
@@ -73,8 +88,8 @@ public class NVMBinaryTVList extends NVMTVList {
int arrayIndex = 0;
int elementIndex = 0;
for (int i = 0; i < size; i++) {
- long time = (long) timestamps.get(arrayIndex).getData(elementIndex);
- Binary value = (Binary) values.get(arrayIndex).getData(elementIndex);
+ long time = (long) timestamps.get(i).getData(0);
+ Binary value = (Binary) values.get(i).getData(0);
tempTimestampsForSort[arrayIndex][elementIndex] = time;
tempValuesForSort[arrayIndex][elementIndex] = value;
@@ -94,8 +109,8 @@ public class NVMBinaryTVList extends NVMTVList {
long time = tempTimestampsForSort[arrayIndex][elementIndex];
Binary value = tempValuesForSort[arrayIndex][elementIndex];
- timestamps.get(arrayIndex).setData(elementIndex, time);
- values.get(arrayIndex).setData(elementIndex, value);
+ timestamps.get(i).setData(0, time);
+ values.get(i).setData(0, value);
elementIndex++;
if (elementIndex == ARRAY_SIZE) {
@@ -189,4 +204,17 @@ public class NVMBinaryTVList extends NVMTVList {
// }
// }
}
+
+ public static void main(String[] args) throws StartupException {
+ NVMSpaceManager.getInstance().init();
+
+ NVMBinaryTVList tvList = new NVMBinaryTVList("sg", "d0", "s0");
+ int size = 5000;
+ for (int i = 0; i < size; i++) {
+ String v = String.valueOf(size - i);
+ tvList.putBinary(i, Binary.valueOf(v));
+ }
+
+ tvList.sort();
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java
index b9814e8..2882aea 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java
@@ -2,6 +2,7 @@ package org.apache.iotdb.db.utils.datastructure;
import static org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool.ARRAY_SIZE;
+import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.nvm.space.NVMDataSpace;
import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -160,9 +161,11 @@ public class NVMFloatTVList extends NVMTVList {
int idx = 0;
int length = time.length;
+ long startTime = System.currentTimeMillis();
for (int i = 0; i < length; i++) {
putFloat(time[i], value[i]);
}
+ PerfMonitor.add("NVMTVList.puts", System.currentTimeMillis() - startTime);
// updateMinTimeAndSorted(time);
//
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java
index 613f860..7bf854a 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java
@@ -165,10 +165,12 @@ public abstract class NVMTVList extends AbstractTVList {
@Override
protected void checkExpansion() {
if ((size % ARRAY_SIZE) == 0) {
+ long time = System.currentTimeMillis();
NVMDataSpace valueSpace = expandValues();
NVMDataSpace timeSpace = NVMPrimitiveArrayPool.getInstance().getPrimitiveDataListByType(TSDataType.INT64, true);
timestamps.add(timeSpace);
NVMSpaceMetadataManager.getInstance().registerTVSpace(timeSpace, valueSpace, sgId, deviceId, measurementId);
+ PerfMonitor.add("NVMTVList.expand", System.currentTimeMillis() - time);
}
}
@@ -236,28 +238,23 @@ public abstract class NVMTVList extends AbstractTVList {
public void sort() {
long time = System.currentTimeMillis();
initTempArrays();
-// System.out.println("init arr:" + (System.currentTimeMillis() - time));
PerfMonitor.add("sort-initarr", System.currentTimeMillis() - time);
time = System.currentTimeMillis();
copyTVToTempArrays();
-// System.out.println("copy to arr:" + (System.currentTimeMillis() - time));
PerfMonitor.add("sort-copytoarr", System.currentTimeMillis() - time);
time = System.currentTimeMillis();
sort(0, size);
-// System.out.println("sort:" + (System.currentTimeMillis() - time));
PerfMonitor.add("sort-sort", System.currentTimeMillis() - time);
time = System.currentTimeMillis();
copyTVFromTempArrays();
-// System.out.println("copy from arr:" + (System.currentTimeMillis() - time));
PerfMonitor.add("sort-copyfromarr", System.currentTimeMillis() - time);
time = System.currentTimeMillis();
clearSortedValue();
clearSortedTime();
-// System.out.println("clear arr:" + (System.currentTimeMillis() - time));
PerfMonitor.add("sort-cleararr", System.currentTimeMillis() - time);
sorted = true;
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index a189e24..ea07fec 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -23,6 +23,7 @@ import static org.apache.iotdb.db.rescon.PrimitiveArrayPool.ARRAY_SIZE;
import java.util.ArrayList;
import java.util.List;
+import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -114,9 +115,11 @@ public abstract class TVList extends AbstractTVList {
@Override
protected void checkExpansion() {
if ((size % ARRAY_SIZE) == 0) {
+ long time = System.currentTimeMillis();
expandValues();
timestamps.add((long[]) PrimitiveArrayPool.getInstance().getPrimitiveDataListByType(
TSDataType.INT64));
+ PerfMonitor.add("TVList.expand", System.currentTimeMillis() - time);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java b/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java
index cf7b25f..75fb57b 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/io/MultiFileLogReader.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.writelog.io;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.NoSuchElementException;
+import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
/**
@@ -47,6 +48,7 @@ public class MultiFileLogReader implements ILogReader {
@Override
public boolean hasNext() throws FileNotFoundException {
+ long time = System.currentTimeMillis();
if (files == null || files.length == 0) {
return false;
}
@@ -54,14 +56,17 @@ public class MultiFileLogReader implements ILogReader {
currentReader = new SingleFileLogReader(files[fileIdx++]);
}
if (currentReader.hasNext()) {
+ PerfMonitor.add("LogReader.hasNext", System.currentTimeMillis() - time);
return true;
}
while (fileIdx < files.length) {
currentReader.open(files[fileIdx++]);
if (currentReader.hasNext()) {
+ PerfMonitor.add("LogReader.hasNext", System.currentTimeMillis() - time);
return true;
}
}
+ PerfMonitor.add("LogReader.hasNext", System.currentTimeMillis() - time);
return false;
}
@@ -70,6 +75,9 @@ public class MultiFileLogReader implements ILogReader {
if (!hasNext()) {
throw new NoSuchElementException();
}
- return currentReader.next();
+ long time = System.currentTimeMillis();
+ PhysicalPlan plan = currentReader.next();
+ PerfMonitor.add("LogReader.next", System.currentTimeMillis() - time);
+ return plan;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
index 9e8d8ec..52dc646 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/LogReplayer.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.version.VersionController;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.exception.storageGroup.StorageGroupProcessorException;
+import org.apache.iotdb.db.nvm.PerfMonitor;
import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
@@ -93,6 +94,7 @@ public class LogReplayer {
ILogReader logReader = logNode.getLogReader();
try {
while (logReader.hasNext()) {
+ long time = System.currentTimeMillis();
PhysicalPlan plan = logReader.next();
if (plan instanceof InsertPlan) {
replayInsert((InsertPlan) plan);
@@ -103,6 +105,7 @@ public class LogReplayer {
} else if (plan instanceof BatchInsertPlan) {
replayBatchInsert((BatchInsertPlan) plan);
}
+ PerfMonitor.add("LogReplayer.replay", System.currentTimeMillis() - time);
}
} catch (IOException e) {
throw new StorageGroupProcessorException("Cannot replay logs" + e.getMessage());
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
index 6e56541..8fbdb64 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/recover/TsFileRecoverPerformer.java
@@ -223,7 +223,7 @@ public class TsFileRecoverPerformer {
long time = System.currentTimeMillis();
logReplayer.replayLogs();
- PerfMonitor.add("LogReplayer.replayLogs", System.currentTimeMillis() - time);
+ PerfMonitor.add("LogReplayer.replayLogs-ALL", System.currentTimeMillis() - time);
flushRecoverdMemtable(recoverMemTable, restorableTsFileIOWriter);
}