You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yu...@apache.org on 2022/01/29 15:44:02 UTC
[iotdb] branch kyy-2022 updated: update
This is an automated email from the ASF dual-hosted git repository.
yuyuankang pushed a commit to branch kyy-2022
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/kyy-2022 by this push:
new e9bba2a update
e9bba2a is described below
commit e9bba2a3920e600682081d5b0edb0cb5b65fd051
Author: yuyuankang <yu...@hotmail.com>
AuthorDate: Sat Jan 29 23:43:13 2022 +0800
update
---
example/session/pom.xml | 6 +-
.../main/java/org/apache/iotdb/ProcessResult.java | 37 ---------
.../org/apache/iotdb/WriteFullGameToIoTDB.java | 54 ++++++++++++
.../iotdb/mac/QueryFullGameExperimentMAC.java | 47 +++++++++++
.../iotdb/moc/QueryFullGameExperimentMOC.java | 56 +++++++++++++
.../main/java/org/apache/iotdb/db/IOMonitor.java | 96 +++++++++++++++++-----
.../apache/iotdb/db/engine/cache/ChunkCache.java | 6 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 2 +
.../dataset/groupby/LocalGroupByExecutor.java | 2 -
.../org/apache/iotdb/db/service/TSServiceImpl.java | 7 +-
.../apache/iotdb/db/tools/TsFileSketchTool.java | 3 +-
11 files changed, 248 insertions(+), 68 deletions(-)
diff --git a/example/session/pom.xml b/example/session/pom.xml
index 3521841..c5fcb73 100644
--- a/example/session/pom.xml
+++ b/example/session/pom.xml
@@ -26,8 +26,8 @@
<version>0.12.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>client-example</artifactId>
- <name>client-example</name>
+ <artifactId>QueryFullGameExperiment</artifactId>
+ <name>QueryFullGameExperiment</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.7</maven.compiler.source>
@@ -55,7 +55,7 @@
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.iotdb.QueryExperiment</mainClass>
+ <mainClass>org.apache.iotdb.mac.QueryFullGameExperimentMAC</mainClass>
</transformer>
</transformers>
</configuration>
diff --git a/example/session/src/main/java/org/apache/iotdb/ProcessResult.java b/example/session/src/main/java/org/apache/iotdb/ProcessResult.java
deleted file mode 100644
index bd0d3ca..0000000
--- a/example/session/src/main/java/org/apache/iotdb/ProcessResult.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package org.apache.iotdb;
-
-import java.io.*;
-
-public class ProcessResult {
-
- public static final String BASE_PATH = "/home/kyy/Documents/kdd";
-
- public static void main(String[] args) throws IOException {
-
- String inFilePath = BASE_PATH + File.separator + "result.txt";
- String outFilePath = BASE_PATH + File.separator + "out.txt";
-
- BufferedReader reader = new BufferedReader(new FileReader(inFilePath));
- FileWriter writer = new FileWriter(outFilePath);
- String readLine = null;
- long metaTime = 0;
- long dataTime = 0;
- long totalTime = 0;
- int counter = 0;
- while ((readLine = reader.readLine()) != null) {
- if (readLine.startsWith("select")) {
- String[] values = readLine.split("\t");
- metaTime += Long.parseLong(values[1].split(": ")[1]);
- dataTime += Long.parseLong(values[3].split(": ")[1]);
- totalTime += Long.parseLong(values[5].split(": ")[1]);
- counter++;
- writer.write(readLine + "\n");
- }
- }
- writer.write("avg_meta: " + (double) metaTime / counter
- + "\t avg_data: " + (double) dataTime / counter
- + "\t avg_total: " + (double) totalTime / counter);
- reader.close();
- writer.close();
- }
-}
diff --git a/example/session/src/main/java/org/apache/iotdb/WriteFullGameToIoTDB.java b/example/session/src/main/java/org/apache/iotdb/WriteFullGameToIoTDB.java
new file mode 100644
index 0000000..263abe9
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/WriteFullGameToIoTDB.java
@@ -0,0 +1,54 @@
+package org.apache.iotdb;
+
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Collections;
+
+public class WriteFullGameToIoTDB {
+
+ public static void main(String[] args)
+ throws IOException, IoTDBConnectionException, StatementExecutionException {
+
+ // String path = args[0];
+ String path = "/home/kyy/Documents/kdd/data/full-game.csv";
+ String measurements = "s6";
+
+ String device = "root.game";
+
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open(false);
+
+ long minTime = -1;
+
+ File f = new File(path);
+ String line = null;
+ BufferedReader reader = new BufferedReader(new FileReader(f));
+ while ((line = reader.readLine()) != null) {
+ String[] split = line.split(",");
+ long timestamp = Long.valueOf(split[1]);
+ if (minTime == -1) {
+ minTime = timestamp;
+ timestamp = 0;
+ } else {
+ timestamp = timestamp - minTime;
+ }
+ timestamp = (long) (timestamp / 1000);
+ long value = Long.valueOf(split[5]);
+ session.insertRecord(
+ device,
+ timestamp,
+ Collections.singletonList(measurements),
+ Collections.singletonList(TSDataType.INT64),
+ value);
+ }
+ session.executeNonQueryStatement("flush");
+ session.close();
+ }
+}
diff --git a/example/session/src/main/java/org/apache/iotdb/mac/QueryFullGameExperimentMAC.java b/example/session/src/main/java/org/apache/iotdb/mac/QueryFullGameExperimentMAC.java
new file mode 100644
index 0000000..f070d21
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/mac/QueryFullGameExperimentMAC.java
@@ -0,0 +1,47 @@
+package org.apache.iotdb.mac;
+
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.session.SessionDataSet;
+import org.apache.iotdb.tsfile.read.common.RowRecord;
+
+import org.apache.thrift.TException;
+
+public class QueryFullGameExperimentMAC {
+
+ private static final String queryFormat =
+ "select %s " + "from %s " + "where time >= %d and time < %d";
+
+ public static Session session;
+
+ public static void main(String[] args)
+ throws IoTDBConnectionException, StatementExecutionException, TException {
+ int intervalNum = Integer.parseInt(args[0]);
+ String measurement = "s6";
+ String device = "root.game";
+ session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open(false);
+ SessionDataSet dataSet;
+ long minTime = 0L;
+ long maxTime = 4264605928301L;
+ long interval = (long) Math.ceil((double) (maxTime - minTime) / intervalNum);
+ maxTime = minTime + interval * intervalNum;
+
+ for (int i = 0; i < intervalNum; i++) {
+ long start = i * interval;
+ long end = (i + 1) * interval;
+ String sql = String.format(queryFormat, measurement, device, start, end);
+ dataSet = session.executeQueryStatement(sql);
+ while (dataSet.hasNext()) {
+ RowRecord r = dataSet.next();
+ }
+ }
+ session.executeNonQueryStatement("clear cache");
+ dataSet = session.executeFinish();
+ String info = dataSet.getFinishResult();
+ System.out.println(info);
+ dataSet.closeOperationHandle();
+ session.close();
+ }
+}
diff --git a/example/session/src/main/java/org/apache/iotdb/moc/QueryFullGameExperimentMOC.java b/example/session/src/main/java/org/apache/iotdb/moc/QueryFullGameExperimentMOC.java
new file mode 100644
index 0000000..4926199
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/moc/QueryFullGameExperimentMOC.java
@@ -0,0 +1,56 @@
+package org.apache.iotdb.moc;
+
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.session.SessionDataSet;
+import org.apache.iotdb.tsfile.read.common.RowRecord;
+
+import org.apache.thrift.TException;
+
+public class QueryFullGameExperimentMOC {
+
+ private static final String queryFormat =
+ "select min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s), max_value(%s) "
+ + "from %s "
+ + "group by ([%d, %d), %dns)";
+
+ public static Session session;
+
+ public static void main(String[] args)
+ throws IoTDBConnectionException, StatementExecutionException, TException {
+ int intervalNum = Integer.parseInt(args[0]);
+ String measurement = "s6";
+ String device = "root.game";
+ session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open(false);
+ SessionDataSet dataSet;
+ long minTime = 0L;
+ long maxTime = 4264605928301L;
+ long interval = (long) Math.ceil((double) (maxTime - minTime) / intervalNum);
+ maxTime = minTime + interval * intervalNum;
+ String sql =
+ String.format(
+ queryFormat,
+ measurement,
+ measurement,
+ measurement,
+ measurement,
+ measurement,
+ measurement,
+ device,
+ minTime,
+ maxTime,
+ interval);
+ dataSet = session.executeQueryStatement(sql);
+ while (dataSet.hasNext()) {
+ RowRecord r = dataSet.next();
+ }
+ session.executeNonQueryStatement("clear cache");
+ dataSet = session.executeFinish();
+ String info = dataSet.getFinishResult();
+ System.out.println(info);
+ dataSet.closeOperationHandle();
+ session.close();
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/IOMonitor.java b/server/src/main/java/org/apache/iotdb/db/IOMonitor.java
index d9627d8..698aa10 100644
--- a/server/src/main/java/org/apache/iotdb/db/IOMonitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/IOMonitor.java
@@ -28,19 +28,36 @@ public class IOMonitor {
public static int metaIONum;
public static long dataIOTime;
public static int dataIONum;
+ public static long readMemChunkTime;
+ public static int readMemChunkNum;
+
public static long totalTime;
public static String sql;
public static List<Long> metaIOTimes = new ArrayList<>();
public static List<Long> dataIOTimes = new ArrayList<>();
+ public static List<Long> readMemChunkTimes = new ArrayList<>();
public static List<Integer> metaIONums = new ArrayList<>();
public static List<Integer> dataIONums = new ArrayList<>();
+ public static List<Integer> readMemChunkNums = new ArrayList<>();
public static List<String> sqls = new ArrayList<>();
public static List<Long> totalTimes = new ArrayList<>();
public static boolean isSet = false;
+ public static void incReadMemChunkTime(long v) {
+ readMemChunkTime += v;
+ readMemChunkNum++;
+ }
+
+ public static void resetReadMemChunkTime() {
+ readMemChunkNums.add(readMemChunkNum);
+ readMemChunkTimes.add(readMemChunkTime);
+ readMemChunkTime = 0;
+ readMemChunkNum = 0;
+ }
+
public static void incMeta(long v) {
metaIOTime += v;
metaIONum++;
@@ -58,6 +75,10 @@ public class IOMonitor {
dataIONum++;
}
+ public static void incTotalTime(long val) {
+ totalTime += val;
+ }
+
private static void resetDataIOTime() {
dataIOTimes.add(dataIOTime);
dataIONums.add(dataIONum);
@@ -65,6 +86,11 @@ public class IOMonitor {
dataIONum = 0;
}
+ public static void resetTotalTime() {
+ totalTimes.add(totalTime);
+ totalTime = 0;
+ }
+
public static void setSQL(String v) {
if (!isSet) {
clear();
@@ -79,16 +105,30 @@ public class IOMonitor {
public static void reset() {
resetMeta();
resetDataIOTime();
+ resetTotalTime();
+ resetReadMemChunkTime();
sqls.add(sql);
sql = null;
}
- private static double getAvg(List<Long> vals) {
+ // private static double getAvg(List<Long> vals) {
+ // return (double) getSum(vals) / vals.size();
+ // }
+
+ private static long getSumLong(List<Long> vals) {
long sum = 0;
- for (long v : vals) {
+ for (Object v : vals) {
+ sum += (long) v;
+ }
+ return sum;
+ }
+
+ private static long getSumInteger(List<Integer> vals) {
+ Integer sum = 0;
+ for (Integer v : vals) {
sum += v;
}
- return (double) sum / vals.size();
+ return sum.longValue();
}
public static void clear() {
@@ -96,8 +136,10 @@ public class IOMonitor {
metaIOTime = 0L;
dataIOTime = 0L;
totalTime = 0L;
+ readMemChunkTime = 0L;
dataIONum = 0;
metaIONum = 0;
+ readMemChunkNum = 0;
sql = null;
metaIOTimes.clear();
@@ -114,32 +156,46 @@ public class IOMonitor {
}
public static String print() {
- reset();
+ if (sql != null) {
+ reset();
+ }
String ret = "";
for (int i = 0; i < sqls.size(); i++) {
ret +=
sqls.get(i)
- + "\t meta IO: "
+ + "\t meta IO: \t"
+ metaIOTimes.get(i)
- + "\t meta num: "
+ + "\t meta num: \t"
+ metaIONums.get(i)
- + "\t data IO: "
+ + "\t data IO: \t"
+ dataIOTimes.get(i)
- + "\t data num: "
+ + "\t data num: \t"
+ dataIONums.get(i)
- + "\t total: "
- + totalTimes.get(i);
- // + "\n";
+ + "\t readMemChunk IO: \t"
+ + readMemChunkTimes.get(i)
+ + "\t readMemChunk num: \t"
+ + readMemChunkNums.get(i)
+ + "\t total: \t"
+ + totalTimes.get(i)
+ + "\n";
}
- // ret +=
- // "avg meta IO: "
- // + getAvg(metaIOTimes)
- // + ", avg data IO: "
- // + getAvg(dataIOTimes)
- // + ", avg total time: "
- // + getAvg(totalTimes)
- // + ", isSet: "
- // + isSet;
+ ret +=
+ "sum meta IO: \t"
+ + getSumLong(metaIOTimes)
+ + "\t sum meta nums: \t"
+ + getSumInteger(metaIONums)
+ + "\t sum data IO: \t"
+ + getSumLong(dataIOTimes)
+ + "\t sum data num: \t"
+ + getSumInteger(dataIONums)
+ + "\t sum readMemChunkTime: \t"
+ + getSumLong(readMemChunkTimes)
+ + "\t sum readMemChunkNum: \t"
+ + getSumInteger(readMemChunkNums)
+ + "\t avg total time: \t"
+ + getSumLong(totalTimes)
+ + "\t isSet: \t"
+ + isSet;
return ret;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 161a96d..02a1f5a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.engine.cache;
+import org.apache.iotdb.db.IOMonitor;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.query.control.FileReaderManager;
@@ -77,7 +78,10 @@ public class ChunkCache {
TsFileSequenceReader reader =
FileReaderManager.getInstance()
.get(chunkMetadata.getFilePath(), chunkMetadata.isClosed());
- return reader.readMemChunk(chunkMetadata);
+ long start = System.nanoTime();
+ Chunk ret = reader.readMemChunk(chunkMetadata);
+ IOMonitor.incReadMemChunkTime(System.nanoTime() - start);
+ return ret;
} catch (IOException e) {
logger.error("Something wrong happened in reading {}", chunkMetadata, e);
throw e;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 424915f..5b91858 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -143,7 +143,9 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
try {
for (Entry<PartialPath, GroupByExecutor> pathToExecutorEntry : pathExecutors.entrySet()) {
GroupByExecutor executor = pathToExecutorEntry.getValue();
+ // long start = System.nanoTime();
List<AggregateResult> aggregations = executor.calcResult(curStartTime, curEndTime);
+ // IOMonitor.incTotalTime(System.nanoTime() - start);
for (int i = 0; i < aggregations.size(); i++) {
int resultIndex = resultIndexes.get(pathToExecutorEntry.getKey()).get(i);
fields[resultIndex] = aggregations.get(i);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java
index b8cec3d..9b5f970 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutor.java
@@ -186,7 +186,6 @@ public class LocalGroupByExecutor implements GroupByExecutor {
@Override
public List<AggregateResult> calcResult(long curStartTime, long curEndTime)
throws IOException, QueryProcessException {
-
// clear result cache
for (AggregateResult result : results) {
result.reset();
@@ -226,7 +225,6 @@ public class LocalGroupByExecutor implements GroupByExecutor {
return results;
}
}
-
return results;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 2a5be92..e19063f 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -715,8 +715,8 @@ public class TSServiceImpl implements TSIService.Iface {
// start record execution time
IOMonitor.setSQL(statement);
- long start = System.nanoTime();
queryCount.incrementAndGet();
+ long start = System.nanoTime();
AUDIT_LOGGER.debug(
"Session {} execute Query: {}", sessionManager.getCurrSessionId(), statement);
long startTime = System.currentTimeMillis();
@@ -826,9 +826,8 @@ public class TSServiceImpl implements TSIService.Iface {
queryTimeManager.unRegisterQuery(queryId);
}
- // finish recording execution time
- long duration = System.nanoTime() - start;
- IOMonitor.totalTimes.add(duration);
+ IOMonitor.incTotalTime(System.nanoTime() - start);
+ IOMonitor.reset();
return resp;
} catch (Exception e) {
releaseQueryResourceNoExceptions(queryId);
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index 506221b..b833b01 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -245,7 +245,8 @@ public class TsFileSketchTool {
}
private static Pair<String, String> checkArgs(String[] args) {
- String filename = "/home/kyy/Documents/kdd/iotdb-server-0.12.4/data/data/sequence/root.kobelco/0/0/1643270757735-1-0-0.tsfile";
+ String filename =
+ "/home/kyy/Documents/kdd/iotdb-server-0.12.4/data/data/sequence/root.kobelco/0/0/1643270757735-1-0-0.tsfile";
String outFile = "TsFile_sketch_view.txt";
if (args.length == 1) {
filename = args[0];