You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2022/02/14 04:06:55 UTC
[iotdb] 21/32: for exp
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d7252fdfddfbd3d31683219dbe57ae91b63b8511
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Feb 7 11:28:50 2022 +0800
for exp
---
example/session/pom.xml | 6 +-
.../org/apache/iotdb/queryExp/QueryFullGame.java | 31 +++--
.../apache/iotdb/queryExp/QueryFullGame_old.java | 104 +++++++++++++++
.../{QueryFullGameVaryTqe.java => QueryMF03.java} | 45 ++++---
.../{QueryFullGameVaryW.java => QueryRecTime.java} | 44 +++---
.../iotdb/queryExp/QuerySyntheticData1VaryW.java | 109 ---------------
...ntheticData1VaryTqe.java => QueryTianyuan.java} | 59 +++++----
...disorder-0delete-varyW-compare-loadDataOnly.png | Bin 51940 -> 0 bytes
...1-0disorder-0delete-varyW-compare-totalOnly.png | Bin 45421 -> 0 bytes
.../synData1-0disorder-0delete-varyW-compare.png | Bin 65501 -> 0 bytes
.../synData1-0disorder-0delete-varyW-cpv.png | Bin 48624 -> 0 bytes
.../synData1-0disorder-0delete-varyW-mac.png | Bin 38037 -> 0 bytes
.../synData1-0disorder-0delete-varyW-moc.png | Bin 49967 -> 0 bytes
.../iotdb/writeData/ExtractDebs2012Data.java | 71 +++++-----
.../iotdb/writeData/ExtractFullGameData.java | 3 +-
...ctDebs2012Data.java => ExtractRecTimeData.java} | 53 ++------
.../{ExtractDebug.java => ExtractTianyuan.java} | 25 ++--
.../apache/iotdb/writeData/TestFullGameTmp.java | 32 -----
.../org/apache/iotdb/writeData/WriteFullGame.java | 87 +++++++-----
.../writeData/WriteFullGameDisorderedToIoTDB.java | 104 +++++++--------
.../WriteFullGameDisorderedWithDeletes.java | 147 +++++++++++----------
.../iotdb/writeData/WriteFullGameToIoTDB.java | 108 +++++++--------
.../{WriteFullGame.java => WriteFullGame_old.java} | 2 +-
.../java/org/apache/iotdb/writeData/WriteMF03.java | 116 ++++++++++++++++
.../org/apache/iotdb/writeData/WriteRecTime.java | 114 ++++++++++++++++
.../iotdb/writeData/WriteSyntheticData1.java | 92 ++++++-------
.../writeData/WriteSyntheticDataWithDeletes.java | 127 +++++++++---------
.../org/apache/iotdb/writeData/WriteTianyuan.java | 114 ++++++++++++++++
28 files changed, 968 insertions(+), 625 deletions(-)
diff --git a/example/session/pom.xml b/example/session/pom.xml
index b709546..8ea7d26 100644
--- a/example/session/pom.xml
+++ b/example/session/pom.xml
@@ -26,8 +26,8 @@
<version>0.12.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>WriteFullGame</artifactId>
- <name>WriteFullGame</name>
+ <artifactId>QueryRecTime</artifactId>
+ <name>QueryRecTime</name>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.7</maven.compiler.source>
@@ -61,7 +61,7 @@
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.iotdb.writeData.WriteFullGame</mainClass>
+ <mainClass>org.apache.iotdb.queryExp.QueryRecTime</mainClass>
</transformer>
</transformers>
</configuration>
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGame.java b/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGame.java
index c2411dd..a8fb254 100644
--- a/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGame.java
+++ b/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGame.java
@@ -14,11 +14,11 @@ public class QueryFullGame {
// => Don't change the sequence of the above six aggregates!
// * (2) group by ([tqs,tqe),IntervalLength) => Make sure (tqe-tqs) is divisible by
// IntervalLength!
- // * (3) NOTE the time unit of interval. Update for different datasets!!!!!!!!!!!
+ // * (3) NOTE the time unit of interval. Update for different datasets!
private static final String queryFormat =
"select min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s), max_value(%s) "
+ "from %s "
- + "group by ([%d, %d), %dns)";
+ + "group by ([%d, %d), %dns)"; // note this "ns" time precision
private static final String queryFormat_UDF =
"select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and time<%4$d";
@@ -30,10 +30,12 @@ public class QueryFullGame {
// fixed time series path
String measurement = "s6";
String device = "root.game";
- // fixed tqs
- long minTime = 0L;
- // 实验自变量1：tqe
- long maxTime = Long.parseLong(args[0]);
+ // used to bound tqs random position
+ long dataMinTime = 0;
+ long dataMaxTime = 617426057627L; // 617426057626+1
+
+ // 实验自变量1：[tqs,tqe) range length, i.e., tqe-tqs
+ long range = Long.parseLong(args[0]);
// 实验自变量2：w数量
int intervalNum = Integer.parseInt(args[1]);
// 实验自变量3：方法
@@ -55,13 +57,24 @@ public class QueryFullGame {
}
}
+ long minTime;
+ long maxTime;
+ long interval;
+ if (range >= (dataMaxTime - dataMinTime)) {
+ minTime = dataMinTime;
+ interval = (long) Math.ceil((double) (dataMaxTime - dataMinTime) / intervalNum);
+ } else {
+ // randomize between [dataMinTime, dataMaxTime-range]
+ minTime =
+ (long) Math.ceil(dataMinTime + Math.random() * (dataMaxTime - range - dataMinTime + 1));
+ interval = (long) Math.ceil((double) range / intervalNum);
+ }
+ maxTime = minTime + interval * intervalNum;
+
session = new Session("127.0.0.1", 6667, "root", "root");
session.open(false);
session.setFetchSize(100000); // this is important. Set it big to avoid multiple fetch.
- long interval = (long) Math.ceil((double) (maxTime - minTime) / intervalNum);
- maxTime = minTime + interval * intervalNum;
-
String sql;
if (approach == 1) { // MAC UDF
sql =
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGame_old.java b/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGame_old.java
new file mode 100644
index 0000000..a86d45a
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGame_old.java
@@ -0,0 +1,104 @@
+// package org.apache.iotdb.queryExp;
+//
+// import org.apache.iotdb.rpc.IoTDBConnectionException;
+// import org.apache.iotdb.rpc.StatementExecutionException;
+// import org.apache.iotdb.session.Session;
+// import org.apache.iotdb.session.SessionDataSet;
+// import org.apache.iotdb.tsfile.read.common.RowRecord;
+//
+// import org.apache.thrift.TException;
+//
+// public class QueryFullGame {
+//
+// // * (1) min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s),
+// max_value(%s)
+// // => Don't change the sequence of the above six aggregates!
+// // * (2) group by ([tqs,tqe),IntervalLength) => Make sure (tqe-tqs) is divisible by
+// // IntervalLength!
+// // * (3) NOTE the time unit of interval. Update for different datasets!!!!!!!!!!!
+// private static final String queryFormat =
+// "select min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s),
+// max_value(%s) "
+// + "from %s "
+// + "group by ([%d, %d), %dns)";
+//
+// private static final String queryFormat_UDF =
+// "select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and
+// time<%4$d";
+//
+// public static Session session;
+//
+// public static void main(String[] args)
+// throws IoTDBConnectionException, StatementExecutionException, TException {
+// // fixed time series path
+// String measurement = "s6";
+// String device = "root.game";
+// // fixed tqs
+// long minTime = 0L;
+// // 实验自变量1：tqe
+// long maxTime = Long.parseLong(args[0]);
+// // 实验自变量2：w数量
+// int intervalNum = Integer.parseInt(args[1]);
+// // 实验自变量3：方法
+// // 1: MAC, 2: MOC, 3: CPV
+// int approach = Integer.parseInt(args[2]);
+// if (approach != 1 && approach != 2 && approach != 3) {
+// throw new TException("Wrong input parameter approach!");
+// }
+// if (approach != 1) {
+// // MOC and CPV sql are the same sql: queryFormat.
+// // Set the server parameter in iotdb-engine.properties: enable_CPV=true for CPV, false for
+// // MOC.
+// if (approach == 2) { // MOC
+// System.out.println(
+// "MAKE SURE you have set the enable_CPV as false in `iotdb-engine.properties` for
+// MOC!");
+// } else { // CPV
+// System.out.println(
+// "MAKE SURE you have set the enable_CPV as true in `iotdb-engine.properties` for
+// CPV!");
+// }
+// }
+//
+// session = new Session("127.0.0.1", 6667, "root", "root");
+// session.open(false);
+// session.setFetchSize(100000); // this is important. Set it big to avoid multiple fetch.
+//
+// long interval = (long) Math.ceil((double) (maxTime - minTime) / intervalNum);
+// maxTime = minTime + interval * intervalNum;
+//
+// String sql;
+// if (approach == 1) { // MAC UDF
+// sql =
+// String.format(queryFormat_UDF, measurement, device, minTime, maxTime, intervalNum); //
+// MAC
+// } else {
+// // MOC and CPV sql use the same sql queryFormat.
+// sql =
+// String.format(
+// queryFormat,
+// measurement,
+// measurement,
+// measurement,
+// measurement,
+// measurement,
+// measurement,
+// device,
+// minTime,
+// maxTime,
+// interval);
+// }
+//
+// SessionDataSet dataSet;
+// dataSet = session.executeQueryStatement(sql);
+// while (dataSet.hasNext()) {
+// RowRecord r = dataSet.next();
+// }
+// session.executeNonQueryStatement("clear cache");
+// dataSet = session.executeFinish();
+// String info = dataSet.getFinishResult();
+// System.out.println(info);
+// dataSet.closeOperationHandle();
+// session.close();
+// }
+// }
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGameVaryTqe.java b/example/session/src/main/java/org/apache/iotdb/queryExp/QueryMF03.java
similarity index 73%
rename from example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGameVaryTqe.java
rename to example/session/src/main/java/org/apache/iotdb/queryExp/QueryMF03.java
index af8f285..bc2d6b2 100644
--- a/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGameVaryTqe.java
+++ b/example/session/src/main/java/org/apache/iotdb/queryExp/QueryMF03.java
@@ -8,17 +8,17 @@ import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.thrift.TException;
-public class QueryFullGameVaryTqe {
+public class QueryMF03 {
// * (1) min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s), max_value(%s)
// => Don't change the sequence of the above six aggregates!
// * (2) group by ([tqs,tqe),IntervalLength) => Make sure (tqe-tqs) is divisible by
// IntervalLength!
- // * (3) NOTE the time unit of interval. Update for different datasets!!!!!!!!!!!
+ // * (3) NOTE the time unit of interval. Update for different datasets!
private static final String queryFormat =
"select min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s), max_value(%s) "
+ "from %s "
- + "group by ([%d, %d), %dns)";
+ + "group by ([%d, %d), %dns)"; // note this "ns" time precision
private static final String queryFormat_UDF =
"select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and time<%4$d";
@@ -28,17 +28,19 @@ public class QueryFullGameVaryTqe {
public static void main(String[] args)
throws IoTDBConnectionException, StatementExecutionException, TException {
// fixed time series path
- String measurement = "s6";
- String device = "root.game";
- // fixed interval num
- int intervalNum = 50;
- // fixed tqs
- long minTime = 0L;
- // 实验自变量1：tqe
- long maxTime = Long.parseLong(args[0]);
- // 实验自变量2：方法
+ String measurement = "mf03"; // [[update]]
+ String device = "root.debs2012"; // [[update]]
+ // used to bound tqs random position
+ long dataMinTime = 1329955200008812200L;
+ long dataMaxTime = 1329965999991045201L; // 617426057626+1
+
+ // 实验自变量1：[tqs,tqe) range length, i.e., tqe-tqs
+ long range = Long.parseLong(args[0]);
+ // 实验自变量2：w数量
+ int intervalNum = Integer.parseInt(args[1]);
+ // 实验自变量3：方法
// 1: MAC, 2: MOC, 3: CPV
- int approach = Integer.parseInt(args[1]);
+ int approach = Integer.parseInt(args[2]);
if (approach != 1 && approach != 2 && approach != 3) {
throw new TException("Wrong input parameter approach!");
}
@@ -55,13 +57,24 @@ public class QueryFullGameVaryTqe {
}
}
+ long minTime;
+ long maxTime;
+ long interval;
+ if (range >= (dataMaxTime - dataMinTime)) {
+ minTime = dataMinTime;
+ interval = (long) Math.ceil((double) (dataMaxTime - dataMinTime) / intervalNum);
+ } else {
+ // randomize between [dataMinTime, dataMaxTime-range]
+ minTime =
+ (long) Math.ceil(dataMinTime + Math.random() * (dataMaxTime - range - dataMinTime + 1));
+ interval = (long) Math.ceil((double) range / intervalNum);
+ }
+ maxTime = minTime + interval * intervalNum;
+
session = new Session("127.0.0.1", 6667, "root", "root");
session.open(false);
session.setFetchSize(100000); // this is important. Set it big to avoid multiple fetch.
- long interval = (long) Math.ceil((double) (maxTime - minTime) / intervalNum);
- maxTime = minTime + interval * intervalNum;
-
String sql;
if (approach == 1) { // MAC UDF
sql =
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGameVaryW.java b/example/session/src/main/java/org/apache/iotdb/queryExp/QueryRecTime.java
similarity index 73%
rename from example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGameVaryW.java
rename to example/session/src/main/java/org/apache/iotdb/queryExp/QueryRecTime.java
index 4758b82..78c225c 100644
--- a/example/session/src/main/java/org/apache/iotdb/queryExp/QueryFullGameVaryW.java
+++ b/example/session/src/main/java/org/apache/iotdb/queryExp/QueryRecTime.java
@@ -8,17 +8,17 @@ import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.thrift.TException;
-public class QueryFullGameVaryW {
+public class QueryRecTime {
// * (1) min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s), max_value(%s)
// => Don't change the sequence of the above six aggregates!
// * (2) group by ([tqs,tqe),IntervalLength) => Make sure (tqe-tqs) is divisible by
// IntervalLength!
- // * (3) NOTE the time unit of interval. Update for different datasets!!!!!!!!!!!
+ // * (3) NOTE the time unit of interval. Update for different datasets!
private static final String queryFormat =
"select min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s), max_value(%s) "
+ "from %s "
- + "group by ([%d, %d), %dns)";
+ + "group by ([%d, %d), %dms)"; // note this "ms" time precision
private static final String queryFormat_UDF =
"select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and time<%4$d";
@@ -28,16 +28,19 @@ public class QueryFullGameVaryW {
public static void main(String[] args)
throws IoTDBConnectionException, StatementExecutionException, TException {
// fixed time series path
- String measurement = "s6";
- String device = "root.game";
- // fixed query total range
- long minTime = 0L;
- long maxTime = 4264605928301L; // unit:ns. Set in iotdb-engine.properties `timestamp_precision`.
- // 实验自变量1：w数量
- int intervalNum = Integer.parseInt(args[0]);
- // 实验自变量2：方法
+ String measurement = "RecTime"; // [[update]]
+ String device = "root.sg1"; // [[update]]
+ // used to bound tqs random position
+ long dataMinTime = 1616194494000L; // [[update]]
+ long dataMaxTime = 1642656230001L; // 617426057626+1 // [[update]]
+
+ // 实验自变量1：[tqs,tqe) range length, i.e., tqe-tqs
+ long range = Long.parseLong(args[0]);
+ // 实验自变量2：w数量
+ int intervalNum = Integer.parseInt(args[1]);
+ // 实验自变量3：方法
// 1: MAC, 2: MOC, 3: CPV
- int approach = Integer.parseInt(args[1]);
+ int approach = Integer.parseInt(args[2]);
if (approach != 1 && approach != 2 && approach != 3) {
throw new TException("Wrong input parameter approach!");
}
@@ -54,13 +57,24 @@ public class QueryFullGameVaryW {
}
}
+ long minTime;
+ long maxTime;
+ long interval;
+ if (range >= (dataMaxTime - dataMinTime)) {
+ minTime = dataMinTime;
+ interval = (long) Math.ceil((double) (dataMaxTime - dataMinTime) / intervalNum);
+ } else {
+ // randomize between [dataMinTime, dataMaxTime-range]
+ minTime =
+ (long) Math.ceil(dataMinTime + Math.random() * (dataMaxTime - range - dataMinTime + 1));
+ interval = (long) Math.ceil((double) range / intervalNum);
+ }
+ maxTime = minTime + interval * intervalNum;
+
session = new Session("127.0.0.1", 6667, "root", "root");
session.open(false);
session.setFetchSize(100000); // this is important. Set it big to avoid multiple fetch.
- long interval = (long) Math.ceil((double) (maxTime - minTime) / intervalNum);
- maxTime = minTime + interval * intervalNum;
-
String sql;
if (approach == 1) { // MAC UDF
sql =
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/QuerySyntheticData1VaryW.java b/example/session/src/main/java/org/apache/iotdb/queryExp/QuerySyntheticData1VaryW.java
deleted file mode 100644
index a57c9a8..0000000
--- a/example/session/src/main/java/org/apache/iotdb/queryExp/QuerySyntheticData1VaryW.java
+++ /dev/null
@@ -1,109 +0,0 @@
-package org.apache.iotdb.queryExp;
-
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.session.Session;
-import org.apache.iotdb.session.SessionDataSet;
-import org.apache.iotdb.tsfile.read.common.RowRecord;
-
-import org.apache.thrift.TException;
-
-/**
- * !!!!!!!Before query data, make sure check the following server parameters:
- *
- * <p>system_dir=/data3/ruilei/iotdb-server-0.12.4/synData1/system
- * data_dirs=/data3/ruilei/iotdb-server-0.12.4/synData1/data
- * wal_dir=/data3/ruilei/iotdb-server-0.12.4/synData1/wal timestamp_precision=ms
- * unseq_tsfile_size=1073741824 # maximum size of unseq TsFile is 1024^3 Bytes
- * seq_tsfile_size=1073741824 # maximum size of seq TsFile is 1024^3 Bytes
- * avg_series_point_number_threshold=10000 # each chunk contains 10000 data points
- * compaction_strategy=NO_COMPACTION # compaction between levels is disabled
- * enable_unseq_compaction=false # unseq compaction is disabled
- */
-public class QuerySyntheticData1VaryW {
-
- // * (1) min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s), max_value(%s)
- // => Don't change the sequence of the above six aggregates!
- // * (2) group by ([tqs,tqe),IntervalLength) => Make sure (tqe-tqs) is divisible by
- // IntervalLength!
- // * (3) NOTE the time unit of interval. Update for different datasets!!!!!!!!!!!
- private static final String queryFormat =
- "select min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s), max_value(%s) "
- + "from %s "
- + "group by ([%d, %d), %dms)";
-
- private static final String queryFormat_UDF =
- "select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and time<%4$d";
-
- public static Session session;
-
- public static void main(String[] args)
- throws IoTDBConnectionException, StatementExecutionException, TException {
- // fix parameters for synthetic data1
- String measurement = "s0";
- String device = "root.vehicle.d0";
- // fixed query total range
- long minTime = 0L;
- long maxTime = 10000000L; // unit:ms. Set in iotdb-engine.properties `timestamp_precision`.
- // 实验自变量1：w数量
- int intervalNum = Integer.parseInt(args[0]);
- // 实验自变量2：方法
- // 1: MAC, 2: MOC, 3: CPV
- int approach = Integer.parseInt(args[1]);
- if (approach != 1 && approach != 2 && approach != 3) {
- throw new TException("Wrong input parameter approach!");
- }
- if (approach != 1) {
- // MOC and CPV sql are the same sql: queryFormat.
- // Set the server parameter in iotdb-engine.properties: enable_CPV=true for CPV, false for
- // MOC.
- if (approach == 2) { // MOC
- System.out.println(
- "MAKE SURE you have set the enable_CPV as false in `iotdb-engine.properties` for MOC!");
- } else { // CPV
- System.out.println(
- "MAKE SURE you have set the enable_CPV as true in `iotdb-engine.properties` for CPV!");
- }
- }
-
- session = new Session("127.0.0.1", 6667, "root", "root");
- session.open(false);
- session.setFetchSize(100000); // this is important. Set it big to avoid multiple fetch.
-
- long interval = (long) Math.ceil((double) (maxTime - minTime) / intervalNum);
- maxTime = minTime + interval * intervalNum;
-
- String sql;
- if (approach == 1) { // MAC UDF
- sql =
- String.format(queryFormat_UDF, measurement, device, minTime, maxTime, intervalNum); // MAC
- } else {
- // MOC and CPV sql use the same sql queryFormat.
- sql =
- String.format(
- queryFormat,
- measurement,
- measurement,
- measurement,
- measurement,
- measurement,
- measurement,
- device,
- minTime,
- maxTime,
- interval);
- }
-
- SessionDataSet dataSet;
- dataSet = session.executeQueryStatement(sql);
- while (dataSet.hasNext()) {
- RowRecord r = dataSet.next();
- }
- session.executeNonQueryStatement("clear cache");
- dataSet = session.executeFinish();
- String info = dataSet.getFinishResult();
- System.out.println(info);
- dataSet.closeOperationHandle();
- session.close();
- }
-}
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/QuerySyntheticData1VaryTqe.java b/example/session/src/main/java/org/apache/iotdb/queryExp/QueryTianyuan.java
similarity index 71%
rename from example/session/src/main/java/org/apache/iotdb/queryExp/QuerySyntheticData1VaryTqe.java
rename to example/session/src/main/java/org/apache/iotdb/queryExp/QueryTianyuan.java
index baf1f02..4bf7bb9 100644
--- a/example/session/src/main/java/org/apache/iotdb/queryExp/QuerySyntheticData1VaryTqe.java
+++ b/example/session/src/main/java/org/apache/iotdb/queryExp/QueryTianyuan.java
@@ -8,29 +8,17 @@ import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.thrift.TException;
-/**
- * !!!!!!!Before query data, make sure check the following server parameters:
- *
- * <p>system_dir=/data3/ruilei/iotdb-server-0.12.4/synData1/system
- * data_dirs=/data3/ruilei/iotdb-server-0.12.4/synData1/data
- * wal_dir=/data3/ruilei/iotdb-server-0.12.4/synData1/wal timestamp_precision=ms
- * unseq_tsfile_size=1073741824 # maximum size of unseq TsFile is 1024^3 Bytes
- * seq_tsfile_size=1073741824 # maximum size of seq TsFile is 1024^3 Bytes
- * avg_series_point_number_threshold=10000 # each chunk contains 10000 data points
- * compaction_strategy=NO_COMPACTION # compaction between levels is disabled
- * enable_unseq_compaction=false # unseq compaction is disabled
- */
-public class QuerySyntheticData1VaryTqe {
+public class QueryTianyuan {
// * (1) min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s), max_value(%s)
// => Don't change the sequence of the above six aggregates!
// * (2) group by ([tqs,tqe),IntervalLength) => Make sure (tqe-tqs) is divisible by
// IntervalLength!
- // * (3) NOTE the time unit of interval. Update for different datasets!!!!!!!!!!!
+ // * (3) NOTE the time unit of interval. Update for different datasets!
private static final String queryFormat =
"select min_time(%s), max_time(%s), first_value(%s), last_value(%s), min_value(%s), max_value(%s) "
+ "from %s "
- + "group by ([%d, %d), %dms)";
+ + "group by ([%d, %d), %dms)"; // note this "ms" time precision
private static final String queryFormat_UDF =
"select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and time<%4$d";
@@ -39,18 +27,20 @@ public class QuerySyntheticData1VaryTqe {
public static void main(String[] args)
throws IoTDBConnectionException, StatementExecutionException, TException {
- // fix parameters for synthetic data1
- String measurement = "s0";
- String device = "root.vehicle.d0";
- // fixed interval num
- int intervalNum = 20;
- // fix tqs
- long minTime = 0L;
- // 实验自变量1：tqe
- long maxTime = Long.parseLong(args[0]);
- // 实验自变量2：方法
+ // fixed time series path
+ String measurement = "KOB_0002_00_67"; // [[update]]
+ String device = "root.kobelco.trans.03.1090001603.2401604"; // [[update]]
+ // used to bound tqs random position
+ long dataMinTime = 1616805035973L; // [[update]]
+ long dataMaxTime = 1627380839564L; // 1627380839563+1 // [[update]]
+
+ // 实验自变量1：[tqs,tqe) range length, i.e., tqe-tqs
+ long range = Long.parseLong(args[0]);
+ // 实验自变量2：w数量
+ int intervalNum = Integer.parseInt(args[1]);
+ // 实验自变量3：方法
// 1: MAC, 2: MOC, 3: CPV
- int approach = Integer.parseInt(args[1]);
+ int approach = Integer.parseInt(args[2]);
if (approach != 1 && approach != 2 && approach != 3) {
throw new TException("Wrong input parameter approach!");
}
@@ -67,13 +57,24 @@ public class QuerySyntheticData1VaryTqe {
}
}
+ long minTime;
+ long maxTime;
+ long interval;
+ if (range >= (dataMaxTime - dataMinTime)) {
+ minTime = dataMinTime;
+ interval = (long) Math.ceil((double) (dataMaxTime - dataMinTime) / intervalNum);
+ } else {
+ // randomize between [dataMinTime, dataMaxTime-range]
+ minTime =
+ (long) Math.ceil(dataMinTime + Math.random() * (dataMaxTime - range - dataMinTime + 1));
+ interval = (long) Math.ceil((double) range / intervalNum);
+ }
+ maxTime = minTime + interval * intervalNum;
+
session = new Session("127.0.0.1", 6667, "root", "root");
session.open(false);
session.setFetchSize(100000); // this is important. Set it big to avoid multiple fetch.
- long interval = (long) Math.ceil((double) (maxTime - minTime) / intervalNum);
- maxTime = minTime + interval * intervalNum;
-
String sql;
if (approach == 1) { // MAC UDF
sql =
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-compare-loadDataOnly.png b/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-compare-loadDataOnly.png
deleted file mode 100644
index f5eb05d..0000000
Binary files a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-compare-loadDataOnly.png and /dev/null differ
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-compare-totalOnly.png b/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-compare-totalOnly.png
deleted file mode 100644
index a844ab1..0000000
Binary files a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-compare-totalOnly.png and /dev/null differ
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-compare.png b/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-compare.png
deleted file mode 100644
index 3e9fa3e..0000000
Binary files a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-compare.png and /dev/null differ
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-cpv.png b/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-cpv.png
deleted file mode 100644
index 90813e0..0000000
Binary files a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-cpv.png and /dev/null differ
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-mac.png b/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-mac.png
deleted file mode 100644
index ec6beb5..0000000
Binary files a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-mac.png and /dev/null differ
diff --git a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-moc.png b/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-moc.png
deleted file mode 100644
index 911efec..0000000
Binary files a/example/session/src/main/java/org/apache/iotdb/queryExp/synData1VaryWExpFigures/synData1-0disorder-0delete-varyW-moc.png and /dev/null differ
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/ExtractDebs2012Data.java b/example/session/src/main/java/org/apache/iotdb/writeData/ExtractDebs2012Data.java
index d05cdec..54cbfa7 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/ExtractDebs2012Data.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/ExtractDebs2012Data.java
@@ -13,17 +13,19 @@ import java.time.format.DateTimeParseException;
public class ExtractDebs2012Data {
- // java ExtractDebs2012Data /data3/raw_data/data/debs2012/allData.txt mf03-3h.csv 0 4 1329955200000000000 1329966000000000000
+ // java ExtractDebs2012Data /data3/raw_data/data/debs2012/allData.txt mf03-3h.csv 0 4
+ // 1329955200000000000 1329966000000000000
public static void main(String[] args) throws IOException {
-// String tmp = "2012-03-20T06:53:51.4554819+00:00";
-// long timestamp = getInstantWithPrecision(tmp, "ns");
-// System.out.println(timestamp);
+ // String tmp = "2012-03-20T06:53:51.4554819+00:00";
+ // long timestamp = getInstantWithPrecision(tmp, "ns");
+ // System.out.println(timestamp);
String inPath = args[0];
String outPath = args[1];
int timeIdx = Integer.parseInt(args[2]); // 0
int valueIdx = Integer.parseInt(args[3]); // mf03: 4
- // [1329955200000000000 Thursday, February 23, 2012 0:00:00~1329966000000000000 Thursday, February 23, 2012 3:00:00)
+ // [1329955200000000000 Thursday, February 23, 2012 0:00:00~1329966000000000000 Thursday,
+ // February 23, 2012 3:00:00)
long startTime = Long.parseLong(args[4]);
long endTime = Long.parseLong(args[5]);
@@ -60,34 +62,36 @@ public class ExtractDebs2012Data {
printWriter.close();
}
-// private static long convertDatetimeStrToLong(String str, ZoneOffset offset, int depth)
-// throws LogicalOperatorException {
-//
-// String timestampPrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
-//
-// if (depth >= 2) {
-// throw new DateTimeException(
-// String.format(
-// "Failed to convert %s to millisecond, zone offset is %s, "
-// + "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00",
-// str, offset));
-// }
-// if (str.contains("Z")) {
-// return convertDatetimeStrToLong(str.substring(0, str.indexOf('Z')) + "+00:00", offset, depth);
-// } else if (str.length() == 10) {
-// return convertDatetimeStrToLong(str + "T00:00:00", offset, depth);
-// } else if (str.length() - str.lastIndexOf('+') != 6
-// && str.length() - str.lastIndexOf('-') != 6) {
-// return convertDatetimeStrToLong(str + offset, offset, depth + 1);
-// } else if (str.contains("[") || str.contains("]")) {
-// throw new DateTimeException(
-// String.format(
-// "%s with [time-region] at end is not supported now, "
-// + "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00",
-// str));
-// }
-// return getInstantWithPrecision(str, timestampPrecision);
-// }
+ // private static long convertDatetimeStrToLong(String str, ZoneOffset offset, int depth)
+ // throws LogicalOperatorException {
+ //
+ // String timestampPrecision =
+ // IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+ //
+ // if (depth >= 2) {
+ // throw new DateTimeException(
+ // String.format(
+ // "Failed to convert %s to millisecond, zone offset is %s, "
+ // + "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00",
+ // str, offset));
+ // }
+ // if (str.contains("Z")) {
+ // return convertDatetimeStrToLong(str.substring(0, str.indexOf('Z')) + "+00:00", offset,
+ // depth);
+ // } else if (str.length() == 10) {
+ // return convertDatetimeStrToLong(str + "T00:00:00", offset, depth);
+ // } else if (str.length() - str.lastIndexOf('+') != 6
+ // && str.length() - str.lastIndexOf('-') != 6) {
+ // return convertDatetimeStrToLong(str + offset, offset, depth + 1);
+ // } else if (str.contains("[") || str.contains("]")) {
+ // throw new DateTimeException(
+ // String.format(
+ // "%s with [time-region] at end is not supported now, "
+ // + "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00",
+ // str));
+ // }
+ // return getInstantWithPrecision(str, timestampPrecision);
+ // }
private static long getInstantWithPrecision(String str, String timestampPrecision)
throws IOException {
@@ -115,4 +119,3 @@ public class ExtractDebs2012Data {
}
}
}
-
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/ExtractFullGameData.java b/example/session/src/main/java/org/apache/iotdb/writeData/ExtractFullGameData.java
index 54fe62f..aa7e6c5 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/ExtractFullGameData.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/ExtractFullGameData.java
@@ -7,7 +7,8 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
-// java ExtractFullGameData /data3/raw_data/data/full-game full-game-sid8-1000000 1 5 8 2000000 3200000
+// java ExtractFullGameData /data3/raw_data/data/full-game full-game-sid8-10min 1 5 8 2000000
+// 3200000
public class ExtractFullGameData {
// sid, ts, x, y, z, |v|, |a|, vx, vy, vz, ax, ay, az
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/ExtractDebs2012Data.java b/example/session/src/main/java/org/apache/iotdb/writeData/ExtractRecTimeData.java
similarity index 56%
copy from example/session/src/main/java/org/apache/iotdb/writeData/ExtractDebs2012Data.java
copy to example/session/src/main/java/org/apache/iotdb/writeData/ExtractRecTimeData.java
index d05cdec..9fe43d1 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/ExtractDebs2012Data.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/ExtractRecTimeData.java
@@ -11,19 +11,19 @@ import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
-public class ExtractDebs2012Data {
+public class ExtractRecTimeData {
- // java ExtractDebs2012Data /data3/raw_data/data/debs2012/allData.txt mf03-3h.csv 0 4 1329955200000000000 1329966000000000000
+ // java D:\pythonPlotDoNotDeleteIt\drawRawData\RecTime\dump0.csv
+ // D:\pythonPlotDoNotDeleteIt\drawRawData\RecTime\dump1.csv 0 1 0 1644144767000
public static void main(String[] args) throws IOException {
-// String tmp = "2012-03-20T06:53:51.4554819+00:00";
-// long timestamp = getInstantWithPrecision(tmp, "ns");
-// System.out.println(timestamp);
+ // String tmp = "2021-03-20T06:54:54.000+08:00";
+ // long timestamp = getInstantWithPrecision(tmp, "ms");
+ // System.out.println(timestamp);
String inPath = args[0];
String outPath = args[1];
int timeIdx = Integer.parseInt(args[2]); // 0
int valueIdx = Integer.parseInt(args[3]); // mf03: 4
- // [1329955200000000000 Thursday, February 23, 2012 0:00:00~1329966000000000000 Thursday, February 23, 2012 3:00:00)
long startTime = Long.parseLong(args[4]);
long endTime = Long.parseLong(args[5]);
@@ -36,59 +36,31 @@ public class ExtractDebs2012Data {
long lastTimestamp = -1;
while ((line = reader.readLine()) != null) {
- String[] split = line.split("\\s+");
+ String[] split = line.split(",");
String timestampStr = split[timeIdx];
- long timestamp = getInstantWithPrecision(timestampStr, "ns");
+ long timestamp = getInstantWithPrecision(timestampStr, "ms");
if (timestamp >= endTime) {
break;
}
if (timestamp < startTime) {
continue;
}
+ if (timestamp <= lastTimestamp) {
+ System.out.println("out-of-order! " + timestamp + "last: " + lastTimestamp);
+ continue;
+ }
long value = Long.parseLong(split[valueIdx]);
printWriter.print(timestamp);
printWriter.print(",");
printWriter.print(value);
printWriter.println();
cnt++;
- if (timestamp < lastTimestamp) {
- System.out.println("out-of-order! " + timestamp);
- }
lastTimestamp = timestamp;
}
reader.close();
printWriter.close();
}
-// private static long convertDatetimeStrToLong(String str, ZoneOffset offset, int depth)
-// throws LogicalOperatorException {
-//
-// String timestampPrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
-//
-// if (depth >= 2) {
-// throw new DateTimeException(
-// String.format(
-// "Failed to convert %s to millisecond, zone offset is %s, "
-// + "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00",
-// str, offset));
-// }
-// if (str.contains("Z")) {
-// return convertDatetimeStrToLong(str.substring(0, str.indexOf('Z')) + "+00:00", offset, depth);
-// } else if (str.length() == 10) {
-// return convertDatetimeStrToLong(str + "T00:00:00", offset, depth);
-// } else if (str.length() - str.lastIndexOf('+') != 6
-// && str.length() - str.lastIndexOf('-') != 6) {
-// return convertDatetimeStrToLong(str + offset, offset, depth + 1);
-// } else if (str.contains("[") || str.contains("]")) {
-// throw new DateTimeException(
-// String.format(
-// "%s with [time-region] at end is not supported now, "
-// + "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00",
-// str));
-// }
-// return getInstantWithPrecision(str, timestampPrecision);
-// }
-
private static long getInstantWithPrecision(String str, String timestampPrecision)
throws IOException {
try {
@@ -115,4 +87,3 @@ public class ExtractDebs2012Data {
}
}
}
-
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/ExtractDebug.java b/example/session/src/main/java/org/apache/iotdb/writeData/ExtractTianyuan.java
similarity index 60%
rename from example/session/src/main/java/org/apache/iotdb/writeData/ExtractDebug.java
rename to example/session/src/main/java/org/apache/iotdb/writeData/ExtractTianyuan.java
index 0bab58d..03ad103 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/ExtractDebug.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/ExtractTianyuan.java
@@ -7,16 +7,13 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
-// java ExtractDebug _OutOfOrder1_maxdelay300000_mindelay0.data debug.csv 3 7
-// 16525359
-public class ExtractDebug {
+public class ExtractTianyuan {
public static void main(String[] args) throws IOException {
String inPath = args[0];
String outPath = args[1];
- int timeIdx = Integer.parseInt(args[2]); // 3
- int valueIdx = Integer.parseInt(args[3]); // 7
- int lineNum = Integer.parseInt(args[4]); // -1 means no constraint
+ int timeIdx = Integer.parseInt(args[2]); // 0
+ int valueIdx = Integer.parseInt(args[3]); // 1
File f = new File(inPath);
FileWriter fileWriter = new FileWriter(outPath);
@@ -24,23 +21,21 @@ public class ExtractDebug {
BufferedReader reader = new BufferedReader(new FileReader(f));
PrintWriter printWriter = new PrintWriter(fileWriter);
int cnt = 0;
- long minTime = -1;
- while (((lineNum > 0 && cnt < lineNum) || lineNum < 0) && (line = reader.readLine()) != null) {
+ long lastTimestamp = -1;
+ while ((line = reader.readLine()) != null) {
String[] split = line.split(",");
long timestamp = Long.parseLong(split[timeIdx]);
- if (minTime == -1) {
- minTime = timestamp;
- timestamp = 0;
- } else {
- timestamp = timestamp - minTime;
- }
- timestamp = (long) (timestamp / 1000);
long value = Long.parseLong(split[valueIdx]);
+ if (timestamp <= lastTimestamp) {
+ System.out.println("out-of-order! " + timestamp + "last: " + lastTimestamp);
+ continue;
+ }
printWriter.print(timestamp);
printWriter.print(",");
printWriter.print(value);
printWriter.println();
cnt++;
+ lastTimestamp = timestamp;
}
reader.close();
printWriter.close();
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/TestFullGameTmp.java b/example/session/src/main/java/org/apache/iotdb/writeData/TestFullGameTmp.java
deleted file mode 100644
index dd481d5..0000000
--- a/example/session/src/main/java/org/apache/iotdb/writeData/TestFullGameTmp.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.apache.iotdb.writeData;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-
-public class TestFullGameTmp {
-
- public static void main(String[] args) throws IOException {
- String path = args[0];
- File f = new File(path);
- String line = null;
- BufferedReader reader = new BufferedReader(new FileReader(f));
- long lastTimestamp = 0L;
- int cnt = 0;
- int totalcnt = 0;
- while ((line = reader.readLine()) != null) {
- String[] split = line.split(",");
- long timestamp = Long.valueOf(split[3]);
- totalcnt++;
- if (timestamp < lastTimestamp) {
- // System.out.println("disordered!");
- cnt++;
- }
- lastTimestamp = timestamp;
- }
- System.out.println("total disorder points=" + cnt);
- System.out.println("total points=" + totalcnt);
- System.out.println(cnt / totalcnt);
- }
-}
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java
index ce37ba8..ad0e863 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java
@@ -12,31 +12,43 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Random;
public class WriteFullGame {
/** Before writing data, make sure check the server parameter configurations. */
public static void main(String[] args)
throws IoTDBConnectionException, StatementExecutionException, IOException {
- String measurements = "s6";
- String device = "root.game";
-
- List<String> deletePaths = new ArrayList<>();
- deletePaths.add(device + "." + measurements);
+ String measurements = "s6"; // [[update]]
+ String device = "root.game"; // [[update]]
+ long chunkAvgTimeLen = 5238417462L; // ns [[update]]
// 实验自变量1：乱序数据源
String filePath = args[0];
- // 实验自变量2：每隔deleteFreq时间（本实验时间单位ms）就执行一次删除
- long deleteFreq = Long.parseLong(args[1]); // 0 means no deletes
- // 实验自变量3：每次删除的时间长度
- long deleteLen = Long.parseLong(args[2]);
+ // 实验自变量2：delete percentage
+ int deletePercentage = Integer.parseInt(args[1]); // 0 means no deletes. 0-100
+ // 实验自变量3：每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ int deleteLenPercentage = Integer.parseInt(args[2]); // 0-100
// 参数4：时间戳idx，从0开始
int timeIdx = Integer.parseInt(args[3]);
// 参数5：值idx，从0开始
int valueIdx = Integer.parseInt(args[4]);
- TSDataType tsDataType = TSDataType.INT64;
+ if (deletePercentage < 0 || deletePercentage > 100) {
+ throw new IOException("WRONG deletePercentage!");
+ }
+ if (deleteLenPercentage < 0 || deleteLenPercentage > 100) {
+ throw new IOException("WRONG deleteLenPercentage!");
+ }
+
+ int avgSeriesPointNumberThreshold = 10000; // fixed
+ int deletePeriod =
+ (int) Math.floor(100 * 1.0 / deletePercentage * avgSeriesPointNumberThreshold);
+ long deleteLen = (long) Math.floor(chunkAvgTimeLen * deleteLenPercentage * 1.0 / 100);
+
+ List<String> deletePaths = new ArrayList<>();
+ deletePaths.add(device + "." + measurements);
+
+ TSDataType tsDataType = TSDataType.INT64; // value types
Session session = new Session("127.0.0.1", 6667, "root", "root");
session.open(false);
@@ -44,17 +56,20 @@ public class WriteFullGame {
// this is to make all following inserts unseq chunks
session.insertRecord(
device,
- System.nanoTime(), // NOTE UPDATE TIME DATATYPE!
+ 1644181628000000000L,
+ // NOTE UPDATE TIME DATATYPE! [[update]]. DONT USE System.nanoTime()!
Collections.singletonList(measurements),
Collections.singletonList(tsDataType), // NOTE UPDATE VALUE DATATYPE!
- 0L); // NOTE UPDATE VALUE DATATYPE!!!
+ 0L); // NOTE UPDATE VALUE DATATYPE!
session.executeNonQueryStatement("flush");
long minTime = -1;
File f = new File(filePath);
String line = null;
BufferedReader reader = new BufferedReader(new FileReader(f));
- int lastDeleteIdx = -1;
+ long lastDeleteMinTime = Long.MAX_VALUE;
+ long lastDeleteMaxTime = Long.MIN_VALUE;
+ int cnt = 0;
while ((line = reader.readLine()) != null) {
String[] split = line.split(",");
long timestamp = Long.parseLong(split[timeIdx]);
@@ -65,24 +80,6 @@ public class WriteFullGame {
timestamp = timestamp - minTime;
}
timestamp = (long) (timestamp / 1000); // turn to ns. original time unit is ps. IoTDB only ns.
-
- if (deleteFreq != 0) {
- int idx = (int) Math.floor(timestamp * 1.0 / deleteFreq);
- // System.out.println(timestamp + "," + deleteFreq + "," + idx);
- if (idx > 0 && idx != lastDeleteIdx) {
- lastDeleteIdx = idx;
- // [timestamp-deleteFreq, timestamp-1]内随机取一个删除时间起点
- long min = timestamp - deleteFreq;
- long max = timestamp - 1;
- long deleteStartTime = min + (((long) (new Random().nextDouble() * (max - min))));
- long deleteEndTime = deleteStartTime + deleteLen - 1;
- session.deleteData(deletePaths, deleteStartTime, deleteEndTime);
-
- // System.out.println("[[[[delete]]]]]" + deleteStartTime + "," + deleteEndTime);
-
- }
- }
-
long value = Long.parseLong(split[valueIdx]);
session.insertRecord(
device,
@@ -90,6 +87,32 @@ public class WriteFullGame {
Collections.singletonList(measurements),
Collections.singletonList(tsDataType),
value);
+ cnt++;
+
+ if (timestamp > lastDeleteMaxTime) {
+ lastDeleteMaxTime = timestamp;
+ }
+ if (timestamp < lastDeleteMinTime) {
+ lastDeleteMinTime = timestamp;
+ }
+
+ if (deletePercentage != 0) {
+ if (cnt >= deletePeriod) {
+ cnt = 0;
+ // randomize deleteStartTime in [lastMinTime, max(lastMaxTime-deleteLen,lastMinTime+1)]
+ long rightBound = Math.max(lastDeleteMaxTime - deleteLen, lastDeleteMinTime + 1);
+ long deleteStartTime =
+ (long)
+ Math.ceil(
+ lastDeleteMinTime + Math.random() * (rightBound - lastDeleteMinTime + 1));
+ long deleteEndTime = deleteStartTime + deleteLen - 1;
+ session.deleteData(deletePaths, deleteStartTime, deleteEndTime);
+ System.out.println("[[[[delete]]]]]" + deleteStartTime + "," + deleteEndTime);
+
+ lastDeleteMinTime = Long.MAX_VALUE;
+ lastDeleteMaxTime = Long.MIN_VALUE;
+ }
+ }
}
session.executeNonQueryStatement("flush");
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameDisorderedToIoTDB.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameDisorderedToIoTDB.java
index e6e4774..d82416e 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameDisorderedToIoTDB.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameDisorderedToIoTDB.java
@@ -1,52 +1,52 @@
-package org.apache.iotdb.writeData;
-
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.session.Session;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.Collections;
-
-public class WriteFullGameDisorderedToIoTDB {
-
- public static void main(String[] args)
- throws IOException, IoTDBConnectionException, StatementExecutionException {
-
- String path = args[0];
-
- String measurements = "s6";
- String device = "root.game";
-
- Session session = new Session("127.0.0.1", 6667, "root", "root");
- session.open(false);
-
- long minTime = -1;
- File f = new File(path);
- String line = null;
- BufferedReader reader = new BufferedReader(new FileReader(f));
- while ((line = reader.readLine()) != null) {
- String[] split = line.split(",");
- long timestamp = Long.valueOf(split[3]);
- if (minTime == -1) {
- minTime = timestamp;
- timestamp = 0;
- } else {
- timestamp = timestamp - minTime;
- }
- timestamp = (long) (timestamp / 1000); // turn to ns
- long value = Long.valueOf(split[7]);
- session.insertRecord(
- device,
- timestamp,
- Collections.singletonList(measurements),
- Collections.singletonList(TSDataType.INT64),
- value);
- }
- session.executeNonQueryStatement("flush");
- session.close();
- }
-}
+// package org.apache.iotdb.writeData;
+//
+// import org.apache.iotdb.rpc.IoTDBConnectionException;
+// import org.apache.iotdb.rpc.StatementExecutionException;
+// import org.apache.iotdb.session.Session;
+// import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+//
+// import java.io.BufferedReader;
+// import java.io.File;
+// import java.io.FileReader;
+// import java.io.IOException;
+// import java.util.Collections;
+//
+// public class WriteFullGameDisorderedToIoTDB {
+//
+// public static void main(String[] args)
+// throws IOException, IoTDBConnectionException, StatementExecutionException {
+//
+// String path = args[0];
+//
+// String measurements = "s6";
+// String device = "root.game";
+//
+// Session session = new Session("127.0.0.1", 6667, "root", "root");
+// session.open(false);
+//
+// long minTime = -1;
+// File f = new File(path);
+// String line = null;
+// BufferedReader reader = new BufferedReader(new FileReader(f));
+// while ((line = reader.readLine()) != null) {
+// String[] split = line.split(",");
+// long timestamp = Long.valueOf(split[3]);
+// if (minTime == -1) {
+// minTime = timestamp;
+// timestamp = 0;
+// } else {
+// timestamp = timestamp - minTime;
+// }
+// timestamp = (long) (timestamp / 1000); // turn to ns
+// long value = Long.valueOf(split[7]);
+// session.insertRecord(
+// device,
+// timestamp,
+// Collections.singletonList(measurements),
+// Collections.singletonList(TSDataType.INT64),
+// value);
+// }
+// session.executeNonQueryStatement("flush");
+// session.close();
+// }
+// }
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameDisorderedWithDeletes.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameDisorderedWithDeletes.java
index fa86a0b..1baa587 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameDisorderedWithDeletes.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameDisorderedWithDeletes.java
@@ -1,73 +1,74 @@
-package org.apache.iotdb.writeData;
-
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.session.Session;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-
-public class WriteFullGameDisorderedWithDeletes {
-
- /** Before writing data, make sure check the server parameter configurations. */
- public static void main(String[] args)
- throws IoTDBConnectionException, StatementExecutionException, IOException {
- String measurements = "s6";
- String device = "root.game";
- List<String> deletePaths = new ArrayList<>();
- deletePaths.add(device + "." + measurements);
-
- // 实验自变量1：乱序数据源
- String filePath = args[0];
- // 实验自变量2：每隔deleteFreq时间（本实验时间单位ms）就执行一次删除
- long deleteFreq = Long.parseLong(args[1]); // 0 means no deletes
- // 实验自变量3：每次删除的时间长度
- long deleteLen = Long.parseLong(args[2]);
-
- Session session = new Session("127.0.0.1", 6667, "root", "root");
- session.open(false);
-
- long minTime = -1;
- File f = new File(filePath);
- String line = null;
- BufferedReader reader = new BufferedReader(new FileReader(f));
- while ((line = reader.readLine()) != null) {
- String[] split = line.split(",");
- long timestamp = Long.valueOf(split[3]);
- if (minTime == -1) {
- minTime = timestamp;
- timestamp = 0;
- } else {
- timestamp = timestamp - minTime;
- }
- timestamp = (long) (timestamp / 1000); // turn to ns. original time unit is ps. IoTDB only ns.
-
- if (deleteFreq != 0 && timestamp != 0 && timestamp % deleteFreq == 0) {
- // [timestamp-deleteFreq, timestamp-1]内随机取一个删除时间起点
- long min = timestamp - deleteFreq;
- long max = timestamp - 1;
- long deleteStartTime = min + (((long) (new Random().nextDouble() * (max - min))));
- long deleteEndTime = deleteStartTime + deleteLen - 1;
- session.deleteData(deletePaths, deleteStartTime, deleteEndTime);
- }
-
- long value = Long.valueOf(split[7]);
- session.insertRecord(
- device,
- timestamp,
- Collections.singletonList(measurements),
- Collections.singletonList(TSDataType.INT64),
- value);
- }
-
- session.executeNonQueryStatement("flush");
- session.close();
- }
-}
+// package org.apache.iotdb.writeData;
+//
+// import org.apache.iotdb.rpc.IoTDBConnectionException;
+// import org.apache.iotdb.rpc.StatementExecutionException;
+// import org.apache.iotdb.session.Session;
+// import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+//
+// import java.io.BufferedReader;
+// import java.io.File;
+// import java.io.FileReader;
+// import java.io.IOException;
+// import java.util.ArrayList;
+// import java.util.Collections;
+// import java.util.List;
+// import java.util.Random;
+//
+// public class WriteFullGameDisorderedWithDeletes {
+//
+// /** Before writing data, make sure check the server parameter configurations. */
+// public static void main(String[] args)
+// throws IoTDBConnectionException, StatementExecutionException, IOException {
+// String measurements = "s6";
+// String device = "root.game";
+// List<String> deletePaths = new ArrayList<>();
+// deletePaths.add(device + "." + measurements);
+//
+// // 实验自变量1：乱序数据源
+// String filePath = args[0];
+// // 实验自变量2：每隔deleteFreq时间（本实验时间单位ms）就执行一次删除
+// long deleteFreq = Long.parseLong(args[1]); // 0 means no deletes
+// // 实验自变量3：每次删除的时间长度
+// long deleteLen = Long.parseLong(args[2]);
+//
+// Session session = new Session("127.0.0.1", 6667, "root", "root");
+// session.open(false);
+//
+// long minTime = -1;
+// File f = new File(filePath);
+// String line = null;
+// BufferedReader reader = new BufferedReader(new FileReader(f));
+// while ((line = reader.readLine()) != null) {
+// String[] split = line.split(",");
+// long timestamp = Long.valueOf(split[3]);
+// if (minTime == -1) {
+// minTime = timestamp;
+// timestamp = 0;
+// } else {
+// timestamp = timestamp - minTime;
+// }
+// timestamp = (long) (timestamp / 1000); // turn to ns. original time unit is ps. IoTDB only
+// ns.
+//
+// if (deleteFreq != 0 && timestamp != 0 && timestamp % deleteFreq == 0) {
+// // [timestamp-deleteFreq, timestamp-1]内随机取一个删除时间起点
+// long min = timestamp - deleteFreq;
+// long max = timestamp - 1;
+// long deleteStartTime = min + (((long) (new Random().nextDouble() * (max - min))));
+// long deleteEndTime = deleteStartTime + deleteLen - 1;
+// session.deleteData(deletePaths, deleteStartTime, deleteEndTime);
+// }
+//
+// long value = Long.valueOf(split[7]);
+// session.insertRecord(
+// device,
+// timestamp,
+// Collections.singletonList(measurements),
+// Collections.singletonList(TSDataType.INT64),
+// value);
+// }
+//
+// session.executeNonQueryStatement("flush");
+// session.close();
+// }
+// }
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameToIoTDB.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameToIoTDB.java
index 8880ab7..4e73065 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameToIoTDB.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGameToIoTDB.java
@@ -1,54 +1,54 @@
-package org.apache.iotdb.writeData;
-
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.session.Session;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.Collections;
-
-public class WriteFullGameToIoTDB {
-
- public static void main(String[] args)
- throws IOException, IoTDBConnectionException, StatementExecutionException {
-
- // String path = args[0];
- String path = "/home/kyy/Documents/kdd/data/full-game.csv";
- String measurements = "s6";
-
- String device = "root.game";
-
- Session session = new Session("127.0.0.1", 6667, "root", "root");
- session.open(false);
-
- long minTime = -1;
-
- File f = new File(path);
- String line = null;
- BufferedReader reader = new BufferedReader(new FileReader(f));
- while ((line = reader.readLine()) != null) {
- String[] split = line.split(",");
- long timestamp = Long.valueOf(split[1]);
- if (minTime == -1) {
- minTime = timestamp;
- timestamp = 0;
- } else {
- timestamp = timestamp - minTime;
- }
- timestamp = (long) (timestamp / 1000);
- long value = Long.valueOf(split[5]);
- session.insertRecord(
- device,
- timestamp,
- Collections.singletonList(measurements),
- Collections.singletonList(TSDataType.INT64),
- value);
- }
- session.executeNonQueryStatement("flush");
- session.close();
- }
-}
+// package org.apache.iotdb.writeData;
+//
+// import org.apache.iotdb.rpc.IoTDBConnectionException;
+// import org.apache.iotdb.rpc.StatementExecutionException;
+// import org.apache.iotdb.session.Session;
+// import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+//
+// import java.io.BufferedReader;
+// import java.io.File;
+// import java.io.FileReader;
+// import java.io.IOException;
+// import java.util.Collections;
+//
+// public class WriteFullGameToIoTDB {
+//
+// public static void main(String[] args)
+// throws IOException, IoTDBConnectionException, StatementExecutionException {
+//
+// // String path = args[0];
+// String path = "/home/kyy/Documents/kdd/data/full-game.csv";
+// String measurements = "s6";
+//
+// String device = "root.game";
+//
+// Session session = new Session("127.0.0.1", 6667, "root", "root");
+// session.open(false);
+//
+// long minTime = -1;
+//
+// File f = new File(path);
+// String line = null;
+// BufferedReader reader = new BufferedReader(new FileReader(f));
+// while ((line = reader.readLine()) != null) {
+// String[] split = line.split(",");
+// long timestamp = Long.valueOf(split[1]);
+// if (minTime == -1) {
+// minTime = timestamp;
+// timestamp = 0;
+// } else {
+// timestamp = timestamp - minTime;
+// }
+// timestamp = (long) (timestamp / 1000);
+// long value = Long.valueOf(split[5]);
+// session.insertRecord(
+// device,
+// timestamp,
+// Collections.singletonList(measurements),
+// Collections.singletonList(TSDataType.INT64),
+// value);
+// }
+// session.executeNonQueryStatement("flush");
+// session.close();
+// }
+// }
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame_old.java
similarity index 99%
copy from example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java
copy to example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame_old.java
index ce37ba8..d2ea08a 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame_old.java
@@ -14,7 +14,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Random;
-public class WriteFullGame {
+public class WriteFullGame_old {
/** Before writing data, make sure check the server parameter configurations. */
public static void main(String[] args)
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteMF03.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteMF03.java
new file mode 100644
index 0000000..0322554
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteMF03.java
@@ -0,0 +1,116 @@
+package org.apache.iotdb.writeData;
+
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+// java -jar /data3/ruilei/rl/mf03_testspace/WriteMF03-0.12.4.jar
+// /data3/raw_data/rl/debs2012/mf03-3h-O_10_10 0 0 0 1 0
+public class WriteMF03 {
+
+ /** Before writing data, make sure check the server parameter configurations. */
+ public static void main(String[] args)
+ throws IoTDBConnectionException, StatementExecutionException, IOException {
+ String measurement = "mf03"; // [[update]]
+ String device = "root.debs2012"; // [[update]]
+
+ // 实验自变量1：乱序数据源
+ String filePath = args[0];
+ // 实验自变量2：delete percentage
+ int deletePercentage = Integer.parseInt(args[1]); // 0 means no deletes. 0-100
+ // 实验自变量3：每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ int deleteLenPercentage = Integer.parseInt(args[2]); // 0-100
+ // 参数4：时间戳idx，从0开始
+ int timeIdx = Integer.parseInt(args[3]);
+ // 参数5：值idx，从0开始
+ int valueIdx = Integer.parseInt(args[4]);
+ // 参数6：chunkAvgTimeLen
+ long chunkAvgTimeLen = Long.parseLong(args[5]);
+
+ if (deletePercentage < 0 || deletePercentage > 100) {
+ throw new IOException("WRONG deletePercentage!");
+ }
+ if (deleteLenPercentage < 0 || deleteLenPercentage > 100) {
+ throw new IOException("WRONG deleteLenPercentage!");
+ }
+
+ int avgSeriesPointNumberThreshold = 10000; // fixed
+ int deletePeriod =
+ (int) Math.floor(100 * 1.0 / deletePercentage * avgSeriesPointNumberThreshold);
+ long deleteLen = (long) Math.floor(chunkAvgTimeLen * deleteLenPercentage * 1.0 / 100);
+
+ List<String> deletePaths = new ArrayList<>();
+ deletePaths.add(device + "." + measurement);
+
+ TSDataType tsDataType = TSDataType.INT64; // value types
+
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open(false);
+
+ // this is to make all following inserts unseq chunks
+ session.insertRecord(
+ device,
+ 1644181628000000000L, // ns
+ // NOTE UPDATE TIME DATATYPE! [[update]]. DONT USE System.nanoTime()!
+ Collections.singletonList(measurement),
+ Collections.singletonList(tsDataType), // NOTE UPDATE VALUE DATATYPE!
+ 0L); // NOTE UPDATE VALUE DATATYPE!
+ session.executeNonQueryStatement("flush");
+
+ File f = new File(filePath);
+ String line = null;
+ BufferedReader reader = new BufferedReader(new FileReader(f));
+ long lastDeleteMinTime = Long.MAX_VALUE;
+ long lastDeleteMaxTime = Long.MIN_VALUE;
+ int cnt = 0;
+ while ((line = reader.readLine()) != null) {
+ String[] split = line.split(",");
+ long timestamp = Long.parseLong(split[timeIdx]);
+ long value = Long.parseLong(split[valueIdx]);
+ session.insertRecord(
+ device,
+ timestamp,
+ Collections.singletonList(measurement),
+ Collections.singletonList(tsDataType),
+ value);
+ cnt++;
+
+ if (timestamp > lastDeleteMaxTime) {
+ lastDeleteMaxTime = timestamp;
+ }
+ if (timestamp < lastDeleteMinTime) {
+ lastDeleteMinTime = timestamp;
+ }
+
+ if (deletePercentage != 0) {
+ if (cnt >= deletePeriod) {
+ cnt = 0;
+ // randomize deleteStartTime in [lastMinTime, max(lastMaxTime-deleteLen,lastMinTime+1)]
+ long rightBound = Math.max(lastDeleteMaxTime - deleteLen, lastDeleteMinTime + 1);
+ long deleteStartTime =
+ (long)
+ Math.ceil(
+ lastDeleteMinTime + Math.random() * (rightBound - lastDeleteMinTime + 1));
+ long deleteEndTime = deleteStartTime + deleteLen - 1;
+ session.deleteData(deletePaths, deleteStartTime, deleteEndTime);
+ System.out.println("[[[[delete]]]]]" + deleteStartTime + "," + deleteEndTime);
+
+ lastDeleteMinTime = Long.MAX_VALUE;
+ lastDeleteMaxTime = Long.MIN_VALUE;
+ }
+ }
+ }
+
+ session.executeNonQueryStatement("flush");
+ session.close();
+ }
+}
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteRecTime.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteRecTime.java
new file mode 100644
index 0000000..b49224b
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteRecTime.java
@@ -0,0 +1,114 @@
+package org.apache.iotdb.writeData;
+
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class WriteRecTime {
+
+ /** Before writing data, make sure check the server parameter configurations. */
+ public static void main(String[] args)
+ throws IoTDBConnectionException, StatementExecutionException, IOException {
+ String measurement = "RecTime"; // [[update]]
+ String device = "root.sg1"; // [[update]]
+
+ // 实验自变量1：乱序数据源
+ String filePath = args[0];
+ // 实验自变量2：delete percentage
+ int deletePercentage = Integer.parseInt(args[1]); // 0 means no deletes. 0-100
+ // 实验自变量3：每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ int deleteLenPercentage = Integer.parseInt(args[2]); // 0-100
+ // 参数4：时间戳idx，从0开始
+ int timeIdx = Integer.parseInt(args[3]);
+ // 参数5：值idx，从0开始
+ int valueIdx = Integer.parseInt(args[4]);
+ // 参数6：chunkAvgTimeLen
+ long chunkAvgTimeLen = Long.parseLong(args[5]);
+
+ if (deletePercentage < 0 || deletePercentage > 100) {
+ throw new IOException("WRONG deletePercentage!");
+ }
+ if (deleteLenPercentage < 0 || deleteLenPercentage > 100) {
+ throw new IOException("WRONG deleteLenPercentage!");
+ }
+
+ int avgSeriesPointNumberThreshold = 10000; // fixed
+ int deletePeriod =
+ (int) Math.floor(100 * 1.0 / deletePercentage * avgSeriesPointNumberThreshold);
+ long deleteLen = (long) Math.floor(chunkAvgTimeLen * deleteLenPercentage * 1.0 / 100);
+
+ List<String> deletePaths = new ArrayList<>();
+ deletePaths.add(device + "." + measurement);
+
+ TSDataType tsDataType = TSDataType.INT64; // value types
+
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open(false);
+
+ // this is to make all following inserts unseq chunks
+ session.insertRecord(
+ device,
+ 1644181628000L, // ms
+ // NOTE UPDATE TIME DATATYPE! [[update]]. DONT USE System.nanoTime()!
+ Collections.singletonList(measurement),
+ Collections.singletonList(tsDataType), // NOTE UPDATE VALUE DATATYPE!
+ 0L); // NOTE UPDATE VALUE DATATYPE!
+ session.executeNonQueryStatement("flush");
+
+ File f = new File(filePath);
+ String line = null;
+ BufferedReader reader = new BufferedReader(new FileReader(f));
+ long lastDeleteMinTime = Long.MAX_VALUE;
+ long lastDeleteMaxTime = Long.MIN_VALUE;
+ int cnt = 0;
+ while ((line = reader.readLine()) != null) {
+ String[] split = line.split(",");
+ long timestamp = Long.parseLong(split[timeIdx]);
+ long value = Long.parseLong(split[valueIdx]);
+ session.insertRecord(
+ device,
+ timestamp,
+ Collections.singletonList(measurement),
+ Collections.singletonList(tsDataType),
+ value);
+ cnt++;
+
+ if (timestamp > lastDeleteMaxTime) {
+ lastDeleteMaxTime = timestamp;
+ }
+ if (timestamp < lastDeleteMinTime) {
+ lastDeleteMinTime = timestamp;
+ }
+
+ if (deletePercentage != 0) {
+ if (cnt >= deletePeriod) {
+ cnt = 0;
+ // randomize deleteStartTime in [lastMinTime, max(lastMaxTime-deleteLen,lastMinTime+1)]
+ long rightBound = Math.max(lastDeleteMaxTime - deleteLen, lastDeleteMinTime + 1);
+ long deleteStartTime =
+ (long)
+ Math.ceil(
+ lastDeleteMinTime + Math.random() * (rightBound - lastDeleteMinTime + 1));
+ long deleteEndTime = deleteStartTime + deleteLen - 1;
+ session.deleteData(deletePaths, deleteStartTime, deleteEndTime);
+ System.out.println("[[[[delete]]]]]" + deleteStartTime + "," + deleteEndTime);
+
+ lastDeleteMinTime = Long.MAX_VALUE;
+ lastDeleteMaxTime = Long.MIN_VALUE;
+ }
+ }
+ }
+
+ session.executeNonQueryStatement("flush");
+ session.close();
+ }
+}
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteSyntheticData1.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteSyntheticData1.java
index e72acc3..fbc9664 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/WriteSyntheticData1.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteSyntheticData1.java
@@ -1,46 +1,46 @@
-package org.apache.iotdb.writeData;
-
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.session.Session;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
-import java.util.Collections;
-
-public class WriteSyntheticData1 {
-
- /**
- * !!!!!!!Before writing data, make sure check the following server parameters:
- *
- * <p>system_dir=/data3/ruilei/iotdb-server-0.12.4/synData1/system
- * data_dirs=/data3/ruilei/iotdb-server-0.12.4/synData1/data
- * wal_dir=/data3/ruilei/iotdb-server-0.12.4/synData1/wal timestamp_precision=ms
- * unseq_tsfile_size=1073741824 # maximum size of unseq TsFile is 1024^3 Bytes
- * seq_tsfile_size=1073741824 # maximum size of seq TsFile is 1024^3 Bytes
- * avg_series_point_number_threshold=10000 # each chunk contains 10000 data points
- * compaction_strategy=NO_COMPACTION # compaction between levels is disabled
- * enable_unseq_compaction=false # unseq compaction is disabled
- */
- public static final String device = "root.vehicle.d0";
-
- public static final String measurements = "s0";
-
- public static void main(String[] args)
- throws IoTDBConnectionException, StatementExecutionException {
-
- Session session = new Session("127.0.0.1", 6667, "root", "root");
- session.open(false);
-
- for (long timestamp = 0; timestamp < 10000000; timestamp++) {
- double value = Math.random();
- session.insertRecord(
- device,
- timestamp,
- Collections.singletonList(measurements),
- Collections.singletonList(TSDataType.DOUBLE),
- value);
- }
- session.executeNonQueryStatement("flush");
- session.close();
- }
-}
+// package org.apache.iotdb.writeData;
+//
+// import org.apache.iotdb.rpc.IoTDBConnectionException;
+// import org.apache.iotdb.rpc.StatementExecutionException;
+// import org.apache.iotdb.session.Session;
+// import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+//
+// import java.util.Collections;
+//
+// public class WriteSyntheticData1 {
+//
+// /**
+// * !!!!!!!Before writing data, make sure check the following server parameters:
+// *
+// * <p>system_dir=/data3/ruilei/iotdb-server-0.12.4/synData1/system
+// * data_dirs=/data3/ruilei/iotdb-server-0.12.4/synData1/data
+// * wal_dir=/data3/ruilei/iotdb-server-0.12.4/synData1/wal timestamp_precision=ms
+// * unseq_tsfile_size=1073741824 # maximum size of unseq TsFile is 1024^3 Bytes
+// * seq_tsfile_size=1073741824 # maximum size of seq TsFile is 1024^3 Bytes
+// * avg_series_point_number_threshold=10000 # each chunk contains 10000 data points
+// * compaction_strategy=NO_COMPACTION # compaction between levels is disabled
+// * enable_unseq_compaction=false # unseq compaction is disabled
+// */
+// public static final String device = "root.vehicle.d0";
+//
+// public static final String measurements = "s0";
+//
+// public static void main(String[] args)
+// throws IoTDBConnectionException, StatementExecutionException {
+//
+// Session session = new Session("127.0.0.1", 6667, "root", "root");
+// session.open(false);
+//
+// for (long timestamp = 0; timestamp < 10000000; timestamp++) {
+// double value = Math.random();
+// session.insertRecord(
+// device,
+// timestamp,
+// Collections.singletonList(measurements),
+// Collections.singletonList(TSDataType.DOUBLE),
+// value);
+// }
+// session.executeNonQueryStatement("flush");
+// session.close();
+// }
+// }
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteSyntheticDataWithDeletes.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteSyntheticDataWithDeletes.java
index 2a24c6d..3a6f918 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/WriteSyntheticDataWithDeletes.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteSyntheticDataWithDeletes.java
@@ -1,63 +1,64 @@
-package org.apache.iotdb.writeData;
-
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
-import org.apache.iotdb.session.Session;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-
-public class WriteSyntheticDataWithDeletes {
-
- /**
- * !!!!!!!Before writing data, make sure check the following server parameters:
- *
- * <p>system_dir data_dirs wal_dir timestamp_precision=ms unseq_tsfile_size=1073741824 # maximum
- * size of unseq TsFile is 1024^3 Bytes seq_tsfile_size=1073741824 # maximum size of seq TsFile is
- * 1024^3 Bytes avg_series_point_number_threshold=10000 # each chunk contains 10000 data points
- * compaction_strategy=NO_COMPACTION # compaction between levels is disabled
- * enable_unseq_compaction=false # unseq compaction is disabled
- */
- public static final String device = "root.vehicle.d0";
-
- public static final String measurements = "s0";
-
- public static void main(String[] args)
- throws IoTDBConnectionException, StatementExecutionException {
- // 实验自变量1：每隔deleteFreq时间（本实验时间单位ms）就执行一次删除
- long deleteFreq = Long.parseLong(args[0]);
- // 实验自变量2：每次删除的时间长度
- long deleteLen = Long.parseLong(args[1]);
-
- Session session = new Session("127.0.0.1", 6667, "root", "root");
- session.open(false);
-
- List<String> paths = new ArrayList<>();
- paths.add(device + "." + measurements);
-
- for (long timestamp = 0; timestamp < 10000000; timestamp++) {
- if (timestamp != 0 && timestamp % deleteFreq == 0) {
- // [timestamp-deleteFreq, timestamp-1]内随机取一个删除时间起点
- long min = timestamp - deleteFreq;
- long max = timestamp - 1;
- long deleteStartTime = min + (((long) (new Random().nextDouble() * (max - min))));
- // delete length 5000ms, half the time length of a chunk
- long deleteEndTime = deleteStartTime + deleteLen;
- session.deleteData(paths, deleteStartTime, deleteEndTime);
- }
-
- double value = Math.random();
- session.insertRecord(
- device,
- timestamp,
- Collections.singletonList(measurements),
- Collections.singletonList(TSDataType.DOUBLE),
- value);
- }
- session.executeNonQueryStatement("flush");
- session.close();
- }
-}
+// package org.apache.iotdb.writeData;
+//
+// import org.apache.iotdb.rpc.IoTDBConnectionException;
+// import org.apache.iotdb.rpc.StatementExecutionException;
+// import org.apache.iotdb.session.Session;
+// import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+//
+// import java.util.ArrayList;
+// import java.util.Collections;
+// import java.util.List;
+// import java.util.Random;
+//
+// public class WriteSyntheticDataWithDeletes {
+//
+// /**
+// * !!!!!!!Before writing data, make sure check the following server parameters:
+// *
+// * <p>system_dir data_dirs wal_dir timestamp_precision=ms unseq_tsfile_size=1073741824 # maximum
+// * size of unseq TsFile is 1024^3 Bytes seq_tsfile_size=1073741824 # maximum size of seq TsFile
+// is
+// * 1024^3 Bytes avg_series_point_number_threshold=10000 # each chunk contains 10000 data points
+// * compaction_strategy=NO_COMPACTION # compaction between levels is disabled
+// * enable_unseq_compaction=false # unseq compaction is disabled
+// */
+// public static final String device = "root.vehicle.d0";
+//
+// public static final String measurements = "s0";
+//
+// public static void main(String[] args)
+// throws IoTDBConnectionException, StatementExecutionException {
+// // 实验自变量1：每隔deleteFreq时间（本实验时间单位ms）就执行一次删除
+// long deleteFreq = Long.parseLong(args[0]);
+// // 实验自变量2：每次删除的时间长度
+// long deleteLen = Long.parseLong(args[1]);
+//
+// Session session = new Session("127.0.0.1", 6667, "root", "root");
+// session.open(false);
+//
+// List<String> paths = new ArrayList<>();
+// paths.add(device + "." + measurements);
+//
+// for (long timestamp = 0; timestamp < 10000000; timestamp++) {
+// if (timestamp != 0 && timestamp % deleteFreq == 0) {
+// // [timestamp-deleteFreq, timestamp-1]内随机取一个删除时间起点
+// long min = timestamp - deleteFreq;
+// long max = timestamp - 1;
+// long deleteStartTime = min + (((long) (new Random().nextDouble() * (max - min))));
+// // delete length 5000ms, half the time length of a chunk
+// long deleteEndTime = deleteStartTime + deleteLen;
+// session.deleteData(paths, deleteStartTime, deleteEndTime);
+// }
+//
+// double value = Math.random();
+// session.insertRecord(
+// device,
+// timestamp,
+// Collections.singletonList(measurements),
+// Collections.singletonList(TSDataType.DOUBLE),
+// value);
+// }
+// session.executeNonQueryStatement("flush");
+// session.close();
+// }
+// }
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteTianyuan.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteTianyuan.java
new file mode 100644
index 0000000..8c4dff3
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteTianyuan.java
@@ -0,0 +1,114 @@
+package org.apache.iotdb.writeData;
+
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class WriteTianyuan {
+
+ /** Before writing data, make sure check the server parameter configurations. */
+ public static void main(String[] args)
+ throws IoTDBConnectionException, StatementExecutionException, IOException {
+ String measurement = "KOB_0002_00_67"; // [[update]]
+ String device = "root.kobelco.trans.03.1090001603.2401604"; // [[update]]
+
+ // 实验自变量1：乱序数据源
+ String filePath = args[0];
+ // 实验自变量2：delete percentage
+ int deletePercentage = Integer.parseInt(args[1]); // 0 means no deletes. 0-100
+ // 实验自变量3：每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ int deleteLenPercentage = Integer.parseInt(args[2]); // 0-100
+ // 参数4：时间戳idx，从0开始
+ int timeIdx = Integer.parseInt(args[3]);
+ // 参数5：值idx，从0开始
+ int valueIdx = Integer.parseInt(args[4]);
+ // 参数6：chunkAvgTimeLen
+ long chunkAvgTimeLen = Long.parseLong(args[5]);
+
+ if (deletePercentage < 0 || deletePercentage > 100) {
+ throw new IOException("WRONG deletePercentage!");
+ }
+ if (deleteLenPercentage < 0 || deleteLenPercentage > 100) {
+ throw new IOException("WRONG deleteLenPercentage!");
+ }
+
+ int avgSeriesPointNumberThreshold = 10000; // fixed
+ int deletePeriod =
+ (int) Math.floor(100 * 1.0 / deletePercentage * avgSeriesPointNumberThreshold);
+ long deleteLen = (long) Math.floor(chunkAvgTimeLen * deleteLenPercentage * 1.0 / 100);
+
+ List<String> deletePaths = new ArrayList<>();
+ deletePaths.add(device + "." + measurement);
+
+ TSDataType tsDataType = TSDataType.INT64; // value types
+
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open(false);
+
+ // this is to make all following inserts unseq chunks
+ session.insertRecord(
+ device,
+ 1644181628000L, // ms
+ // NOTE UPDATE TIME DATATYPE! [[update]]. DONT USE System.nanoTime()!
+ Collections.singletonList(measurement),
+ Collections.singletonList(tsDataType), // NOTE UPDATE VALUE DATATYPE!
+ 0L); // NOTE UPDATE VALUE DATATYPE!
+ session.executeNonQueryStatement("flush");
+
+ File f = new File(filePath);
+ String line = null;
+ BufferedReader reader = new BufferedReader(new FileReader(f));
+ long lastDeleteMinTime = Long.MAX_VALUE;
+ long lastDeleteMaxTime = Long.MIN_VALUE;
+ int cnt = 0;
+ while ((line = reader.readLine()) != null) {
+ String[] split = line.split(",");
+ long timestamp = Long.parseLong(split[timeIdx]);
+ long value = Long.parseLong(split[valueIdx]);
+ session.insertRecord(
+ device,
+ timestamp,
+ Collections.singletonList(measurement),
+ Collections.singletonList(tsDataType),
+ value);
+ cnt++;
+
+ if (timestamp > lastDeleteMaxTime) {
+ lastDeleteMaxTime = timestamp;
+ }
+ if (timestamp < lastDeleteMinTime) {
+ lastDeleteMinTime = timestamp;
+ }
+
+ if (deletePercentage != 0) {
+ if (cnt >= deletePeriod) {
+ cnt = 0;
+ // randomize deleteStartTime in [lastMinTime, max(lastMaxTime-deleteLen,lastMinTime+1)]
+ long rightBound = Math.max(lastDeleteMaxTime - deleteLen, lastDeleteMinTime + 1);
+ long deleteStartTime =
+ (long)
+ Math.ceil(
+ lastDeleteMinTime + Math.random() * (rightBound - lastDeleteMinTime + 1));
+ long deleteEndTime = deleteStartTime + deleteLen - 1;
+ session.deleteData(deletePaths, deleteStartTime, deleteEndTime);
+ System.out.println("[[[[delete]]]]]" + deleteStartTime + "," + deleteEndTime);
+
+ lastDeleteMinTime = Long.MAX_VALUE;
+ lastDeleteMaxTime = Long.MIN_VALUE;
+ }
+ }
+ }
+
+ session.executeNonQueryStatement("flush");
+ session.close();
+ }
+}