You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2022/02/14 04:06:50 UTC
[iotdb] 16/32: update
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d7cd2b1caa1cb5d387291e7c96e97313762ee12f
Author: Lei Rui <10...@qq.com>
AuthorDate: Sat Feb 5 00:57:23 2022 +0800
update
---
.../iotdb/writeData/ExtractFullGameData.java | 13 ++-
.../org/apache/iotdb/writeData/WriteFullGame.java | 20 +++-
.../apache/iotdb/writeData/WriteOverlapChunks.java | 108 +++++++++++++++++++++
.../iotdb/db/integration/m4/MyDebugTest2.java | 35 +++----
4 files changed, 152 insertions(+), 24 deletions(-)
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/ExtractFullGameData.java b/example/session/src/main/java/org/apache/iotdb/writeData/ExtractFullGameData.java
index d343070..a7f70b5 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/ExtractFullGameData.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/ExtractFullGameData.java
@@ -7,24 +7,31 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
+// java ExtractFullGameData /data3/raw_data/data/full-game full-game-small 1 5 495760
public class ExtractFullGameData {
public static void main(String[] args) throws IOException {
String inPath = args[0];
String outPath = args[1];
+ int timeIdx = Integer.parseInt(args[2]);
+ int valueIdx = Integer.parseInt(args[3]);
+ int lineNum = Integer.parseInt(args[4]); // -1 means no constraint
+
File f = new File(inPath);
FileWriter fileWriter = new FileWriter(outPath);
String line;
BufferedReader reader = new BufferedReader(new FileReader(f));
PrintWriter printWriter = new PrintWriter(fileWriter);
- while ((line = reader.readLine()) != null) {
+ int cnt = 0;
+ while (((lineNum > 0 && cnt < lineNum) || lineNum < 0) && (line = reader.readLine()) != null) {
String[] split = line.split(",");
- long timestamp = Long.valueOf(split[3]);
- long value = Long.valueOf(split[4]);
+ long timestamp = Long.parseLong(split[timeIdx]);
+ long value = Long.parseLong(split[valueIdx]);
printWriter.print(timestamp);
printWriter.print(",");
printWriter.print(value);
printWriter.println();
+ cnt++;
}
reader.close();
printWriter.close();
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java
index 899cc84..c2ae869 100644
--- a/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteFullGame.java
@@ -36,18 +36,29 @@ public class WriteFullGame {
// 参数5：值idx，从0开始
int valueIdx = Integer.parseInt(args[4]);
+ TSDataType tsDataType = TSDataType.INT64;
+
Session session = new Session("127.0.0.1", 6667, "root", "root");
session.open(false);
+ // this is to make all following inserts unseq chunks
+ session.insertRecord(
+ device,
+ System.nanoTime(),
+ Collections.singletonList(measurements),
+ Collections.singletonList(tsDataType), // NOTE UPDATE VALUE DATATYPE!
+ 0L); // NOTE UPDATE VALUE DATATYPE!!!
+ session.executeNonQueryStatement("flush");
+
long minTime = -1;
File f = new File(filePath);
String line = null;
BufferedReader reader = new BufferedReader(new FileReader(f));
while ((line = reader.readLine()) != null) {
String[] split = line.split(",");
- long timestamp = Long.valueOf(split[timeIdx]);
+ long timestamp = Long.parseLong(split[timeIdx]);
if (minTime == -1) {
- minTime = timestamp;
+ minTime = timestamp; // assume first timestamp is never disordered. is global minimal.
timestamp = 0;
} else {
timestamp = timestamp - minTime;
@@ -63,12 +74,13 @@ public class WriteFullGame {
session.deleteData(deletePaths, deleteStartTime, deleteEndTime);
}
- long value = Long.valueOf(split[valueIdx]);
+ long value = Long.parseLong(split[valueIdx]);
+
session.insertRecord(
device,
timestamp,
Collections.singletonList(measurements),
- Collections.singletonList(TSDataType.INT64),
+ Collections.singletonList(tsDataType),
value);
}
diff --git a/example/session/src/main/java/org/apache/iotdb/writeData/WriteOverlapChunks.java b/example/session/src/main/java/org/apache/iotdb/writeData/WriteOverlapChunks.java
new file mode 100644
index 0000000..288cb82
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/writeData/WriteOverlapChunks.java
@@ -0,0 +1,108 @@
+package org.apache.iotdb.writeData;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * 每次快要写到10000个点（chunk size）还剩下OD%个点的时候，OP%的概率决定要不要乱序. 如果要乱序，
+ * 方法是剩下的这OD%个点，和接下来再OD%个点，合在一起2*OD%个点随机打乱然后再写。 0<=OP<=100 0<OD<=50.
+ *
+ * <p>(1) WriteFullGameToIoTDB. (2) java ExtractFullGameData /data3/raw_data/data/full-game
+ * full-game-small 1 5 495760. (3) java WriteOverlapChunks D:\exp\full-game-small
+ * D:\exp\full-game-small-OP90-OD50 0 1 90 50.
+ */
+public class WriteOverlapChunks {
+
+ public static void main(String[] args) throws IOException {
+ // default
+ int avgSeriesPointNumberThreshold = 10000;
+
+ String inPath = args[0];
+ String outPath = args[1];
+ int timeIdx = Integer.parseInt(args[2]);
+ int valueIdx = Integer.parseInt(args[3]);
+ int overlapPercentage = Integer.parseInt(args[4]); // 0-100
+ int overlapDepth = Integer.parseInt(args[5]); // 0-50
+
+ if (overlapPercentage < 0 || overlapPercentage > 100) {
+ throw new IOException("WRONG overlapPercentage");
+ }
+ if (overlapDepth > 50 || overlapDepth < 0) { // 暂时只实现最多相邻两个chunk重叠，重叠深度从0到50%
+ throw new IOException("WRONG pointNum");
+ }
+ int pointNum = (int) Math.floor(avgSeriesPointNumberThreshold * (overlapDepth / 100.0));
+
+ File f = new File(inPath);
+ FileWriter fileWriter = new FileWriter(outPath);
+ String line;
+ BufferedReader reader = new BufferedReader(new FileReader(f));
+ PrintWriter printWriter = new PrintWriter(fileWriter);
+ int cnt = 0;
+ while ((line = reader.readLine()) != null) {
+ String[] split = line.split(",");
+ long timestamp = Long.parseLong(split[timeIdx]); // time
+ long value = Long.parseLong(split[valueIdx]); // value
+
+ // note the first timestamp is never disordered. is global minimal.
+ printWriter.print(timestamp);
+ printWriter.print(",");
+ printWriter.print(value);
+ printWriter.println();
+ cnt++;
+
+ if (overlapPercentage != 0) {
+ while (cnt % avgSeriesPointNumberThreshold == avgSeriesPointNumberThreshold - pointNum) {
+ if (new Random().nextDouble() <= overlapPercentage / 100.0) {
+ // disturb the next 2*pointNum points
+ long[] timestampArray = new long[2 * pointNum];
+ long[] valueArray = new long[2 * pointNum];
+ List<Integer> idx = new ArrayList<>();
+ int n = 0;
+ while (n < 2 * pointNum && (line = reader.readLine()) != null) {
+ // don't change the sequence of the above two conditions
+ split = line.split(",");
+ timestampArray[n] = Long.parseLong(split[timeIdx]);
+ valueArray[n] = Long.parseLong(split[valueIdx]);
+ idx.add(n);
+ n++;
+ }
+ Collections.shuffle(idx);
+ for (Integer integer : idx) {
+ int k = integer;
+ printWriter.print(timestampArray[k]);
+ printWriter.print(",");
+ printWriter.print(valueArray[k]);
+ printWriter.println();
+ cnt++;
+ }
+ } else {
+ int n = 0;
+ while (n < 2 * pointNum && (line = reader.readLine()) != null) {
+ // don't change the sequence of the above two conditions
+ split = line.split(",");
+ timestamp = Long.parseLong(split[timeIdx]); // time
+ value = Long.parseLong(split[valueIdx]); // value
+ printWriter.print(timestamp);
+ printWriter.print(",");
+ printWriter.print(value);
+ printWriter.println();
+ cnt++;
+ n++;
+ }
+ }
+ }
+ }
+ }
+ System.out.println(cnt);
+ reader.close();
+ printWriter.close();
+ }
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyDebugTest2.java b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyDebugTest2.java
index 13af30a..d97ec68 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/m4/MyDebugTest2.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/m4/MyDebugTest2.java
@@ -19,34 +19,35 @@
package org.apache.iotdb.db.integration.m4;
-import static org.junit.Assert.fail;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.Statement;
-import java.util.Locale;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Locale;
+
+import static org.junit.Assert.fail;
+
public class MyDebugTest2 {
private static final String TIMESTAMP_STR = "Time";
private static String[] creationSqls =
- new String[]{
- "SET STORAGE GROUP TO root.game",
- "CREATE TIMESERIES root.game.s6 WITH DATATYPE=INT64",
+ new String[] {
+ "SET STORAGE GROUP TO root.game", "CREATE TIMESERIES root.game.s6 WITH DATATYPE=INT64",
};
private final String d0s0 = "root.vehicle.d0.s0";
@@ -96,9 +97,9 @@ public class MyDebugTest2 {
public void test1() {
// 现像：fullGame实验中，CPV第一个interval结果还是对的，第二个Interval开始结果不对了
- String[] res = new String[]{};
+ String[] res = new String[] {};
try (Connection connection =
- DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
Statement statement = connection.createStatement()) {
boolean hasResultSet =
statement.execute(
@@ -135,8 +136,8 @@ public class MyDebugTest2 {
private static void prepareData1() {
try (Connection connection =
- DriverManager.getConnection(
- Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
Statement statement = connection.createStatement()) {
for (String sql : creationSqls) {