You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yu...@apache.org on 2022/01/27 12:00:05 UTC
[iotdb] branch kyy-2022 updated: experiment
This is an automated email from the ASF dual-hosted git repository.
yuyuankang pushed a commit to branch kyy-2022
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/kyy-2022 by this push:
new 967598b experiment
967598b is described below
commit 967598b86d820781cd00f3d13e75494db1b3d943
Author: yuyuankang <yu...@hotmail.com>
AuthorDate: Thu Jan 27 18:55:26 2022 +0800
experiment
---
example/session/pom.xml | 24 +++++
.../src/main/java/org/apache/iotdb/MyTest.java | 109 +++++++++++++++++++++
.../main/java/org/apache/iotdb/ProcessResult.java | 37 +++++++
.../java/org/apache/iotdb/QueryExperiment.java | 35 ++++---
.../java/org/apache/iotdb/WriteCSVToIoTDB.java | 46 +++++++++
.../resources/conf/iotdb-engine.properties | 10 +-
.../main/java/org/apache/iotdb/db/IOMonitor.java | 22 ++---
.../apache/iotdb/db/tools/TsFileSketchTool.java | 2 +-
8 files changed, 254 insertions(+), 31 deletions(-)
diff --git a/example/session/pom.xml b/example/session/pom.xml
index 139470c..3521841 100644
--- a/example/session/pom.xml
+++ b/example/session/pom.xml
@@ -40,4 +40,28 @@
<version>${project.version}</version>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>1.2.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.apache.iotdb.QueryExperiment</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/example/session/src/main/java/org/apache/iotdb/MyTest.java b/example/session/src/main/java/org/apache/iotdb/MyTest.java
new file mode 100644
index 0000000..e553be0
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/MyTest.java
@@ -0,0 +1,109 @@
+// package org.apache.iotdb;/*
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+//
+// import org.apache.iotdb.db.conf.IoTDBConfig;
+// import org.apache.iotdb.db.conf.IoTDBDescriptor;
+// import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
+// import org.apache.iotdb.db.utils.EnvironmentUtils;
+// import org.apache.iotdb.jdbc.Config;
+// import org.junit.AfterClass;
+// import org.junit.BeforeClass;
+// import org.junit.Test;
+//
+// import java.sql.Connection;
+// import java.sql.DriverManager;
+// import java.sql.Statement;
+// import java.util.Locale;
+//
+// public class MyTest {
+// private static String[] creationSqls =
+// new String[]{
+// "SET STORAGE GROUP TO root.vehicle.d0",
+// "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=FLOAT, ENCODING=plain",
+// };
+//
+// private final String d0s0 = "root.vehicle.d0.s0";
+//
+// private static final String insertTemplate =
+// "INSERT INTO root.vehicle.d0(timestamp,s0)" + " VALUES(%d,%f)";
+//
+// private static final IoTDBConfig ioTDBConfig = IoTDBDescriptor.getInstance().getConfig();
+// private static int avgSeriesPointNumberThreshold;
+//
+// @BeforeClass
+// public static void setUp() throws Exception {
+// avgSeriesPointNumberThreshold = ioTDBConfig.getAvgSeriesPointNumberThreshold();
+//
+//
+// IoTDBDescriptor.getInstance().getConfig().setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
+//
+// EnvironmentUtils.envSetUp();
+//
+// Class.forName(Config.JDBC_DRIVER_NAME);
+// }
+//
+// @AfterClass
+// public static void tearDown() throws Exception {
+//// EnvironmentUtils.cleanEnv();
+// IoTDBDescriptor.getInstance()
+// .getConfig()
+// .setCompactionStrategy(CompactionStrategy.LEVEL_COMPACTION);
+//
+// ioTDBConfig.setAvgSeriesPointNumberThreshold(avgSeriesPointNumberThreshold);
+// }
+//
+// @Test
+// public void test1() {
+// prepareData1();
+// }
+//
+// private static void prepareData1() {
+// try (Connection connection =
+// DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+// Statement statement = connection.createStatement()) {
+//
+// String[] creationSqls =
+// new String[]{
+// "SET STORAGE GROUP TO root.vehicle.d0",
+// "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=FLOAT,
+// ENCODING=plain",
+// };
+// for (String sql : creationSqls) {
+// statement.execute(sql);
+// }
+//
+// String insertTemplate =
+// "INSERT INTO root.vehicle.d0(timestamp,s0)" + " VALUES(%d,%f)";
+//
+// ioTDBConfig.setSeqTsFileSize(1024*1024*1024);// 1G
+// ioTDBConfig.setUnSeqTsFileSize(1024*1024*1024); // 1G
+// ioTDBConfig.setAvgSeriesPointNumberThreshold(10000); // this step cannot be omitted
+//
+// for (int i = 1; i <= 50000; i++) {
+// statement.addBatch(String.format(Locale.ENGLISH, insertTemplate, i,
+// Math.random()));
+// }
+// statement.executeBatch();
+// statement.clearBatch();
+// statement.execute("flush");
+// } catch (Exception e) {
+// e.printStackTrace();
+// }
+// }
+// }
diff --git a/example/session/src/main/java/org/apache/iotdb/ProcessResult.java b/example/session/src/main/java/org/apache/iotdb/ProcessResult.java
new file mode 100644
index 0000000..bd0d3ca
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/ProcessResult.java
@@ -0,0 +1,37 @@
+package org.apache.iotdb;
+
+import java.io.*;
+
+public class ProcessResult {
+
+ public static final String BASE_PATH = "/home/kyy/Documents/kdd";
+
+ public static void main(String[] args) throws IOException {
+
+ String inFilePath = BASE_PATH + File.separator + "result.txt";
+ String outFilePath = BASE_PATH + File.separator + "out.txt";
+
+ BufferedReader reader = new BufferedReader(new FileReader(inFilePath));
+ FileWriter writer = new FileWriter(outFilePath);
+ String readLine = null;
+ long metaTime = 0;
+ long dataTime = 0;
+ long totalTime = 0;
+ int counter = 0;
+ while ((readLine = reader.readLine()) != null) {
+ if (readLine.startsWith("select")) {
+ String[] values = readLine.split("\t");
+ metaTime += Long.parseLong(values[1].split(": ")[1]);
+ dataTime += Long.parseLong(values[3].split(": ")[1]);
+ totalTime += Long.parseLong(values[5].split(": ")[1]);
+ counter++;
+ writer.write(readLine + "\n");
+ }
+ }
+ writer.write("avg_meta: " + (double) metaTime / counter
+ + "\t avg_data: " + (double) dataTime / counter
+ + "\t avg_total: " + (double) totalTime / counter);
+ reader.close();
+ writer.close();
+ }
+}
diff --git a/example/session/src/main/java/org/apache/iotdb/QueryExperiment.java b/example/session/src/main/java/org/apache/iotdb/QueryExperiment.java
index bc0e525..303db60 100644
--- a/example/session/src/main/java/org/apache/iotdb/QueryExperiment.java
+++ b/example/session/src/main/java/org/apache/iotdb/QueryExperiment.java
@@ -21,8 +21,8 @@ public class QueryExperiment {
+ "group by ([%d, %d), %dms)";
private static final long totalIntervalLengthMS = 1 * 60 * 60 * 1000L;
private static final int totalIntervalNumber = 1000;
- private static Random random = new Random();
- private static final int sqlNum = 100;
+ private static Random random = new Random(System.currentTimeMillis());
+ private static final int sqlNum = 1;
public static void main(String[] args)
throws IoTDBConnectionException, StatementExecutionException, TException {
@@ -34,18 +34,25 @@ public class QueryExperiment {
session = new Session("127.0.0.1", 6667, "root", "root");
session.open(false);
- SessionDataSet dataSet =
- session.executeQueryStatement(
- "select min_time(" + measurement + "), max_time(" + measurement + ") from " + device);
- long minTime = -1;
- long maxTime = -1;
- while (dataSet.hasNext()) {
- RowRecord r = dataSet.next();
- minTime = r.getFields().get(0).getLongV();
- maxTime = r.getFields().get(1).getLongV();
- }
- assert minTime != -1;
- assert maxTime != -1;
+ SessionDataSet dataSet;
+
+ // dataSet =
+ // session.executeQueryStatement(
+ // "select min_time(" + measurement + "), max_time(" + measurement + ") from " +
+ // device);
+ // long minTime = -1;
+ // long maxTime = -1;
+ // while (dataSet.hasNext()) {
+ // RowRecord r = dataSet.next();
+ // minTime = r.getFields().get(0).getLongV();
+ // maxTime = r.getFields().get(1).getLongV();
+ // System.out.println(minTime + ", " + maxTime);
+ // }
+ // assert minTime != -1;
+ // assert maxTime != -1;
+
+ long minTime = 1616805035973L;
+ long maxTime = 1627380839563L;
for (int i = 0; i < sqlNum; i++) {
long startTime = nextLong(minTime, maxTime - totalIntervalLengthMS);
diff --git a/example/session/src/main/java/org/apache/iotdb/WriteCSVToIoTDB.java b/example/session/src/main/java/org/apache/iotdb/WriteCSVToIoTDB.java
new file mode 100644
index 0000000..f4369de
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/WriteCSVToIoTDB.java
@@ -0,0 +1,46 @@
+package org.apache.iotdb;
+
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.Session;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Collections;
+
+public class WriteCSVToIoTDB {
+
+ public static final String fullPath = "root.kobelco.trans.03.1090001603.2401604.KOB_0002_00_67";
+ public static final String path = "/home/kyy/Documents/kdd/data/" + fullPath + ".csv";
+
+ public static void main(String[] args)
+ throws IOException, IoTDBConnectionException, StatementExecutionException {
+
+ String[] nodes = fullPath.split("\\.");
+ String measurements = nodes[nodes.length - 1];
+ String device = fullPath.replace("." + measurements, "");
+
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open(false);
+
+ File f = new File(path);
+ String line = null;
+ BufferedReader reader = new BufferedReader(new FileReader(f));
+ while ((line = reader.readLine()) != null) {
+ String[] split = line.split(",");
+ long timestamp = Long.valueOf(split[0]);
+ long value = Long.valueOf(split[1]);
+ session.insertRecord(
+ device,
+ timestamp,
+ Collections.singletonList(measurements),
+ Collections.singletonList(TSDataType.INT64),
+ value);
+ }
+ session.executeNonQueryStatement("flush");
+ session.close();
+ }
+}
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 8e63e07..28a3195 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -185,11 +185,11 @@ timestamp_precision=ms
# When a unSequence TsFile's file size (in byte) exceeds this, the TsFile is forced closed.
# It may cause memTable size smaller if it is a large value
-# unseq_tsfile_size=1
+unseq_tsfile_size=1073741824
# When a sequence TsFile's file size (in byte) exceeds this, the TsFile is forced closed.
# It may cause memTable size smaller if it is a large value
-# seq_tsfile_size=1
+seq_tsfile_size=1073741824
# Size of log buffer in each metadata operation plan(in byte).
# If the size of a metadata operation plan is larger than this parameter, then it will be rejected by MManager
@@ -244,7 +244,7 @@ timestamp_precision=ms
# close_tsfile_check_interval_in_ms=600000
# When the average point number of timeseries in memtable exceeds this, the memtable is flushed to disk. The default threshold is 10000.
-# avg_series_point_number_threshold=10000
+avg_series_point_number_threshold=10000
# How many threads can concurrently flush. When <= 0, use CPU core number.
# concurrent_flush_thread=0
@@ -364,11 +364,11 @@ timestamp_precision=ms
### Merge Configurations
####################
# LEVEL_COMPACTION, NO_COMPACTION
-# compaction_strategy=LEVEL_COMPACTION
+compaction_strategy=NO_COMPACTION
# Works when the compaction_strategy is LEVEL_COMPACTION.
# Whether to merge unseq files into seq files or not.
-# enable_unseq_compaction=true
+enable_unseq_compaction=false
# Start compaction task at this delay, unit is ms
# compaction_interval=30000
diff --git a/server/src/main/java/org/apache/iotdb/db/IOMonitor.java b/server/src/main/java/org/apache/iotdb/db/IOMonitor.java
index 760074d..d9627d8 100644
--- a/server/src/main/java/org/apache/iotdb/db/IOMonitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/IOMonitor.java
@@ -128,18 +128,18 @@ public class IOMonitor {
+ "\t data num: "
+ dataIONums.get(i)
+ "\t total: "
- + totalTimes.get(i)
- + "\n";
+ + totalTimes.get(i);
+ // + "\n";
}
- ret +=
- "avg meta IO: "
- + getAvg(metaIOTimes)
- + ", avg data IO: "
- + getAvg(dataIOTimes)
- + ", avg total time: "
- + getAvg(totalTimes)
- + ", isSet: "
- + isSet;
+ // ret +=
+ // "avg meta IO: "
+ // + getAvg(metaIOTimes)
+ // + ", avg data IO: "
+ // + getAvg(dataIOTimes)
+ // + ", avg total time: "
+ // + getAvg(totalTimes)
+ // + ", isSet: "
+ // + isSet;
return ret;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index a84a1ed..506221b 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -245,7 +245,7 @@ public class TsFileSketchTool {
}
private static Pair<String, String> checkArgs(String[] args) {
- String filename = "1643183676699-1-2-1.tsfile";
+ String filename = "/home/kyy/Documents/kdd/iotdb-server-0.12.4/data/data/sequence/root.kobelco/0/0/1643270757735-1-0-0.tsfile";
String outFile = "TsFile_sketch_view.txt";
if (args.length == 1) {
filename = args[0];