You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2022/10/08 14:42:30 UTC
[iotdb] branch research/M4-visualization updated: a
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/research/M4-visualization by this push:
new c5140f6491 a
c5140f6491 is described below
commit c5140f6491a52f8a771e88ef5b798e53d918b481
Author: Lei Rui <10...@qq.com>
AuthorDate: Sat Oct 8 22:35:30 2022 +0800
a
---
.../org/apache/iotdb/db/service/TSServiceImpl.java | 115 ++++++----
.../apache/iotdb/session/MyBasicOperationTest.java | 110 +++++++++
...est.java => MyRealDataTest1_WriteAndQuery.java} | 212 +++++++++--------
...eryTest.java => MyRealDataTest2_OnlyQuery.java} | 32 +--
....java => MyRealDataTest3_WriteAndRawQuery.java} | 255 +++++++++++----------
.../tsfile/common/constant/TsFileConstant.java | 19 +-
.../encoding/decoder/DeltaBinaryDecoder.java | 38 +--
7 files changed, 469 insertions(+), 312 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index b68e34cdd8..4636f867b5 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -18,6 +18,22 @@
*/
package org.apache.iotdb.db.service;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.sql.SQLException;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.apache.iotdb.db.IOMonitor;
import org.apache.iotdb.db.auth.AuthException;
import org.apache.iotdb.db.auth.AuthorityChecker;
@@ -131,29 +147,13 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
-
-import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.sql.SQLException;
-import java.time.ZoneId;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-/** Thrift RPC implementation at server side. */
+/**
+ * Thrift RPC implementation at server side.
+ */
public class TSServiceImpl implements TSIService.Iface {
private static final Logger LOGGER = LoggerFactory.getLogger(TSServiceImpl.class);
@@ -327,7 +327,9 @@ public class TSServiceImpl implements TSIService.Iface {
}
}
- /** release single operation resource */
+ /**
+ * release single operation resource
+ */
public void releaseQueryResource(long queryId) throws StorageEngineException {
sessionManager.releaseQueryResource(queryId);
}
@@ -608,13 +610,13 @@ public class TSServiceImpl implements TSIService.Iface {
return physicalPlan.isQuery()
? internalExecuteQueryStatement(
- statement,
- req.statementId,
- physicalPlan,
- req.fetchSize,
- req.timeout,
- sessionManager.getUsername(req.getSessionId()),
- req.isEnableRedirectQuery())
+ statement,
+ req.statementId,
+ physicalPlan,
+ req.fetchSize,
+ req.timeout,
+ sessionManager.getUsername(req.getSessionId()),
+ req.isEnableRedirectQuery())
: executeUpdateStatement(physicalPlan, req.getSessionId());
} catch (InterruptedException e) {
LOGGER.error(INFO_INTERRUPT_ERROR, req, e);
@@ -641,13 +643,13 @@ public class TSServiceImpl implements TSIService.Iface {
// System.out.println("====DEBUG====: fetchSize=" + req.fetchSize);
return physicalPlan.isQuery()
? internalExecuteQueryStatement(
- statement,
- req.statementId,
- physicalPlan,
- req.fetchSize,
- req.timeout,
- sessionManager.getUsername(req.getSessionId()),
- req.isEnableRedirectQuery())
+ statement,
+ req.statementId,
+ physicalPlan,
+ req.fetchSize,
+ req.timeout,
+ sessionManager.getUsername(req.getSessionId()),
+ req.isEnableRedirectQuery())
: RpcUtils.getTSExecuteStatementResp(
TSStatusCode.EXECUTE_STATEMENT_ERROR, "Statement is not a query statement.");
} catch (InterruptedException e) {
@@ -669,7 +671,14 @@ public class TSServiceImpl implements TSIService.Iface {
IOMonitor.print()
+ ". timeColumnTS2DIFFLoadBatchCost="
+ TsFileConstant.timeColumnTS2DIFFLoadBatchCost.getSum()
- + "us");
+ + "us"
+// + ". countForRegularEqual="
+// + TsFileConstant.countForRegularEqual
+// + ". countForRegularNOTEqual="
+// + TsFileConstant.countForRegularNOTEqual
+// + ". countForRegularZero="
+// + TsFileConstant.countForRegularZero
+ );
IOMonitor.finish();
return ret;
}
@@ -685,13 +694,13 @@ public class TSServiceImpl implements TSIService.Iface {
processor.rawDataQueryReqToPhysicalPlan(req, sessionManager.getZoneId(req.sessionId));
return physicalPlan.isQuery()
? internalExecuteQueryStatement(
- "",
- req.statementId,
- physicalPlan,
- req.fetchSize,
- config.getQueryTimeoutThreshold(),
- sessionManager.getUsername(req.sessionId),
- req.isEnableRedirectQuery())
+ "",
+ req.statementId,
+ physicalPlan,
+ req.fetchSize,
+ config.getQueryTimeoutThreshold(),
+ sessionManager.getUsername(req.sessionId),
+ req.isEnableRedirectQuery())
: RpcUtils.getTSExecuteStatementResp(
TSStatusCode.EXECUTE_STATEMENT_ERROR, "Statement is not a query statement.");
} catch (InterruptedException e) {
@@ -707,7 +716,7 @@ public class TSServiceImpl implements TSIService.Iface {
/**
* @param plan must be a plan for Query: FillQueryPlan, AggregationPlan, GroupByTimePlan, UDFPlan,
- * some AuthorPlan
+ * some AuthorPlan
*/
@SuppressWarnings({"squid:S3776", "squid:S1141"}) // Suppress high Cognitive Complexity warning
private TSExecuteStatementResp internalExecuteQueryStatement(
@@ -719,8 +728,8 @@ public class TSServiceImpl implements TSIService.Iface {
String username,
boolean enableRedirect)
throws QueryProcessException, SQLException, StorageEngineException,
- QueryFilterOptimizationException, MetadataException, IOException, InterruptedException,
- TException, AuthException {
+ QueryFilterOptimizationException, MetadataException, IOException, InterruptedException,
+ TException, AuthException {
// start record execution time
IOMonitor.setSQL(statement);
@@ -858,7 +867,9 @@ public class TSServiceImpl implements TSIService.Iface {
dataSet.getDataTypes().stream().map(Enum::toString).collect(Collectors.toList()));
}
- /** get ResultSet schema */
+ /**
+ * get ResultSet schema
+ */
private TSExecuteStatementResp getQueryColumnHeaders(PhysicalPlan physicalPlan, String username)
throws AuthException, TException, QueryProcessException, MetadataException {
@@ -1127,10 +1138,12 @@ public class TSServiceImpl implements TSIService.Iface {
return encoder;
}
- /** create QueryDataSet and buffer it for fetchResults */
+ /**
+ * create QueryDataSet and buffer it for fetchResults
+ */
private QueryDataSet createQueryDataSet(long queryId, PhysicalPlan physicalPlan, int fetchSize)
throws QueryProcessException, QueryFilterOptimizationException, StorageEngineException,
- IOException, MetadataException, SQLException, TException, InterruptedException {
+ IOException, MetadataException, SQLException, TException, InterruptedException {
QueryContext context = genQueryContext(queryId, physicalPlan.isDebug());
if (physicalPlan instanceof QueryPlan) {
@@ -1189,7 +1202,7 @@ public class TSServiceImpl implements TSIService.Iface {
statement, sessionManager.getZoneId(sessionId), DEFAULT_FETCH_SIZE);
return physicalPlan.isQuery()
? RpcUtils.getTSExecuteStatementResp(
- TSStatusCode.EXECUTE_STATEMENT_ERROR, "Statement is a query statement.")
+ TSStatusCode.EXECUTE_STATEMENT_ERROR, "Statement is a query statement.")
: executeUpdateStatement(physicalPlan, sessionId);
}
@@ -1619,7 +1632,9 @@ public class TSServiceImpl implements TSIService.Iface {
return insertTabletPlan;
}
- /** construct one InsertMultiTabletPlan and process it */
+ /**
+ * construct one InsertMultiTabletPlan and process it
+ */
public TSStatus insertTabletsInternal(TSInsertTabletsReq req) throws IllegalPathException {
List<InsertTabletPlan> insertTabletPlanList = new ArrayList<>();
InsertMultiTabletPlan insertMultiTabletPlan = new InsertMultiTabletPlan();
diff --git a/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest.java b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest.java
new file mode 100644
index 0000000000..4ef67976ef
--- /dev/null
+++ b/session/src/test/java/org/apache/iotdb/session/MyBasicOperationTest.java
@@ -0,0 +1,110 @@
+package org.apache.iotdb.session;
+
+import java.text.DecimalFormat;
+import java.util.Random;
+import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+
+public class MyBasicOperationTest {
+
+ public static void main(String[] args) {
+ // op1: long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
+ // op2: compare bytes
+
+ int repeat = 1000000;
+ DescriptiveStatistics op1 = new DescriptiveStatistics();
+ DescriptiveStatistics op2 = new DescriptiveStatistics();
+ for (int k = 0; k < repeat; k++) {
+ // prepare test data
+ Random r = new Random();
+ int low = 0; // inclusive
+ int high = 256; // exclusive
+ int packNum = 128;
+ int packWidth = 8; //equal to one byte length
+ byte[] buf = new byte[packNum];
+ for (int i = 0; i < packNum; i++) {
+ int v = r.nextInt(high - low) + low;
+ BytesUtils.longToBytes(v, buf, i * packWidth, packWidth);
+ }
+
+ // test op1
+ long sum = 0;
+ long start = System.nanoTime();
+ for (int i = 0; i < packNum; i++) {
+ sum += BytesUtils.bytesToLong(buf, packWidth * i, packWidth);
+ }
+ long elapsedTime = System.nanoTime() - start;
+ System.out.println(elapsedTime / 1000.0 + "us");
+ System.out.println(sum);
+ op1.addValue(elapsedTime / 1000.0);
+
+ // test op2
+ sum = 0;
+ byte regular = 1;
+ start = System.nanoTime();
+ for (int i = 0; i < packNum; i++) {
+ if (regular != buf[i]) {
+ sum++;
+ }
+ }
+ elapsedTime = System.nanoTime() - start;
+ System.out.println(elapsedTime / 1000.0 + "us");
+ System.out.println(sum);
+ op2.addValue(elapsedTime / 1000.0);
+ }
+ printStat(op1, "op1-convertBitToLong");
+ printStat(op2, "op2-compareByte");
+ System.out.println("op1/op2=" + op1.getMean() / op2.getMean());
+ System.out.println("op2/op1=" + op2.getMean() / op1.getMean());
+ }
+
+ private static String printStat(DescriptiveStatistics statistics, String name) {
+ DecimalFormat df = new DecimalFormat("#,###.00");
+ double max = statistics.getMax();
+ double min = statistics.getMin();
+ double mean = statistics.getMean();
+ double std = statistics.getStandardDeviation();
+ double p25 = statistics.getPercentile(25);
+ double p50 = statistics.getPercentile(50);
+ double p75 = statistics.getPercentile(75);
+ double p90 = statistics.getPercentile(90);
+ double p95 = statistics.getPercentile(95);
+ String res =
+ name + "_stats"
+ + ": "
+ + "num="
+ + statistics.getN()
+ + ", "
+ + "sum="
+ + df.format(statistics.getSum())
+ + "us,"
+ + "mean="
+ + df.format(mean)
+ + ", "
+ + "min="
+ + df.format(min)
+ + ", "
+ + "max="
+ + df.format(max)
+ + ", "
+ + "std="
+ + df.format(std)
+ + ", "
+ + "p25="
+ + df.format(p25)
+ + ", "
+ + "p50="
+ + df.format(p50)
+ + ", "
+ + "p75="
+ + df.format(p75)
+ + ", "
+ + "p90="
+ + df.format(p90)
+ + ", "
+ + "p95="
+ + df.format(p95);
+ System.out.println(res);
+ return res;
+ }
+}
diff --git a/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
similarity index 75%
copy from session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
copy to session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
index 7efce9a4ed..2704989f0c 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest1_WriteAndQuery.java
@@ -1,5 +1,24 @@
package org.apache.iotdb.session;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.allRegularBytesSize;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.byteArrayLengthStatistics;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForHitNewDeltas;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForNotHitNewDeltas;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularEqual;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNOTEqual;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularZero;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.prepareAllRegulars;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.regularNewDeltasStatistics;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.timeColumnTS2DIFFLoadBatchCost;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
@@ -10,32 +29,12 @@ import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.SessionDataSet.DataIterator;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.text.DecimalFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.allRegularBytesSize;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.byteArrayLengthStatistics;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForHitNewDeltas;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForNotHitNewDeltas;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularEqual;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNOTEqual;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNewDeltas;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.prepareAllRegulars;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.timeColumnTS2DIFFLoadBatchCost;
-
-public class MyRealDataWriteQueryTest {
+public class MyRealDataTest1_WriteAndQuery {
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
@@ -52,77 +51,101 @@ public class MyRealDataWriteQueryTest {
private static final String queryFormat_UDF =
"select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and time<%4$d";
-// private static String device = "root.game";
-// private static String measurement = "s6";
-// private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
-// private static String timestamp_precision = "ns"; // ns, us, ms
-// private static long dataMinTime = 0;
-// private static long dataMaxTime = 617426057626L;
-// private static long total_time_length = dataMaxTime - dataMinTime;
-// private static int total_point_number = 1200000;
-// private static int iotdb_chunk_point_size = 100000;
-// private static long chunkAvgTimeLen = (long) Math
-// .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
-// private static String filePath =
-//
-// "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv";
-// private static int deletePercentage = 0; // 0 means no deletes. 0-100
-// private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
-// private static int timeIdx = 0; // 时间戳idx，从0开始
-// private static int valueIdx = 1; // 值idx，从0开始
-// private static int w = 2;
-// private static long range = total_time_length;
-// private static boolean enableRegularityTimeDecode = false;
-// private static long regularTimeInterval = 511996L;
-// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
-
- // private static String device = "root.debs2012";
- // private static String measurement = "mf03";
- // private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
- // private static String timestamp_precision = "ns"; // ns, us, ms
- // private static long dataMinTime = 1329955200008812200L;
- // private static long dataMaxTime = 1329965999991045200L;
- // private static long total_time_length = dataMaxTime - dataMinTime;
- // private static int total_point_number = 1076102;
- // private static int iotdb_chunk_point_size = 100000;
- // private static long chunkAvgTimeLen = (long) Math
- // .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
- // private static String filePath =
- //
- // "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\MF03.csv";
- // private static int deletePercentage = 0; // 0 means no deletes. 0-100
- // private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
- // private static int timeIdx = 0; // 时间戳idx，从0开始
- // private static int valueIdx = 1; // 值idx，从0开始
- // private static int w = 2;
- // private static long range = total_time_length;
- // private static boolean enableRegularityTimeDecode = true;
- // private static long regularTimeInterval = 10000900L;
- // private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
-
- private static String device = "root.kobelco.trans.03.1090001603.2401604";
- private static String measurement = "KOB_0002_00_67";
+// private static String device = "root.game";
+// private static String measurement = "s6";
+// private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
+// private static String timestamp_precision = "ns"; // ns, us, ms
+// private static long dataMinTime = 0;
+// private static long dataMaxTime = 617426057626L;
+// private static long total_time_length = dataMaxTime - dataMinTime;
+// private static int total_point_number = 1200000;
+// private static int iotdb_chunk_point_size = 100000;
+// private static long chunkAvgTimeLen = (long) Math
+// .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
+// private static String filePath =
+// "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv";
+// private static int deletePercentage = 0; // 0 means no deletes. 0-100
+// private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+// private static int timeIdx = 0; // 时间戳idx，从0开始
+// private static int valueIdx = 1; // 值idx，从0开始
+// private static int w = 2;
+// private static long range = total_time_length;
+// private static boolean enableRegularityTimeDecode = true;
+// private static long regularTimeInterval = 511996L;
+// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+
+// private static String device = "root.debs2012";
+// private static String measurement = "mf03";
+// private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
+// private static String timestamp_precision = "ns"; // ns, us, ms
+// private static long dataMinTime = 1329955200008812200L;
+// private static long dataMaxTime = 1329965999991045200L;
+// private static long total_time_length = dataMaxTime - dataMinTime;
+// private static int total_point_number = 1076102;
+// private static int iotdb_chunk_point_size = 100000;
+// private static long chunkAvgTimeLen = (long) Math
+// .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
+// private static String filePath =
+// "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\MF03.csv";
+// private static int deletePercentage = 0; // 0 means no deletes. 0-100
+// private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+// private static int timeIdx = 0; // 时间戳idx，从0开始
+// private static int valueIdx = 1; // 值idx，从0开始
+// private static int w = 2;
+// private static long range = total_time_length;
+// private static boolean enableRegularityTimeDecode = true;
+// private static long regularTimeInterval = 10000900L;
+// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+
+// private static String device = "root.kobelco.trans.03.1090001603.2401604";
+// private static String measurement = "KOB_0002_00_67";
+// private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
+// private static String timestamp_precision = "ms"; // ns, us, ms
+// private static long dataMinTime = 1616805035973L;
+// private static long dataMaxTime = 1627380839563L;
+// private static long total_time_length = dataMaxTime - dataMinTime;
+// private static int total_point_number = 1943180;
+// private static int iotdb_chunk_point_size = 100000;
+// private static long chunkAvgTimeLen =
+// (long)
+// Math.ceil(
+// total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
+// private static String filePath =
+// "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\KOB_2.csv";
+// private static int deletePercentage = 0; // 0 means no deletes. 0-100
+// private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+// private static int timeIdx = 0; // 时间戳idx，从0开始
+// private static int valueIdx = 1; // 值idx，从0开始
+// private static int w = 3;
+// private static long range = total_time_length;
+// private static boolean enableRegularityTimeDecode = true;
+// private static long regularTimeInterval = 1000L;
+// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+
+
+ private static String device = "root.sg1";
+ private static String measurement = "RcvTime";
private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
private static String timestamp_precision = "ms"; // ns, us, ms
- private static long dataMinTime = 1616805035973L;
- private static long dataMaxTime = 1627380839563L;
+ private static long dataMinTime = 1616194494000L;
+ private static long dataMaxTime = 1642656230000L;
private static long total_time_length = dataMaxTime - dataMinTime;
- private static int total_point_number = 1943180;
+ private static int total_point_number = 1330764;
private static int iotdb_chunk_point_size = 100000;
private static long chunkAvgTimeLen =
(long)
Math.ceil(
total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
private static String filePath =
- "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\KOB_2.csv";
+ "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\RcvTime.csv";
private static int deletePercentage = 0; // 0 means no deletes. 0-100
private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
private static int timeIdx = 0; // 时间戳idx，从0开始
private static int valueIdx = 1; // 值idx，从0开始
private static int w = 3;
private static long range = total_time_length;
- private static boolean enableRegularityTimeDecode = false;
- private static long regularTimeInterval = 1000L;
+ private static boolean enableRegularityTimeDecode = true;
+ private static long regularTimeInterval = 2000L;
private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
@Before
@@ -167,10 +190,12 @@ public class MyRealDataWriteQueryTest {
@After
public void tearDown() throws Exception {
- // EnvironmentUtils.cleanEnv();
+ EnvironmentUtils.cleanEnv();
}
- /** Before writing data, make sure check the server parameter configurations. */
+ /**
+ * Before writing data, make sure check the server parameter configurations.
+ */
// Usage: java -jar WriteData-0.12.4.jar device measurement dataType timestamp_precision
// total_time_length total_point_number iotdb_chunk_point_size filePath deleteFreq deleteLen
// timeIdx valueIdx
@@ -328,30 +353,31 @@ public class MyRealDataWriteQueryTest {
+ "p95="
+ df.format(p95));
- System.out.println("Equal Num: " + countForRegularEqual.getN());
- System.out.println("NOT Equal Num: " + countForRegularNOTEqual.getN());
+ System.out.println("Equal Num: " + countForRegularEqual);
+ System.out.println("NOT Equal Num: " + countForRegularNOTEqual);
+ System.out.println("zero Num: " + countForRegularZero);
System.out.println("hit Num: " + countForHitNewDeltas.getN());
System.out.println("NOT hit Num: " + countForNotHitNewDeltas.getN());
- max = countForRegularNewDeltas.getMax();
- min = countForRegularNewDeltas.getMin();
- mean = countForRegularNewDeltas.getMean();
- std = countForRegularNewDeltas.getStandardDeviation();
- p25 = countForRegularNewDeltas.getPercentile(25);
- p50 = countForRegularNewDeltas.getPercentile(50);
- p75 = countForRegularNewDeltas.getPercentile(75);
- p90 = countForRegularNewDeltas.getPercentile(90);
- p95 = countForRegularNewDeltas.getPercentile(95);
+ max = regularNewDeltasStatistics.getMax();
+ min = regularNewDeltasStatistics.getMin();
+ mean = regularNewDeltasStatistics.getMean();
+ std = regularNewDeltasStatistics.getStandardDeviation();
+ p25 = regularNewDeltasStatistics.getPercentile(25);
+ p50 = regularNewDeltasStatistics.getPercentile(50);
+ p75 = regularNewDeltasStatistics.getPercentile(75);
+ p90 = regularNewDeltasStatistics.getPercentile(90);
+ p95 = regularNewDeltasStatistics.getPercentile(95);
System.out.println(
- "countForRegularNewDeltas_stats"
+ "regularNewDeltas_stats"
+ ": "
+ "num="
- + countForRegularNewDeltas.getN()
+ + regularNewDeltasStatistics.getN()
+ ", "
// num is inaccurate because I let alone the last chunk
+ "sum="
- + df.format(countForRegularNewDeltas.getSum())
+ + df.format(regularNewDeltasStatistics.getSum())
+ "us,"
+ "mean="
+ df.format(mean)
diff --git a/session/src/test/java/org/apache/iotdb/session/MyRealDataQueryTest.java b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest2_OnlyQuery.java
similarity index 96%
rename from session/src/test/java/org/apache/iotdb/session/MyRealDataQueryTest.java
rename to session/src/test/java/org/apache/iotdb/session/MyRealDataTest2_OnlyQuery.java
index 08fd63bf2c..ed038f0a97 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyRealDataQueryTest.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest2_OnlyQuery.java
@@ -27,11 +27,12 @@ import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForHit
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForNotHitNewDeltas;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularEqual;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNOTEqual;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNewDeltas;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularZero;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.regularNewDeltasStatistics;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.prepareAllRegulars;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.timeColumnTS2DIFFLoadBatchCost;
-public class MyRealDataQueryTest {
+public class MyRealDataTest2_OnlyQuery {
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
@@ -323,30 +324,31 @@ public class MyRealDataQueryTest {
+ "p95="
+ df.format(p95));
- System.out.println("Equal Num: " + countForRegularEqual.getN());
- System.out.println("NOT Equal Num: " + countForRegularNOTEqual.getN());
+ System.out.println("Equal Num: " + countForRegularEqual);
+ System.out.println("NOT Equal Num: " + countForRegularNOTEqual);
+ System.out.println("zero Num: " + countForRegularZero);
System.out.println("hit Num: " + countForHitNewDeltas.getN());
System.out.println("NOT hit Num: " + countForNotHitNewDeltas.getN());
- max = countForRegularNewDeltas.getMax();
- min = countForRegularNewDeltas.getMin();
- mean = countForRegularNewDeltas.getMean();
- std = countForRegularNewDeltas.getStandardDeviation();
- p25 = countForRegularNewDeltas.getPercentile(25);
- p50 = countForRegularNewDeltas.getPercentile(50);
- p75 = countForRegularNewDeltas.getPercentile(75);
- p90 = countForRegularNewDeltas.getPercentile(90);
- p95 = countForRegularNewDeltas.getPercentile(95);
+ max = regularNewDeltasStatistics.getMax();
+ min = regularNewDeltasStatistics.getMin();
+ mean = regularNewDeltasStatistics.getMean();
+ std = regularNewDeltasStatistics.getStandardDeviation();
+ p25 = regularNewDeltasStatistics.getPercentile(25);
+ p50 = regularNewDeltasStatistics.getPercentile(50);
+ p75 = regularNewDeltasStatistics.getPercentile(75);
+ p90 = regularNewDeltasStatistics.getPercentile(90);
+ p95 = regularNewDeltasStatistics.getPercentile(95);
System.out.println(
"countForRegularNewDeltas_stats"
+ ": "
+ "num="
- + countForRegularNewDeltas.getN()
+ + regularNewDeltasStatistics.getN()
+ ", "
// num is inaccurate because I let alone the last chunk
+ "sum="
- + df.format(countForRegularNewDeltas.getSum())
+ + df.format(regularNewDeltasStatistics.getSum())
+ "us,"
+ "mean="
+ df.format(mean)
diff --git a/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest3_WriteAndRawQuery.java
similarity index 79%
rename from session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
rename to session/src/test/java/org/apache/iotdb/session/MyRealDataTest3_WriteAndRawQuery.java
index 7efce9a4ed..41a10c572f 100644
--- a/session/src/test/java/org/apache/iotdb/session/MyRealDataWriteQueryTest.java
+++ b/session/src/test/java/org/apache/iotdb/session/MyRealDataTest3_WriteAndRawQuery.java
@@ -1,5 +1,24 @@
package org.apache.iotdb.session;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.allRegularBytesSize;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.byteArrayLengthStatistics;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForHitNewDeltas;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForNotHitNewDeltas;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularEqual;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNOTEqual;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularZero;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.prepareAllRegulars;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.regularNewDeltasStatistics;
+import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.timeColumnTS2DIFFLoadBatchCost;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
@@ -10,32 +29,12 @@ import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.SessionDataSet.DataIterator;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.text.DecimalFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.allRegularBytesSize;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.byteArrayLengthStatistics;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForHitNewDeltas;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForNotHitNewDeltas;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularEqual;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNOTEqual;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.countForRegularNewDeltas;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.prepareAllRegulars;
-import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.timeColumnTS2DIFFLoadBatchCost;
-
-public class MyRealDataWriteQueryTest {
+public class MyRealDataTest3_WriteAndRawQuery {
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
@@ -52,29 +51,28 @@ public class MyRealDataWriteQueryTest {
private static final String queryFormat_UDF =
"select M4(%1$s,'tqs'='%3$d','tqe'='%4$d','w'='%5$d') from %2$s where time>=%3$d and time<%4$d";
-// private static String device = "root.game";
-// private static String measurement = "s6";
-// private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
-// private static String timestamp_precision = "ns"; // ns, us, ms
-// private static long dataMinTime = 0;
-// private static long dataMaxTime = 617426057626L;
-// private static long total_time_length = dataMaxTime - dataMinTime;
-// private static int total_point_number = 1200000;
-// private static int iotdb_chunk_point_size = 100000;
-// private static long chunkAvgTimeLen = (long) Math
-// .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
-// private static String filePath =
-//
-// "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv";
-// private static int deletePercentage = 0; // 0 means no deletes. 0-100
-// private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
-// private static int timeIdx = 0; // 时间戳idx，从0开始
-// private static int valueIdx = 1; // 值idx，从0开始
-// private static int w = 2;
-// private static long range = total_time_length;
-// private static boolean enableRegularityTimeDecode = false;
-// private static long regularTimeInterval = 511996L;
-// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+ private static String device = "root.game";
+ private static String measurement = "s6";
+ private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
+ private static String timestamp_precision = "ns"; // ns, us, ms
+ private static long dataMinTime = 0;
+ private static long dataMaxTime = 617426057626L;
+ private static long total_time_length = dataMaxTime - dataMinTime;
+ private static int total_point_number = 1200000;
+ private static int iotdb_chunk_point_size = 100000;
+ private static long chunkAvgTimeLen = (long) Math
+ .ceil(total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
+ private static String filePath =
+ "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\BallSpeed.csv";
+ private static int deletePercentage = 0; // 0 means no deletes. 0-100
+ private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+ private static int timeIdx = 0; // 时间戳idx，从0开始
+ private static int valueIdx = 1; // 值idx，从0开始
+ private static int w = 2;
+ private static long range = total_time_length;
+ private static boolean enableRegularityTimeDecode = true;
+ private static long regularTimeInterval = 511996L;
+ private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
// private static String device = "root.debs2012";
// private static String measurement = "mf03";
@@ -100,30 +98,30 @@ public class MyRealDataWriteQueryTest {
// private static long regularTimeInterval = 10000900L;
// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
- private static String device = "root.kobelco.trans.03.1090001603.2401604";
- private static String measurement = "KOB_0002_00_67";
- private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
- private static String timestamp_precision = "ms"; // ns, us, ms
- private static long dataMinTime = 1616805035973L;
- private static long dataMaxTime = 1627380839563L;
- private static long total_time_length = dataMaxTime - dataMinTime;
- private static int total_point_number = 1943180;
- private static int iotdb_chunk_point_size = 100000;
- private static long chunkAvgTimeLen =
- (long)
- Math.ceil(
- total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
- private static String filePath =
- "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\KOB_2.csv";
- private static int deletePercentage = 0; // 0 means no deletes. 0-100
- private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
- private static int timeIdx = 0; // 时间戳idx，从0开始
- private static int valueIdx = 1; // 值idx，从0开始
- private static int w = 3;
- private static long range = total_time_length;
- private static boolean enableRegularityTimeDecode = false;
- private static long regularTimeInterval = 1000L;
- private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
+// private static String device = "root.kobelco.trans.03.1090001603.2401604";
+// private static String measurement = "KOB_0002_00_67";
+// private static TSDataType tsDataType = TSDataType.INT64; // TSDataType.DOUBLE;
+// private static String timestamp_precision = "ms"; // ns, us, ms
+// private static long dataMinTime = 1616805035973L;
+// private static long dataMaxTime = 1627380839563L;
+// private static long total_time_length = dataMaxTime - dataMinTime;
+// private static int total_point_number = 1943180;
+// private static int iotdb_chunk_point_size = 100000;
+// private static long chunkAvgTimeLen =
+// (long)
+// Math.ceil(
+// total_time_length / Math.ceil(total_point_number * 1.0 / iotdb_chunk_point_size));
+// private static String filePath =
+// "D:\\github\\m4-lsm\\M4-visualization-exp\\src\\main\\java\\org\\apache\\iotdb\\datasets\\KOB_2.csv";
+// private static int deletePercentage = 0; // 0 means no deletes. 0-100
+// private static int deleteLenPercentage = 0; // 0-100 每次删除的时间长度，用chunkAvgTimeLen的百分比表示
+// private static int timeIdx = 0; // 时间戳idx，从0开始
+// private static int valueIdx = 1; // 值idx，从0开始
+// private static int w = 3;
+// private static long range = total_time_length;
+// private static boolean enableRegularityTimeDecode = true;
+// private static long regularTimeInterval = 1000L;
+// private static String approach = "mac"; // 选择查询执行算法: 1: MAC, 2: MOC, 3: CPV
@Before
public void setUp() throws Exception {
@@ -167,10 +165,12 @@ public class MyRealDataWriteQueryTest {
@After
public void tearDown() throws Exception {
- // EnvironmentUtils.cleanEnv();
+ EnvironmentUtils.cleanEnv();
}
- /** Before writing data, make sure check the server parameter configurations. */
+ /**
+ * Before writing data, make sure check the server parameter configurations.
+ */
// Usage: java -jar WriteData-0.12.4.jar device measurement dataType timestamp_precision
// total_time_length total_point_number iotdb_chunk_point_size filePath deleteFreq deleteLen
// timeIdx valueIdx
@@ -220,27 +220,28 @@ public class MyRealDataWriteQueryTest {
// query result size is no more than 8000*4=32000.
session.setFetchSize(1000000);
- String sql;
- if (approach.equals("mac")) {
- // MAC UDF
- sql = String.format(queryFormat_UDF, measurement, device, minTime, maxTime, w); // MAC
- } else {
- // MOC and CPV sql use the same sql queryFormat.
- sql =
- String.format(
- queryFormat,
- measurement,
- measurement,
- measurement,
- measurement,
- measurement,
- measurement,
- device,
- minTime,
- maxTime,
- interval,
- timestamp_precision); // note the time precision unit
- }
+// String sql;
+// if (approach.equals("mac")) {
+// // MAC UDF
+// sql = String.format(queryFormat_UDF, measurement, device, minTime, maxTime, w); // MAC
+// } else {
+// // MOC and CPV sql use the same sql queryFormat.
+// sql =
+// String.format(
+// queryFormat,
+// measurement,
+// measurement,
+// measurement,
+// measurement,
+// measurement,
+// measurement,
+// device,
+// minTime,
+// maxTime,
+// interval,
+// timestamp_precision); // note the time precision unit
+// }
+ String sql = String.format("select %s from %s", measurement, device);
System.out.println("[QueryData] sql=" + sql);
long c = 0;
@@ -249,28 +250,29 @@ public class MyRealDataWriteQueryTest {
// System.out.println(dataSet.getColumnNames());
while (iterator.next()) { // this way avoid constructing rowRecord
c++;
- String ans;
- if (approach.equals("mac")) {
- ans =
- String.format(
- "%s,%s",
- iterator.getString(1), // time
- iterator.getString(2)); // M4
- } else {
- ans =
- String.format(
- "%s,%s,%s,%s,%s,%s,%s",
- iterator.getString(1), // time
- iterator.getString(2), // min_time
- iterator.getString(3), // max_time
- iterator.getString(4), // first_value
- iterator.getString(5), // last_value
- iterator.getString(6), // min_value & bottomTime
- iterator.getString(7)); // max_value & topTime
- }
- System.out.println(ans);
+// String ans;
+// if (approach.equals("mac")) {
+// ans =
+// String.format(
+// "%s,%s",
+// iterator.getString(1), // time
+// iterator.getString(2)); // M4
+// } else {
+// ans =
+// String.format(
+// "%s,%s,%s,%s,%s,%s,%s",
+// iterator.getString(1), // time
+// iterator.getString(2), // min_time
+// iterator.getString(3), // max_time
+// iterator.getString(4), // first_value
+// iterator.getString(5), // last_value
+// iterator.getString(6), // min_value & bottomTime
+// iterator.getString(7)); // max_value & topTime
+// }
+// System.out.println(ans);
}
- Assert.assertEquals(w, c);
+// Assert.assertEquals(w, c);
+ System.out.println("total point number=" + c);
// session.executeNonQueryStatement("clear cache");
dataSet = session.executeFinish();
@@ -328,30 +330,31 @@ public class MyRealDataWriteQueryTest {
+ "p95="
+ df.format(p95));
- System.out.println("Equal Num: " + countForRegularEqual.getN());
- System.out.println("NOT Equal Num: " + countForRegularNOTEqual.getN());
+ System.out.println("Equal Num: " + countForRegularEqual);
+ System.out.println("NOT Equal Num: " + countForRegularNOTEqual);
+ System.out.println("zero Num: " + countForRegularZero);
System.out.println("hit Num: " + countForHitNewDeltas.getN());
System.out.println("NOT hit Num: " + countForNotHitNewDeltas.getN());
- max = countForRegularNewDeltas.getMax();
- min = countForRegularNewDeltas.getMin();
- mean = countForRegularNewDeltas.getMean();
- std = countForRegularNewDeltas.getStandardDeviation();
- p25 = countForRegularNewDeltas.getPercentile(25);
- p50 = countForRegularNewDeltas.getPercentile(50);
- p75 = countForRegularNewDeltas.getPercentile(75);
- p90 = countForRegularNewDeltas.getPercentile(90);
- p95 = countForRegularNewDeltas.getPercentile(95);
+ max = regularNewDeltasStatistics.getMax();
+ min = regularNewDeltasStatistics.getMin();
+ mean = regularNewDeltasStatistics.getMean();
+ std = regularNewDeltasStatistics.getStandardDeviation();
+ p25 = regularNewDeltasStatistics.getPercentile(25);
+ p50 = regularNewDeltasStatistics.getPercentile(50);
+ p75 = regularNewDeltasStatistics.getPercentile(75);
+ p90 = regularNewDeltasStatistics.getPercentile(90);
+ p95 = regularNewDeltasStatistics.getPercentile(95);
System.out.println(
- "countForRegularNewDeltas_stats"
+ "regularNewDeltas_stats"
+ ": "
+ "num="
- + countForRegularNewDeltas.getN()
+ + regularNewDeltasStatistics.getN()
+ ", "
// num is inaccurate because I let alone the last chunk
+ "sum="
- + df.format(countForRegularNewDeltas.getSum())
+ + df.format(regularNewDeltasStatistics.getSum())
+ "us,"
+ "mean="
+ df.format(mean)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
index 0425f9d793..349ddf6d9d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/constant/TsFileConstant.java
@@ -34,20 +34,21 @@ public class TsFileConstant {
public static DescriptiveStatistics timeColumnTS2DIFFLoadBatchCost = new DescriptiveStatistics();
- public static DescriptiveStatistics countForRegularEqual = new DescriptiveStatistics();
- public static DescriptiveStatistics countForRegularNOTEqual = new DescriptiveStatistics();
- public static DescriptiveStatistics countForRegularZero = new DescriptiveStatistics();
+ public static long countForRegularEqual = 0; // equal to regular
+ public static long countForRegularNOTEqual = 0; // not equal to regular
+ public static long countForRegularZero = 0; // packWidth=0 no need
- public static DescriptiveStatistics countForRegularNewDeltas = new DescriptiveStatistics();
+ public static DescriptiveStatistics regularNewDeltasStatistics = new DescriptiveStatistics();
- public static DescriptiveStatistics countForHitNewDeltas = new DescriptiveStatistics();
- public static DescriptiveStatistics countForNotHitNewDeltas = new DescriptiveStatistics();
+ public static DescriptiveStatistics countForHitNewDeltas = new DescriptiveStatistics(); // allRegularBytes contains
+ public static DescriptiveStatistics countForNotHitNewDeltas = new DescriptiveStatistics(); // allRegularBytes not contains
- public static DescriptiveStatistics byteArrayLengthStatistics = new DescriptiveStatistics();
+ public static DescriptiveStatistics byteArrayLengthStatistics = new DescriptiveStatistics(); // regular bytes length
- public static DescriptiveStatistics allRegularBytesSize = new DescriptiveStatistics();
+ public static DescriptiveStatistics allRegularBytesSize = new DescriptiveStatistics(); // allRegularBytes map size
public static DescriptiveStatistics prepareAllRegulars = new DescriptiveStatistics();
- private TsFileConstant() {}
+ private TsFileConstant() {
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index 8b09da3bc7..705ac4f603 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -237,15 +237,15 @@ public abstract class DeltaBinaryDecoder extends Decoder {
if (enableRegularityTimeDecode) {
long newRegularDelta = regularTimeInterval - minDeltaBase;
- // System.out.println("newRegularDelta = " + newRegularDelta);
- // countForRegularNewDeltas.addValue(newRegularDelta);
+ System.out.println("newRegularDelta = " + newRegularDelta);
+ TsFileConstant.regularNewDeltasStatistics.addValue(newRegularDelta);
if (packWidth == 0) {
for (int i = 0; i < packNum; i++) {
data[i] = previous + minDeltaBase; // v=0
previous = data[i];
- // System.out.println("[RL]0");
- // TsFileConstant.countForRegularZero.addValue(1);
+ System.out.println("[RL]0");
+ TsFileConstant.countForRegularZero++;
}
} else if (newRegularDelta < 0
|| newRegularDelta
@@ -254,19 +254,19 @@ public abstract class DeltaBinaryDecoder extends Decoder {
long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
data[i] = previous + minDeltaBase + v;
previous = data[i];
- // System.out.println("[RL]no");
- // TsFileConstant.countForRegularNOTEqual.addValue(1);
+ System.out.println("[RL]no");
+ TsFileConstant.countForRegularNOTEqual++;
}
} else {
// long start1 = System.nanoTime();
Map<Integer, byte[]> regularBytes;
if (allRegularBytes.containsKey(new Pair<>(newRegularDelta, packWidth))) {
regularBytes = allRegularBytes.get(new Pair<>(newRegularDelta, packWidth));
- // countForHitNewDeltas.addValue(1);
- // System.out.println("here");
+ TsFileConstant.countForHitNewDeltas.addValue(1);
+ System.out.println("here");
} else {
- // countForNotHitNewDeltas.addValue(1);
- // System.out.println("here");
+ TsFileConstant.countForNotHitNewDeltas.addValue(1);
+ System.out.println("here");
regularBytes = new HashMap<>();
for (int i = 0; i < 8; i++) {
@@ -326,8 +326,8 @@ public abstract class DeltaBinaryDecoder extends Decoder {
byte[] byteArray = regularBytes.get(pos);
int posByteIdx = i * packWidth / 8;
- // System.out.println("byteArray length=" + byteArray.length);
- // byteArrayLengthStatistics.addValue(byteArray.length);
+ System.out.println("byteArray length=" + byteArray.length);
+ TsFileConstant.byteArrayLengthStatistics.addValue(byteArray.length);
for (int k = 0; k < byteArray.length; k++, posByteIdx++) {
byte regular = byteArray[k];
byte data = deltaBuf[posByteIdx];
@@ -352,13 +352,13 @@ public abstract class DeltaBinaryDecoder extends Decoder {
if (equal) {
data[i] = previous + regularTimeInterval;
- // System.out.println("[RL]equals");
- // TsFileConstant.countForRegularEqual.addValue(1);
+ System.out.println("[RL]equals");
+ TsFileConstant.countForRegularEqual++;
} else {
long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
data[i] = previous + minDeltaBase + v;
- // System.out.println("[RL]no");
- // TsFileConstant.countForRegularNOTEqual.addValue(1);
+ System.out.println("[RL]no");
+ TsFileConstant.countForRegularNOTEqual++;
}
// data[i] = previous + regularTimeInterval;
previous = data[i];
@@ -368,12 +368,12 @@ public abstract class DeltaBinaryDecoder extends Decoder {
readPack();
}
+ TsFileConstant.allRegularBytesSize.addValue(allRegularBytes.size());
+ System.out.println("allRegularBytes size=" + allRegularBytes.size());
+
long runTime = System.nanoTime() - start; // ns
TsFileConstant.timeColumnTS2DIFFLoadBatchCost.addValue(runTime / 1000.0); // us
- // allRegularBytesSize.addValue(allRegularBytes.size());
- // System.out.println("allRegularBytes size=" + allRegularBytes.size());
-
return firstValue;
}