You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2024/10/27 15:41:58 UTC
(iotdb) 18/31: add
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fc7fcb5d61ec2b797c9456e0ada9d41b012f260f
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Oct 7 22:03:56 2024 +0800
add
---
.../resources/conf/iotdb-engine.properties | 3 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 9 ++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +
.../groupby/LocalGroupByExecutorTri_ILTS.java | 69 ++++++++-
.../iotdb/db/integration/tri/MyTest_ILTS_2.java | 159 +++++++++++++++++++++
5 files changed, 237 insertions(+), 7 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index b88b4557743..d68392e0e9d 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -25,7 +25,8 @@ enable_Tri=""
# segment error threshold for SimPiece, SC, FSW
epsilon=100
-#for
+auto_p1n=false
+
p1t=0
p1v=0
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 1bd01c3a5f5..b9e25221d8b 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -437,6 +437,7 @@ public class IoTDBConfig {
private double epsilon = 100; // for SimPiece
+ private boolean autoP1n = true;
private long p1t;
private double p1v;
private long pnt;
@@ -491,6 +492,14 @@ public class IoTDBConfig {
this.acc_iterRepeat = acc_iterRepeat;
}
+ public boolean getAutoP1n() {
+ return autoP1n;
+ }
+
+ public void setAutoP1n(boolean autoP1n) {
+ this.autoP1n = autoP1n;
+ }
+
public long getP1t() {
return p1t;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 70c573e1e1e..10993aa53df 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -543,6 +543,10 @@ public class IoTDBDescriptor {
conf.setEpsilon(
Double.parseDouble(
properties.getProperty("epsilon", Double.toString(conf.getEpsilon()))));
+
+ conf.setAutoP1n(
+ Boolean.parseBoolean(
+ properties.getProperty("auto_p1n", Boolean.toString(conf.getAutoP1n())).trim()));
conf.setP1t(Long.parseLong(properties.getProperty("p1t", Long.toString(conf.getP1t()))));
conf.setP1v(
Double.parseDouble(properties.getProperty("p1v", Double.toString(conf.getP1v()))));
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java
index 26a397ba229..4312cfb49fa 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java
@@ -48,6 +48,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -63,10 +64,10 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor {
// keys: 0,1,...,(int) Math.floor((endTime * 1.0 - startTime) / interval)-1
private Map<Integer, List<ChunkSuit4Tri>> splitChunkList = new HashMap<>();
- private final long p1t = CONFIG.getP1t();
- private final double p1v = CONFIG.getP1v();
- private final long pnt = CONFIG.getPnt();
- private final double pnv = CONFIG.getPnv();
+ private long p1t;
+ private double p1v;
+ private long pnt;
+ private double pnv;
private long lt;
private double lv;
@@ -136,6 +137,56 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor {
}
}
+ if (CONFIG.getAutoP1n()) {
+ // get real p1
+ List<ChunkSuit4Tri> firstBucket = splitChunkList.get(0);
+ sortByStartTime(firstBucket);
+ ChunkSuit4Tri firstChunk = firstBucket.get(0);
+ if (firstChunk.pageReader == null) {
+ firstChunk.pageReader =
+ FileLoaderUtils.loadPageReaderList4CPV(firstChunk.chunkMetadata, this.timeFilter);
+ }
+ PageReader pageReader = firstChunk.pageReader;
+ for (int j = 0; j < firstChunk.chunkMetadata.getStatistics().getCount(); j++) {
+ long timestamp = pageReader.timeBuffer.getLong(j * 8);
+ if (timestamp < startTime) {
+ continue;
+ } else if (timestamp >= startTime) {
+ ByteBuffer valueBuffer = pageReader.valueBuffer;
+ double v = valueBuffer.getDouble(pageReader.timeBufferLength + j * 8);
+ p1t = timestamp;
+ p1v = v;
+ break;
+ }
+ }
+
+ // get real pn
+ List<ChunkSuit4Tri> lastBucket = splitChunkList.get(N1 - 1);
+ sortByStartTime(lastBucket);
+ ChunkSuit4Tri lastChunk = lastBucket.get(lastBucket.size() - 1);
+ if (lastChunk.pageReader == null) {
+ lastChunk.pageReader =
+ FileLoaderUtils.loadPageReaderList4CPV(lastChunk.chunkMetadata, this.timeFilter);
+ }
+ pageReader = lastChunk.pageReader;
+ for (int j = 0; j < lastChunk.chunkMetadata.getStatistics().getCount(); j++) {
+ long timestamp = pageReader.timeBuffer.getLong(j * 8);
+ if (timestamp > endTime) { // pn can be at endTime
+ break;
+ } else {
+ ByteBuffer valueBuffer = pageReader.valueBuffer;
+ double v = valueBuffer.getDouble(pageReader.timeBufferLength + j * 8);
+ pnt = timestamp;
+ pnv = v;
+ }
+ }
+ } else {
+ p1t = CONFIG.getP1t();
+ p1v = CONFIG.getP1v();
+ pnt = CONFIG.getPnt();
+ pnv = CONFIG.getPnv();
+ }
+
} catch (IOException e) {
throw new QueryProcessException(e.getMessage());
}
@@ -144,6 +195,12 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor {
// start);
}
+ private static void sortByStartTime(List<ChunkSuit4Tri> list) {
+ Collections.sort(
+ list,
+ (a, b) -> Long.compare(a.chunkMetadata.getStartTime(), b.chunkMetadata.getStartTime()));
+ }
+
@Override
public void addAggregateResult(AggregateResult aggrResult) {
results.add(aggrResult);
@@ -170,8 +227,8 @@ public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor {
for (; num < numIterations; num++) {
// NOTE: init lt&lv at the start of each iteration is a must, because they are modified in
// each iteration
- lt = CONFIG.getP1t();
- lv = CONFIG.getP1v();
+ lt = p1t;
+ lv = p1v;
boolean allSameFlag = true;
boolean currentLeftSame = true;
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS_2.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS_2.java
new file mode 100644
index 00000000000..37fc35f5914
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS_2.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.integration.tri;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.jdbc.IoTDBStatement;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Locale;
+
+import static org.junit.Assert.fail;
+
+public class MyTest_ILTS_2 {
+
+ /*
+ * Requirements:
+ * (1) Don't change the sequence of the above two aggregates
+ * (2) Assume each chunk has only one page.
+ * (3) Assume all chunks are sequential and no deletes.
+ * (4) Assume plain encoding, UNCOMPRESSED, Long or Double data type, no compaction
+ * (5) Assume no empty bucket
+ */
+ private static final String TIMESTAMP_STR = "Time";
+
+ private static String[] creationSqls =
+ new String[] {
+ "SET STORAGE GROUP TO root.vehicle.d0",
+ "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=DOUBLE, ENCODING=PLAIN",
+ // IoTDB int data type does not support plain encoding, so use long data type
+ };
+
+ private final String d0s0 = "root.vehicle.d0.s0";
+
+ private static final String insertTemplate =
+ "INSERT INTO root.vehicle.d0(timestamp,s0)" + " VALUES(%d,%f)";
+
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+
+ @Before
+ public void setUp() throws Exception {
+ TSFileDescriptor.getInstance().getConfig().setTimeEncoder("PLAIN");
+ config.setTimestampPrecision("ms");
+ config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
+
+ config.setEnableTri("ILTS");
+ config.setAcc_avg(true);
+ config.setAcc_rectangle(true);
+ config.setAcc_convex(true);
+ config.setAcc_iterRepeat(true);
+ TSFileDescriptor.getInstance().getConfig().setWriteConvexHull(true);
+
+ config.setEnableCPV(false);
+ TSFileDescriptor.getInstance().getConfig().setEnableMinMaxLSM(false);
+ TSFileDescriptor.getInstance().getConfig().setUseStatistics(false);
+
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ }
+
+ @Test
+ public void test1() {
+ prepareData1();
+ config.setNumIterations(8);
+ String res = "10.0[2],1.0[10],8.0[30],5.0[55],20.0[62],1.0[90],7.0[102],";
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute(
+ "SELECT min_value(s0)"
+ // TODO not real min_value here, actually controlled by enableTri
+ + ",max_value(s0),min_time(s0), max_time(s0), first_value(s0), last_value(s0)"
+ + " FROM root.vehicle.d0 group by ([2,102),20ms)");
+ // (102-2)/(7-2)=20ms
+ // note keep no empty buckets
+
+ Assert.assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int i = 0;
+ while (resultSet.next()) {
+ String ans = resultSet.getString(2);
+ System.out.println(ans);
+ Assert.assertEquals(res, ans);
+ }
+ }
+ System.out.println(((IoTDBStatement) statement).executeFinish());
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void prepareData1() {
+ // data:
+ // https://user-images.githubusercontent.com/33376433/152003603-6b4e7494-00ff-47e4-bf6e-cab3c8600ce2.png
+ // slightly modified
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : creationSqls) {
+ statement.execute(sql);
+ }
+
+ long[] t = new long[] {1, 2, 10, 20, 22, 30, 40, 55, 60, 62, 65, 70, 72, 80, 90, 102};
+ double[] v = new double[] {5, 10, 1, 5, 4, 8, 2, 5, 15, 20, 8, 18, 4, 11, 1, 7};
+ config.setP1t(t[0]);
+ config.setP1v(v[0]);
+ config.setPnt(t[t.length - 1]);
+ config.setPnv(v[v.length - 1]);
+
+ for (int i = 0; i < t.length; i++) {
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, t[i], v[i]));
+ if ((i + 1) % 4 == 0) {
+ statement.execute("FLUSH");
+ }
+ }
+ statement.execute("FLUSH");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}