You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2024/07/21 15:47:44 UTC
(iotdb) branch research/LTS-visualization updated (77bb3783fa8 -> ba493800bf8)
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a change to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 77bb3783fa8 Add readme
new 00003e52f15 s
new c77a57e5041 add
new 603dc93b930 add
new 6e5f82d318b add
new 9cb402f77e4 add
new 3f4796ae153 add
new 33562646f9c Add
new 742a1b885b6 add
new 35bd7926d72 Add
new 44b0c441c7d add
new 0c93bdc7c1f add
new ba493800bf8 update readme
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 7 +-
.../resources/conf/iotdb-engine.properties | 7 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 12 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 +
.../groupby/GroupByWithoutValueFilterDataSet.java | 43 +++-
.../groupby/LocalGroupByExecutorTri_FSW.java | 163 ++++++++++++
.../groupby/LocalGroupByExecutorTri_ILTS.java | 4 -
.../groupby/LocalGroupByExecutorTri_LTTB.java | 4 -
.../groupby/LocalGroupByExecutorTri_SC.java | 163 ++++++++++++
.../groupby/LocalGroupByExecutorTri_SimPiece.java | 193 ++++++++++++++
...4.java => LocalGroupByExecutorTri_Uniform.java} | 66 +----
.../groupby/LocalGroupByExecutorTri_Visval.java | 160 ++++++++++++
.../Encoding/FloatEncoder.java} | 29 +--
.../db/query/simpiece/Encoding/IntEncoder.java} | 32 +--
.../db/query/simpiece/Encoding/UIntEncoder.java | 48 ++++
.../simpiece/Encoding/VariableByteEncoder.java | 91 +++++++
.../org/apache/iotdb/db/query/simpiece/FSW.java | 95 +++++++
.../iotdb/db/query/simpiece/MySample_fsw.java | 126 ++++++++++
.../iotdb/db/query/simpiece/MySample_fsw_full.java | 192 ++++++++++++++
.../db/query/simpiece/MySample_shrinkingcone.java | 125 +++++++++
.../simpiece/MySample_shrinkingcone_full.java | 183 ++++++++++++++
.../iotdb/db/query/simpiece/MySample_simpiece.java | 126 ++++++++++
.../db/query/simpiece/MySample_simpiece_full.java | 193 ++++++++++++++
.../org/apache/iotdb/db/query/simpiece/Point.java | 26 +-
.../iotdb/db/query/simpiece/ShrinkingCone.java | 80 ++++++
.../apache/iotdb/db/query/simpiece/SimPiece.java | 278 +++++++++++++++++++++
.../SimPieceSegment.java} | 50 ++--
.../apache/iotdb/db/query/simpiece/TimeSeries.java | 20 +-
.../iotdb/db/query/simpiece/TimeSeriesReader.java | 260 +++++++++++++++++++
.../org/apache/iotdb/db/query/simpiece/Visval.java | 70 ++++++
.../iotdb/db/query/simpiece/VisvalPoint.java | 13 +-
.../apache/iotdb/db/integration/tri/MyTest_M4.java | 3 +
32 files changed, 2718 insertions(+), 147 deletions(-)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_FSW.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SC.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SimPiece.java
copy server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/{LocalGroupByExecutorTri_M4.java => LocalGroupByExecutorTri_Uniform.java} (89%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java
copy server/src/main/java/org/apache/iotdb/db/query/{externalsort/ExternalSortJobPart.java => simpiece/Encoding/FloatEncoder.java} (59%)
copy server/src/{test/java/org/apache/iotdb/db/query/dataset/ShowDevicesResultTest.java => main/java/org/apache/iotdb/db/query/simpiece/Encoding/IntEncoder.java} (54%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/UIntEncoder.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/VariableByteEncoder.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/FSW.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full.java
copy service-rpc/src/main/java/org/apache/iotdb/rpc/NonOpenTransport.java => server/src/main/java/org/apache/iotdb/db/query/simpiece/Point.java (67%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/ShrinkingCone.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/SimPiece.java
copy server/src/main/java/org/apache/iotdb/db/query/{reader/series/BaseManagedSeriesReader.java => simpiece/SimPieceSegment.java} (51%)
copy cluster/src/main/java/org/apache/iotdb/cluster/exception/NotInSameGroupException.java => server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeries.java (67%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeriesReader.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
copy cluster/src/main/java/org/apache/iotdb/cluster/exception/QueryTimeOutException.java => server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalPoint.java (79%)
(iotdb) 05/12: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9cb402f77e4b799826a426a5f13582a97054ebe3
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jul 14 15:09:55 2024 +0800
add
---
.../resources/conf/iotdb-engine.properties | 4 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 3 +
.../groupby/LocalGroupByExecutorTri_FSW.java | 163 +++++++++++++++++++++
.../org/apache/iotdb/db/query/simpiece/FSW.java | 98 +++++++++++++
.../iotdb/db/query/simpiece/MySample_fsw.java | 78 ++++++++++
5 files changed, 344 insertions(+), 2 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 18bcfd54fa5..7cc6cfcc97a 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -19,10 +19,10 @@
####################
### enable Tri
####################
-# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece, SC
+# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece, SC, FSW
enable_Tri=MinMax
-# SimPiece segment error threshold
+# segment error threshold for SimPiece, SC, FSW
epsilon=100
#for
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 6a4f559c775..5818d4b6a04 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -441,6 +441,9 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
} else if (CONFIG.getEnableTri().equals("SC")) {
return new LocalGroupByExecutorTri_SC(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
+ } else if (CONFIG.getEnableTri().equals("FSW")) {
+ return new LocalGroupByExecutorTri_FSW(
+ path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
} else {
logger.info("No matched enable_tri!");
return new LocalGroupByExecutor(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_FSW.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_FSW.java
new file mode 100644
index 00000000000..699f88dd824
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_FSW.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.dataset.groupby;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.query.aggregation.AggregateResult;
+import org.apache.iotdb.db.query.aggregation.impl.MinValueAggrResult;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
+import org.apache.iotdb.db.query.reader.series.SeriesReader;
+import org.apache.iotdb.db.query.simpiece.FSW;
+import org.apache.iotdb.db.query.simpiece.Point;
+import org.apache.iotdb.db.query.simpiece.TimeSeries;
+import org.apache.iotdb.db.query.simpiece.TimeSeriesReader;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
+import org.apache.iotdb.tsfile.read.common.ChunkSuit4Tri;
+import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.utils.Pair;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+public class LocalGroupByExecutorTri_FSW implements GroupByExecutor {
+
+ private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
+
+ // Aggregate result buffer of this path
+ private final List<AggregateResult> results = new ArrayList<>();
+
+ TimeSeries timeSeries;
+
+ double epsilon = CONFIG.getEpsilon();
+
+ public LocalGroupByExecutorTri_FSW(
+ PartialPath path,
+ Set<String> allSensors,
+ TSDataType dataType,
+ QueryContext context,
+ Filter timeFilter,
+ TsFileFilter fileFilter,
+ boolean ascending)
+ throws StorageEngineException, QueryProcessException {
+ // long start = System.nanoTime();
+
+ // get all data sources
+ QueryDataSource queryDataSource =
+ QueryResourceManager.getInstance().getQueryDataSource(path, context, timeFilter);
+
+ // update filter by TTL
+ // this.timeFilter = queryDataSource.updateFilterUsingTTL(timeFilter);
+
+ SeriesReader seriesReader =
+ new SeriesReader(
+ path,
+ allSensors,
+ // fix bug: here use the aggregation type as the series data type,
+ // not using pageReader.getAllSatisfiedPageData is ok
+ dataType,
+ context,
+ queryDataSource,
+ timeFilter,
+ null,
+ fileFilter,
+ ascending);
+
+ try {
+ // : this might be bad to load all chunk metadata at first
+ List<ChunkSuit4Tri> futureChunkList = new ArrayList<>();
+ futureChunkList.addAll(seriesReader.getAllChunkMetadatas4Tri());
+ // order futureChunkList by chunk startTime
+ futureChunkList.sort(
+ new Comparator<ChunkSuit4Tri>() {
+ public int compare(ChunkSuit4Tri o1, ChunkSuit4Tri o2) {
+ return ((Comparable) (o1.chunkMetadata.getStartTime()))
+ .compareTo(o2.chunkMetadata.getStartTime());
+ }
+ });
+
+ GroupByFilter groupByFilter = (GroupByFilter) timeFilter;
+ long startTime = groupByFilter.getStartTime();
+ long endTime = groupByFilter.getEndTime();
+
+ timeSeries = TimeSeriesReader.getTimeSeriesFromTsFiles(futureChunkList, startTime, endTime);
+
+ } catch (IOException e) {
+ throw new QueryProcessException(e.getMessage());
+ }
+ }
+
+ @Override
+ public void addAggregateResult(AggregateResult aggrResult) {
+ results.add(aggrResult);
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(
+ long curStartTime, long curEndTime, long startTime, long endTime, long interval)
+ throws IOException {
+ // group by curStartTime and curEndTime are not used in Sim-Piece segmentation
+
+ StringBuilder series = new StringBuilder();
+
+ // clear result cache
+ for (AggregateResult result : results) {
+ result.reset();
+ }
+
+ // shrinking cone
+ int length = timeSeries.length();
+ List<Point> points = timeSeries.data;
+
+ // Shrinking cone
+ List<Point> reducedPoints = FSW.reducePoints(timeSeries.data, epsilon);
+
+ for (Point p : reducedPoints) {
+ series.append(p.getValue()).append("[").append(p.getTimestamp()).append("]").append(",");
+ }
+
+ MinValueAggrResult minValueAggrResult = (MinValueAggrResult) results.get(0);
+ minValueAggrResult.updateResult(new MinMaxInfo<>(series.toString(), 0));
+ return results;
+ }
+
+ @Override
+ public Pair<Long, Object> peekNextNotNullValue(long nextStartTime, long nextEndTime)
+ throws IOException {
+ throw new IOException("no implemented");
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(long curStartTime, long curEndTime)
+ throws IOException, QueryProcessException {
+ throw new IOException("no implemented");
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/FSW.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/FSW.java
new file mode 100644
index 00000000000..4b2e0163e1a
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/FSW.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.util.ArrayList;
+import java.util.List;
+
+// Liu, Xiaoyan, Lin, Zhenjiang, Wang, Huaiqing, 2008. Novel online methods for time series
+// segmentation.
+// FSW: feasible space window
+// adapted from https://github.com/zhourongleiden/AEPLA.git
+public class FSW {
+
+ public static List<Point> reducePoints(List<Point> points, double epsilon) {
+ int length = points.size();
+
+ // Precompute upper and lower bounds
+ double[] p_upper = new double[length];
+ double[] p_lower = new double[length];
+ for (int j = 0; j < length; j++) {
+ double v = points.get(j).getValue();
+ p_upper[j] = v + epsilon;
+ p_lower[j] = v - epsilon;
+ }
+
+ // init the first segment
+ int i = 0;
+ int seg_no = 0;
+ int csp_id = 0;
+ List<int[]> segmentPoint = new ArrayList<>();
+ segmentPoint.add(new int[] {0, 0});
+ double upSlope = Double.POSITIVE_INFINITY;
+ double lowSlope = Double.NEGATIVE_INFINITY;
+
+ while (i < length - 1) {
+ i++;
+ upSlope =
+ Math.min(
+ upSlope,
+ ((p_upper[i] - points.get(segmentPoint.get(seg_no)[1]).getValue())
+ / (points.get(i).getTimestamp()
+ - points.get(segmentPoint.get(seg_no)[1]).getTimestamp())));
+ // (i - segmentPoint.get(seg_no)[1])));
+ lowSlope =
+ Math.max(
+ lowSlope,
+ ((p_lower[i] - points.get(segmentPoint.get(seg_no)[1]).getValue())
+ / (points.get(i).getTimestamp()
+ - points.get(segmentPoint.get(seg_no)[1]).getTimestamp())));
+ // (i - segmentPoint.get(seg_no)[1])));
+ if (upSlope < lowSlope) {
+ seg_no += 1;
+ segmentPoint.add(new int[] {seg_no, csp_id});
+ i = csp_id;
+ upSlope = Double.POSITIVE_INFINITY;
+ lowSlope = Double.NEGATIVE_INFINITY;
+ } else {
+ double s =
+ (points.get(i).getValue() - points.get(segmentPoint.get(seg_no)[1]).getValue())
+ / (points.get(i).getTimestamp()
+ - points.get(segmentPoint.get(seg_no)[1]).getTimestamp());
+ // (i - segmentPoint.get(seg_no)[1]);
+ if (s >= lowSlope && s <= upSlope) {
+ csp_id = i;
+ }
+ }
+ }
+
+ // deal with the last segment
+ if (segmentPoint.get(seg_no)[1] < length - 1) {
+ seg_no += 1;
+ segmentPoint.add(new int[] {seg_no, length - 1});
+ }
+
+ List<Point> result = new ArrayList<>();
+ for (int[] ints : segmentPoint) {
+ result.add(points.get(ints[1]));
+ }
+ return result;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
new file mode 100644
index 00000000000..854962e96a6
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.util.List;
+
+public class MySample_fsw {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ boolean series = true; // 从1开始编号列而不是时间戳列
+ String[] datasetNameList =
+ new String[] {
+ // "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ // "NISTPV-Ground-2015-RTD_C_3"
+ };
+
+ int[] noutList = new int[] {0};
+
+ double[] r = new double[] {0.1, 0.5, 1.3};
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ // int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
+ // int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
+ // int N = end - start; // -1 for all
+
+ int start = 0;
+ int end = 10000;
+ int N = end - start;
+
+ for (int nout : noutList) {
+ // apply Sim-Piece on the input file, outputting nout points saved in csvFile
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ TimeSeries ts =
+ TimeSeriesReader.getMyTimeSeries(
+ inputStream, delimiter, false, N, start, hasHeader, series);
+
+ double epsilon = 0.1;
+ List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
+ try (PrintWriter writer =
+ new PrintWriter(
+ new FileWriter(datasetName + "-" + N + "-" + reducedPoints.size() + "-sc.csv"))) {
+ for (Point p : reducedPoints) {
+ writer.println(p.getTimestamp() + "," + p.getValue());
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
(iotdb) 01/12: s
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 00003e52f15a9d8aa83a165f35348ed95439b06f
Author: Lei Rui <10...@qq.com>
AuthorDate: Sat Jul 13 16:23:23 2024 +0800
s
---
.../resources/conf/iotdb-engine.properties | 5 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 +
.../groupby/GroupByWithoutValueFilterDataSet.java | 6 +-
.../groupby/LocalGroupByExecutorTri_ILTS.java | 4 -
.../groupby/LocalGroupByExecutorTri_LTTB.java | 4 -
.../groupby/LocalGroupByExecutorTri_SimPiece.java | 199 +++++++++++++++
.../db/query/simpiece/Encoding/FloatEncoder.java | 38 +++
.../db/query/simpiece/Encoding/IntEncoder.java | 46 ++++
.../db/query/simpiece/Encoding/UIntEncoder.java | 48 ++++
.../simpiece/Encoding/VariableByteEncoder.java | 91 +++++++
.../apache/iotdb/db/query/simpiece/MySample.java | 117 +++++++++
.../org/apache/iotdb/db/query/simpiece/Point.java | 41 +++
.../apache/iotdb/db/query/simpiece/SimPiece.java | 278 +++++++++++++++++++++
.../iotdb/db/query/simpiece/SimPieceSegment.java | 63 +++++
.../apache/iotdb/db/query/simpiece/TimeSeries.java | 40 +++
.../iotdb/db/query/simpiece/TimeSeriesReader.java | 166 ++++++++++++
17 files changed, 1149 insertions(+), 10 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 1e935c8bdba..60c5df8bf15 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -19,9 +19,12 @@
####################
### enable Tri
####################
-# MinMax, MinMaxLTTB, M4, LTTB, ILTS
+# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece
enable_Tri=MinMax
+# SimPiece segment error threshold
+epsilon=100
+
#for
p1t=0
p1v=0
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 765f9e2ed05..e78243dd9f1 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -435,6 +435,8 @@ public class IoTDBConfig {
private String enableTri = "ILTS"; // MinMax, MinMaxLTTB, M4, LTTB, ILTS
+ private double epsilon = 100; // for SimPiece
+
private long p1t;
private double p1v;
private long pnt;
@@ -1486,6 +1488,14 @@ public class IoTDBConfig {
return enableTri;
}
+ public double getEpsilon() {
+ return epsilon;
+ }
+
+ public void setEpsilon(double epsilon) {
+ this.epsilon = epsilon;
+ }
+
public void setEnablePerformanceTracing(boolean enablePerformanceTracing) {
this.enablePerformanceTracing = enablePerformanceTracing;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 8f8fd7b31df..70c573e1e1e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -540,6 +540,9 @@ public class IoTDBDescriptor {
properties.getProperty("enable_CPV", Boolean.toString(conf.isEnableCPV())).trim()));
conf.setEnableTri(properties.getProperty("enable_Tri", conf.getEnableTri()).trim());
+ conf.setEpsilon(
+ Double.parseDouble(
+ properties.getProperty("epsilon", Double.toString(conf.getEpsilon()))));
conf.setP1t(Long.parseLong(properties.getProperty("p1t", Long.toString(conf.getP1t()))));
conf.setP1v(
Double.parseDouble(properties.getProperty("p1v", Double.toString(conf.getP1v()))));
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index d1d42ac41aa..7bc5433d854 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -137,7 +137,8 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
if (CONFIG.getEnableTri().equals("MinMax")
|| CONFIG.getEnableTri().equals("M4")
|| CONFIG.getEnableTri().equals("LTTB")
- || CONFIG.getEnableTri().equals("ILTS")) {
+ || CONFIG.getEnableTri().equals("ILTS")
+ || CONFIG.getEnableTri().equals("SimPiece")) {
return nextWithoutConstraintTri_allInOne();
} else if (CONFIG.getEnableTri().equals("MinMaxLTTB")) {
return nextWithoutConstraintTri_MinMaxLTTB();
@@ -431,6 +432,9 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
}
return new LocalGroupByExecutorTri_ILTS(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
+ } else if (CONFIG.getEnableTri().equals("SimPiece")) {
+ return new LocalGroupByExecutorTri_SimPiece(
+ path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
} else {
logger.info("No matched enable_tri!");
return new LocalGroupByExecutor(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java
index d4c6122fbbe..b22a7d8d9a9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java
@@ -44,9 +44,6 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
import org.apache.iotdb.tsfile.utils.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -59,7 +56,6 @@ import java.util.Set;
public class LocalGroupByExecutorTri_ILTS implements GroupByExecutor {
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private static final Logger M4_CHUNK_METADATA = LoggerFactory.getLogger("M4_CHUNK_METADATA");
// Aggregate result buffer of this path
private final List<AggregateResult> results = new ArrayList<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_LTTB.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_LTTB.java
index a8e4fe77f2e..cba1543aff8 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_LTTB.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_LTTB.java
@@ -43,9 +43,6 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.reader.page.PageReader;
import org.apache.iotdb.tsfile.utils.Pair;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -57,7 +54,6 @@ import java.util.Set;
public class LocalGroupByExecutorTri_LTTB implements GroupByExecutor {
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private static final Logger M4_CHUNK_METADATA = LoggerFactory.getLogger("M4_CHUNK_METADATA");
// Aggregate result buffer of this path
private final List<AggregateResult> results = new ArrayList<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SimPiece.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SimPiece.java
new file mode 100644
index 00000000000..0a9b4fbefa5
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SimPiece.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.dataset.groupby;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.query.aggregation.AggregateResult;
+import org.apache.iotdb.db.query.aggregation.impl.MinValueAggrResult;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
+import org.apache.iotdb.db.query.reader.series.SeriesReader;
+import org.apache.iotdb.db.query.simpiece.SimPiece;
+import org.apache.iotdb.db.query.simpiece.SimPieceSegment;
+import org.apache.iotdb.db.query.simpiece.TimeSeries;
+import org.apache.iotdb.db.query.simpiece.TimeSeriesReader;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
+import org.apache.iotdb.tsfile.read.common.ChunkSuit4Tri;
+import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.utils.Pair;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+public class LocalGroupByExecutorTri_SimPiece implements GroupByExecutor {
+
+ private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
+
+ // Aggregate result buffer of this path
+ private final List<AggregateResult> results = new ArrayList<>();
+
+ TimeSeries timeSeries;
+
+ double epsilon = CONFIG.getEpsilon();
+
+ public LocalGroupByExecutorTri_SimPiece(
+ PartialPath path,
+ Set<String> allSensors,
+ TSDataType dataType,
+ QueryContext context,
+ Filter timeFilter,
+ TsFileFilter fileFilter,
+ boolean ascending)
+ throws StorageEngineException, QueryProcessException {
+ // long start = System.nanoTime();
+
+ // get all data sources
+ QueryDataSource queryDataSource =
+ QueryResourceManager.getInstance().getQueryDataSource(path, context, timeFilter);
+
+ // update filter by TTL
+ // this.timeFilter = queryDataSource.updateFilterUsingTTL(timeFilter);
+
+ SeriesReader seriesReader =
+ new SeriesReader(
+ path,
+ allSensors,
+ // fix bug: here use the aggregation type as the series data type,
+ // not using pageReader.getAllSatisfiedPageData is ok
+ dataType,
+ context,
+ queryDataSource,
+ timeFilter,
+ null,
+ fileFilter,
+ ascending);
+
+ try {
+ // : this might be bad to load all chunk metadata at first
+ List<ChunkSuit4Tri> futureChunkList = new ArrayList<>();
+ futureChunkList.addAll(seriesReader.getAllChunkMetadatas4Tri());
+ // order futureChunkList by chunk startTime
+ futureChunkList.sort(
+ new Comparator<ChunkSuit4Tri>() {
+ public int compare(ChunkSuit4Tri o1, ChunkSuit4Tri o2) {
+ return ((Comparable) (o1.chunkMetadata.getStartTime()))
+ .compareTo(o2.chunkMetadata.getStartTime());
+ }
+ });
+
+ GroupByFilter groupByFilter = (GroupByFilter) timeFilter;
+ long startTime = groupByFilter.getStartTime();
+ long endTime = groupByFilter.getEndTime();
+
+ timeSeries = TimeSeriesReader.getTimeSeriesFromTsFiles(futureChunkList, startTime, endTime);
+
+ } catch (IOException e) {
+ throw new QueryProcessException(e.getMessage());
+ }
+ }
+
+ @Override
+ public void addAggregateResult(AggregateResult aggrResult) {
+ results.add(aggrResult);
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(
+ long curStartTime, long curEndTime, long startTime, long endTime, long interval)
+ throws IOException {
+ // group by curStartTime and curEndTime are not used in Sim-Piece segmentation
+
+ StringBuilder series = new StringBuilder();
+
+ // clear result cache
+ for (AggregateResult result : results) {
+ result.reset();
+ }
+
+ //
+ // series.append(CONFIG.getP1v()).append("[").append(CONFIG.getP1t()).append("]").append(",");
+
+ SimPiece simPiece = new SimPiece(timeSeries.data, epsilon);
+ List<SimPieceSegment> segments = simPiece.segments;
+ segments.sort(Comparator.comparingLong(SimPieceSegment::getInitTimestamp));
+ for (int i = 0; i < segments.size() - 1; i++) {
+ // end point of this segment
+ double v =
+ (segments.get(i + 1).getInitTimestamp() - segments.get(i).getInitTimestamp())
+ * segments.get(i).getA()
+ + segments.get(i).getB();
+
+ series
+ .append(segments.get(i).getB()) // start point of this segment
+ .append("[")
+ .append(segments.get(i).getInitTimestamp())
+ .append("]")
+ .append(",")
+ .append(v) // end point of this segment
+ .append("[")
+ .append(segments.get(i + 1).getInitTimestamp())
+ .append("]")
+ .append(",");
+ }
+ // the two end points of the last segment
+ double v =
+ (simPiece.lastTimeStamp - segments.get(segments.size() - 1).getInitTimestamp())
+ * segments.get(segments.size() - 1).getA()
+ + segments.get(segments.size() - 1).getB();
+
+ series
+ .append(segments.get(segments.size() - 1).getB())
+ .append("[")
+ .append(segments.get(segments.size() - 1).getInitTimestamp())
+ .append("]")
+ .append(",")
+ .append(v)
+ .append("[")
+ .append(simPiece.lastTimeStamp)
+ .append("]")
+ .append(",");
+
+ //
+ // series.append(CONFIG.getPnv()).append("[").append(CONFIG.getPnt()).append("]").append(",");
+
+ MinValueAggrResult minValueAggrResult = (MinValueAggrResult) results.get(0);
+ minValueAggrResult.updateResult(new MinMaxInfo<>(series.toString(), 0));
+
+ return results;
+ }
+
+ @Override
+ public Pair<Long, Object> peekNextNotNullValue(long nextStartTime, long nextEndTime)
+ throws IOException {
+ throw new IOException("no implemented");
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(long curStartTime, long curEndTime)
+ throws IOException, QueryProcessException {
+ throw new IOException("no implemented");
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/FloatEncoder.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/FloatEncoder.java
new file mode 100644
index 00000000000..f8811760cf9
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/FloatEncoder.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece.Encoding;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+public class FloatEncoder {
+ public static void write(float number, ByteArrayOutputStream outputStream) throws IOException {
+ int intBits = Float.floatToIntBits(number);
+ IntEncoder.write(intBits, outputStream);
+ }
+
+ public static float read(ByteArrayInputStream inputStream) throws IOException {
+ int number = IntEncoder.read(inputStream);
+ return Float.intBitsToFloat(number);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/IntEncoder.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/IntEncoder.java
new file mode 100644
index 00000000000..40d913abd2b
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/IntEncoder.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece.Encoding;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class IntEncoder {
+ public static void write(int number, ByteArrayOutputStream outputStream) throws IOException {
+ ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
+ buffer.putInt(number);
+ outputStream.write(buffer.array());
+ }
+
+ public static int read(ByteArrayInputStream inputStream) throws IOException {
+ byte[] byteArray = new byte[Integer.BYTES];
+ int k = inputStream.read(byteArray);
+ if (k != Integer.BYTES) throw new IOException();
+ ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
+ buffer.put(byteArray);
+ buffer.flip();
+
+ return buffer.getInt();
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/UIntEncoder.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/UIntEncoder.java
new file mode 100644
index 00000000000..f7f25168a08
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/UIntEncoder.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece.Encoding;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class UIntEncoder {
+ public static void write(long number, ByteArrayOutputStream outputStream) throws IOException {
+ if (number > Math.pow(2, 8 * 4) - 1 || number < 0)
+ throw new UnsupportedOperationException("Can't save number " + number + " as unsigned int");
+ ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
+ buffer.putInt((int) (number & 0xffffffffL));
+ outputStream.write(buffer.array());
+ }
+
+ public static long read(ByteArrayInputStream inputStream) throws IOException {
+ byte[] byteArray = new byte[Integer.BYTES];
+ int k = inputStream.read(byteArray);
+ if (k != Integer.BYTES) throw new IOException();
+ ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
+ buffer.put(byteArray);
+ buffer.flip();
+
+ return buffer.getInt() & 0xffffffffL;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/VariableByteEncoder.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/VariableByteEncoder.java
new file mode 100644
index 00000000000..93115f461a0
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Encoding/VariableByteEncoder.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece.Encoding;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+
+/*
+ * Source code by:
+ * https://github.com/lemire/JavaFastPFOR/blob/master/src/main/java/me/lemire/integercompression/VariableByte.java
+ */
+public class VariableByteEncoder {
+
+ private static byte extract7bits(int i, long val) {
+ return (byte) ((val >> (7 * i)) & ((1 << 7) - 1));
+ }
+
+ private static byte extract7bitsmaskless(int i, long val) {
+ return (byte) ((val >> (7 * i)));
+ }
+
+ public static void write(int number, ByteArrayOutputStream outputStream) {
+ final long val = number & 0xFFFFFFFFL;
+
+ if (val < (1 << 7)) {
+ outputStream.write((byte) (val | (1 << 7)));
+ } else if (val < (1 << 14)) {
+ outputStream.write(extract7bits(0, val));
+ outputStream.write((byte) (extract7bitsmaskless(1, (val)) | (1 << 7)));
+ } else if (val < (1 << 21)) {
+ outputStream.write(extract7bits(0, val));
+ outputStream.write(extract7bits(1, val));
+ outputStream.write((byte) (extract7bitsmaskless(2, (val)) | (1 << 7)));
+ } else if (val < (1 << 28)) {
+ outputStream.write(extract7bits(0, val));
+ outputStream.write(extract7bits(1, val));
+ outputStream.write(extract7bits(2, val));
+ outputStream.write((byte) (extract7bitsmaskless(3, (val)) | (1 << 7)));
+ } else {
+ outputStream.write(extract7bits(0, val));
+ outputStream.write(extract7bits(1, val));
+ outputStream.write(extract7bits(2, val));
+ outputStream.write(extract7bits(3, val));
+ outputStream.write((byte) (extract7bitsmaskless(4, (val)) | (1 << 7)));
+ }
+ }
+
+ public static int read(ByteArrayInputStream inputStream) {
+ byte in;
+ int number;
+
+ in = (byte) inputStream.read();
+ number = in & 0x7F;
+ if (in < 0) return number;
+
+ in = (byte) inputStream.read();
+ number = ((in & 0x7F) << 7) | number;
+ if (in < 0) return number;
+
+ in = (byte) inputStream.read();
+ number = ((in & 0x7F) << 14) | number;
+ if (in < 0) return number;
+
+ in = (byte) inputStream.read();
+ number = ((in & 0x7F) << 21) | number;
+ if (in < 0) return number;
+
+ number = (((byte) inputStream.read() & 0x7F) << 28) | number;
+
+ return number;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample.java
new file mode 100644
index 00000000000..a54d60ac1d1
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Comparator;
+import java.util.List;
+
+public class MySample {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ boolean series = true; // 从1开始编号列而不是时间戳列
+ String[] datasetNameList =
+ new String[] {
+ "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3"
+ };
+ int[] noutList = new int[] {100};
+ double[] r = new double[] {0.1, 0.5, 1.3};
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
+ int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
+ int N = end - start; // -1 for all
+
+ for (int nout : noutList) {
+ // apply Sim-Piece on the input file, outputting nout points saved in csvFile
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ TimeSeries ts =
+ TimeSeriesReader.getMyTimeSeries(
+ inputStream, delimiter, false, N, start, hasHeader, series);
+ double epsilon = getSimPieceParam(nout, ts, 1e-6);
+ System.out.println(datasetName + ": n=" + N + ",m=" + nout + ",epsilon=" + epsilon);
+ SimPiece simPiece = new SimPiece(ts.data, epsilon);
+ List<SimPieceSegment> segments = simPiece.segments;
+ segments.sort(Comparator.comparingLong(SimPieceSegment::getInitTimestamp));
+ try (PrintWriter writer =
+ new PrintWriter(new FileWriter(datasetName + "-" + N + "-" + nout + ".csv"))) {
+ for (int i = 0; i < segments.size() - 1; i++) {
+ // start point of this segment
+ writer.println(segments.get(i).getInitTimestamp() + "," + segments.get(i).getB());
+ // end point of this segment
+ double v =
+ (segments.get(i + 1).getInitTimestamp() - segments.get(i).getInitTimestamp())
+ * segments.get(i).getA()
+ + segments.get(i).getB();
+ writer.println(segments.get(i + 1).getInitTimestamp() + "," + v);
+ }
+ // the two end points of the last segment
+ writer.println(
+ segments.get(segments.size() - 1).getInitTimestamp()
+ + ","
+ + segments.get(segments.size() - 1).getB());
+ double v =
+ (simPiece.lastTimeStamp - segments.get(segments.size() - 1).getInitTimestamp())
+ * segments.get(segments.size() - 1).getA()
+ + segments.get(segments.size() - 1).getB();
+ writer.println(simPiece.lastTimeStamp + "," + v);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ public static double getSimPieceParam(int nout, TimeSeries ts, double accuracy)
+ throws IOException {
+ double epsilon = ts.range * 0.001;
+ while (true) {
+ SimPiece simPiece = new SimPiece(ts.data, epsilon);
+ if (simPiece.segments.size() * 2 > nout) { // note *2 for disjoint
+ epsilon *= 2;
+ } else {
+ break;
+ }
+ }
+ double left = epsilon / 2;
+ double right = epsilon;
+ while (Math.abs(right - left) > accuracy) {
+ double mid = (left + right) / 2;
+ SimPiece simPiece = new SimPiece(ts.data, mid);
+ if (simPiece.segments.size() * 2 > nout) { // note *2 for disjoint
+ left = mid;
+ } else {
+ right = mid;
+ }
+ }
+ return (left + right) / 2;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Point.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Point.java
new file mode 100644
index 00000000000..e8d96065a0c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Point.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+public class Point {
+
+ private final long timestamp;
+ private final double value;
+
+ public Point(long timestamp, double value) {
+ this.timestamp = timestamp;
+ this.value = value;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public double getValue() {
+ return value;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/SimPiece.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/SimPiece.java
new file mode 100644
index 00000000000..a7c491f81c9
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/SimPiece.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import org.apache.iotdb.db.query.simpiece.Encoding.FloatEncoder;
+import org.apache.iotdb.db.query.simpiece.Encoding.UIntEncoder;
+import org.apache.iotdb.db.query.simpiece.Encoding.VariableByteEncoder;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.*;
+
+public class SimPiece {
+ public ArrayList<SimPieceSegment> segments;
+
+ private double epsilon;
+ public long lastTimeStamp;
+
+ public SimPiece(List<Point> points, double epsilon) throws IOException {
+ if (points.isEmpty()) throw new IOException();
+
+ this.epsilon = epsilon;
+ this.lastTimeStamp = points.get(points.size() - 1).getTimestamp();
+ this.segments = mergePerB(compress(points));
+ }
+
+ // public SimPiece(byte[] bytes, boolean variableByte, boolean zstd) throws IOException {
+ // readByteArray(bytes, variableByte, zstd);
+ // }
+
+ private double quantization(double value) {
+ return Math.round(value / epsilon) * epsilon;
+ }
+
+ private int createSegment(int startIdx, List<Point> points, ArrayList<SimPieceSegment> segments) {
+ long initTimestamp = points.get(startIdx).getTimestamp();
+ double b = quantization(points.get(startIdx).getValue());
+ if (startIdx + 1 == points.size()) {
+ segments.add(new SimPieceSegment(initTimestamp, -Double.MAX_VALUE, Double.MAX_VALUE, b));
+ return startIdx + 1;
+ }
+ double aMax =
+ ((points.get(startIdx + 1).getValue() + epsilon) - b)
+ / (points.get(startIdx + 1).getTimestamp() - initTimestamp);
+ double aMin =
+ ((points.get(startIdx + 1).getValue() - epsilon) - b)
+ / (points.get(startIdx + 1).getTimestamp() - initTimestamp);
+ if (startIdx + 2 == points.size()) {
+ segments.add(new SimPieceSegment(initTimestamp, aMin, aMax, b));
+ return startIdx + 2;
+ }
+
+ for (int idx = startIdx + 2; idx < points.size(); idx++) {
+ double upValue = points.get(idx).getValue() + epsilon;
+ double downValue = points.get(idx).getValue() - epsilon;
+
+ double upLim = aMax * (points.get(idx).getTimestamp() - initTimestamp) + b;
+ double downLim = aMin * (points.get(idx).getTimestamp() - initTimestamp) + b;
+ if ((downValue > upLim || upValue < downLim)) {
+ segments.add(new SimPieceSegment(initTimestamp, aMin, aMax, b));
+ return idx;
+ }
+
+ if (upValue < upLim)
+ aMax = Math.max((upValue - b) / (points.get(idx).getTimestamp() - initTimestamp), aMin);
+ if (downValue > downLim)
+ aMin = Math.min((downValue - b) / (points.get(idx).getTimestamp() - initTimestamp), aMax);
+ }
+ segments.add(new SimPieceSegment(initTimestamp, aMin, aMax, b));
+
+ return points.size();
+ }
+
+ private ArrayList<SimPieceSegment> compress(List<Point> points) {
+ ArrayList<SimPieceSegment> segments = new ArrayList<>();
+ int currentIdx = 0;
+ while (currentIdx < points.size()) currentIdx = createSegment(currentIdx, points, segments);
+
+ return segments;
+ }
+
+ private ArrayList<SimPieceSegment> mergePerB(ArrayList<SimPieceSegment> segments) {
+ double aMinTemp = -Double.MAX_VALUE;
+ double aMaxTemp = Double.MAX_VALUE;
+ double b = Double.NaN;
+ ArrayList<Long> timestamps = new ArrayList<>();
+ ArrayList<SimPieceSegment> mergedSegments = new ArrayList<>();
+
+ segments.sort(
+ Comparator.comparingDouble(SimPieceSegment::getB)
+ .thenComparingDouble(SimPieceSegment::getA));
+ for (int i = 0; i < segments.size(); i++) {
+ if (b != segments.get(i).getB()) {
+ if (timestamps.size() == 1)
+ mergedSegments.add(new SimPieceSegment(timestamps.get(0), aMinTemp, aMaxTemp, b));
+ else {
+ for (Long timestamp : timestamps)
+ mergedSegments.add(new SimPieceSegment(timestamp, aMinTemp, aMaxTemp, b));
+ }
+ timestamps.clear();
+ timestamps.add(segments.get(i).getInitTimestamp());
+ aMinTemp = segments.get(i).getAMin();
+ aMaxTemp = segments.get(i).getAMax();
+ b = segments.get(i).getB();
+ continue;
+ }
+ if (segments.get(i).getAMin() <= aMaxTemp && segments.get(i).getAMax() >= aMinTemp) {
+ timestamps.add(segments.get(i).getInitTimestamp());
+ aMinTemp = Math.max(aMinTemp, segments.get(i).getAMin());
+ aMaxTemp = Math.min(aMaxTemp, segments.get(i).getAMax());
+ } else {
+ if (timestamps.size() == 1) mergedSegments.add(segments.get(i - 1));
+ else {
+ for (long timestamp : timestamps)
+ mergedSegments.add(new SimPieceSegment(timestamp, aMinTemp, aMaxTemp, b));
+ }
+ timestamps.clear();
+ timestamps.add(segments.get(i).getInitTimestamp());
+ aMinTemp = segments.get(i).getAMin();
+ aMaxTemp = segments.get(i).getAMax();
+ }
+ }
+ if (!timestamps.isEmpty()) {
+ if (timestamps.size() == 1)
+ mergedSegments.add(new SimPieceSegment(timestamps.get(0), aMinTemp, aMaxTemp, b));
+ else {
+ for (long timestamp : timestamps)
+ mergedSegments.add(new SimPieceSegment(timestamp, aMinTemp, aMaxTemp, b));
+ }
+ }
+
+ return mergedSegments;
+ }
+
+ public List<Point> decompress() {
+ segments.sort(Comparator.comparingLong(SimPieceSegment::getInitTimestamp));
+ List<Point> points = new ArrayList<>();
+ long currentTimeStamp = segments.get(0).getInitTimestamp();
+
+ for (int i = 0; i < segments.size() - 1; i++) {
+ while (currentTimeStamp < segments.get(i + 1).getInitTimestamp()) {
+ points.add(
+ new Point(
+ currentTimeStamp,
+ segments.get(i).getA() * (currentTimeStamp - segments.get(i).getInitTimestamp())
+ + segments.get(i).getB()));
+ currentTimeStamp++;
+ }
+ }
+
+ while (currentTimeStamp <= lastTimeStamp) {
+ points.add(
+ new Point(
+ currentTimeStamp,
+ segments.get(segments.size() - 1).getA()
+ * (currentTimeStamp - segments.get(segments.size() - 1).getInitTimestamp())
+ + segments.get(segments.size() - 1).getB()));
+ currentTimeStamp++;
+ }
+
+ return points;
+ }
+
+ private void toByteArrayPerBSegments(
+ ArrayList<SimPieceSegment> segments, boolean variableByte, ByteArrayOutputStream outStream)
+ throws IOException {
+ TreeMap<Integer, HashMap<Double, ArrayList<Long>>> input = new TreeMap<>();
+ for (SimPieceSegment segment : segments) {
+ double a = segment.getA();
+ int b = (int) Math.round(segment.getB() / epsilon);
+ long t = segment.getInitTimestamp();
+ if (!input.containsKey(b)) input.put(b, new HashMap<>());
+ if (!input.get(b).containsKey(a)) input.get(b).put(a, new ArrayList<>());
+ input.get(b).get(a).add(t);
+ }
+
+ VariableByteEncoder.write(input.size(), outStream);
+ if (input.isEmpty()) return;
+ int previousB = input.firstKey();
+ VariableByteEncoder.write(previousB, outStream);
+ for (Map.Entry<Integer, HashMap<Double, ArrayList<Long>>> bSegments : input.entrySet()) {
+ VariableByteEncoder.write(bSegments.getKey() - previousB, outStream);
+ previousB = bSegments.getKey();
+ VariableByteEncoder.write(bSegments.getValue().size(), outStream);
+ for (Map.Entry<Double, ArrayList<Long>> aSegment : bSegments.getValue().entrySet()) {
+ FloatEncoder.write(aSegment.getKey().floatValue(), outStream);
+ if (variableByte) Collections.sort(aSegment.getValue());
+ VariableByteEncoder.write(aSegment.getValue().size(), outStream);
+ long previousTS = 0;
+ for (Long timestamp : aSegment.getValue()) {
+ if (variableByte) VariableByteEncoder.write((int) (timestamp - previousTS), outStream);
+ else UIntEncoder.write(timestamp, outStream);
+ previousTS = timestamp;
+ }
+ }
+ }
+ }
+
+ //
+ // public byte[] toByteArray(boolean variableByte, boolean zstd) throws IOException {
+ // ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+ // byte[] bytes;
+ //
+ // FloatEncoder.write((float) epsilon, outStream);
+ //
+ // toByteArrayPerBSegments(segments, variableByte, outStream);
+ //
+ // if (variableByte) VariableByteEncoder.write((int) lastTimeStamp, outStream);
+ // else UIntEncoder.write(lastTimeStamp, outStream);
+ //
+ // if (zstd) bytes = Zstd.compress(outStream.toByteArray());
+ // else bytes = outStream.toByteArray();
+ //
+ // outStream.close();
+ //
+ // return bytes;
+ // }
+
+ private ArrayList<SimPieceSegment> readMergedPerBSegments(
+ boolean variableByte, ByteArrayInputStream inStream) throws IOException {
+ ArrayList<SimPieceSegment> segments = new ArrayList<>();
+ long numB = VariableByteEncoder.read(inStream);
+ if (numB == 0) return segments;
+ int previousB = VariableByteEncoder.read(inStream);
+ for (int i = 0; i < numB; i++) {
+ int b = VariableByteEncoder.read(inStream) + previousB;
+ previousB = b;
+ int numA = VariableByteEncoder.read(inStream);
+ for (int j = 0; j < numA; j++) {
+ float a = FloatEncoder.read(inStream);
+ int numTimestamps = VariableByteEncoder.read(inStream);
+ long timestamp = 0;
+ for (int k = 0; k < numTimestamps; k++) {
+ if (variableByte) timestamp += VariableByteEncoder.read(inStream);
+ else timestamp = UIntEncoder.read(inStream);
+ segments.add(new SimPieceSegment(timestamp, a, (float) (b * epsilon)));
+ }
+ }
+ }
+
+ return segments;
+ }
+
+ // private void readByteArray(byte[] input, boolean variableByte, boolean zstd) throws
+ // IOException {
+ // byte[] binary;
+ // if (zstd) binary = Zstd.decompress(input, input.length * 2); //TODO: How to know apriori
+ // original size?
+ // else binary = input;
+ // ByteArrayInputStream inStream = new ByteArrayInputStream(binary);
+ //
+ // this.epsilon = FloatEncoder.read(inStream);
+ // this.segments = readMergedPerBSegments(variableByte, inStream);
+ // if (variableByte) this.lastTimeStamp = VariableByteEncoder.read(inStream);
+ // else this.lastTimeStamp = UIntEncoder.read(inStream);
+ // inStream.close();
+ // }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/SimPieceSegment.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/SimPieceSegment.java
new file mode 100644
index 00000000000..e443f06027e
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/SimPieceSegment.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+public class SimPieceSegment {
+
+ private final long initTimestamp;
+ private final double aMin;
+ private final double aMax;
+ private final double a;
+ private final double b;
+
+ public SimPieceSegment(long initTimestamp, double a, double b) {
+ this(initTimestamp, a, a, b);
+ }
+
+ public SimPieceSegment(long initTimestamp, double aMin, double aMax, double b) {
+ this.initTimestamp = initTimestamp;
+ this.aMin = aMin;
+ this.aMax = aMax;
+ this.a = (aMin + aMax) / 2;
+ this.b = b;
+ }
+
+ public long getInitTimestamp() {
+ return initTimestamp;
+ }
+
+ public double getAMin() {
+ return aMin;
+ }
+
+ public double getAMax() {
+ return aMax;
+ }
+
+ public double getA() {
+ return a;
+ }
+
+ public double getB() {
+ return b;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeries.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeries.java
new file mode 100644
index 00000000000..31ab7307e20
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeries.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.util.List;
+
+public class TimeSeries {
+ public List<Point> data;
+ public double range;
+ public int size;
+
+ public TimeSeries(List<Point> data, double range) {
+ this.data = data;
+ this.range = range;
+ this.size = data.size() * (4 + 4);
+ }
+
+ public int length() {
+ return data.size();
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeriesReader.java
new file mode 100644
index 00000000000..129baeefeda
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeriesReader.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import org.apache.iotdb.db.utils.FileLoaderUtils;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.read.common.ChunkSuit4Tri;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.GZIPInputStream;
+
+public class TimeSeriesReader {
+
+ public static TimeSeries getTimeSeriesFromTsFiles(
+ List<ChunkSuit4Tri> chunkSuit4TriList, long startTime, long endTime) throws IOException {
+ // assume chunkSuit4TriList already sorted in increasing time order
+ ArrayList<Point> ts = new ArrayList<>();
+ double max = Double.MIN_VALUE;
+ double min = Double.MAX_VALUE;
+
+ for (ChunkSuit4Tri chunkSuit4Tri : chunkSuit4TriList) {
+ ChunkMetadata chunkMetadata = chunkSuit4Tri.chunkMetadata;
+ long chunkMinTime = chunkMetadata.getStartTime();
+ long chunkMaxTime = chunkMetadata.getEndTime();
+ if (chunkMaxTime < startTime) {
+ continue;
+ } else if (chunkMinTime >= endTime) {
+ break;
+ } else {
+ PageReader pageReader =
+ FileLoaderUtils.loadPageReaderList4CPV(
+ chunkSuit4Tri.chunkMetadata,
+ null); // note do not assign to chunkSuit4Tri.pageReader
+ for (int j = 0; j < chunkSuit4Tri.chunkMetadata.getStatistics().getCount(); j++) {
+ IOMonitor2.DCP_D_getAllSatisfiedPageData_traversedPointNum++;
+ long timestamp = pageReader.timeBuffer.getLong(j * 8);
+ if (timestamp < startTime) {
+ continue;
+ } else if (timestamp >= endTime) {
+ break;
+ } else { // rightStartTime<=t<rightEndTime
+ ByteBuffer valueBuffer = pageReader.valueBuffer;
+ double value = valueBuffer.getDouble(pageReader.timeBufferLength + j * 8);
+ ts.add(new Point(timestamp, value));
+ max = Math.max(max, value);
+ min = Math.min(min, value);
+ }
+ }
+ }
+ }
+ return new TimeSeries(ts, max - min);
+ }
+
+ public static TimeSeries getTimeSeries(InputStream inputStream, String delimiter, boolean gzip) {
+ ArrayList<Point> ts = new ArrayList<>();
+ double max = Double.MIN_VALUE;
+ double min = Double.MAX_VALUE;
+
+ try {
+ if (gzip) {
+ inputStream = new GZIPInputStream(inputStream);
+ }
+ Reader decoder = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
+ BufferedReader bufferedReader = new BufferedReader(decoder);
+
+ String line;
+ while ((line = bufferedReader.readLine()) != null) {
+ String[] elements = line.split(delimiter);
+ long timestamp = Long.parseLong(elements[0]);
+ double value = Double.parseDouble(elements[1]);
+ ts.add(new Point(timestamp, value));
+
+ max = Math.max(max, value);
+ min = Math.min(min, value);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return new TimeSeries(ts, max - min);
+ }
+
+ public static TimeSeries getMyTimeSeries(
+ InputStream inputStream,
+ String delimiter,
+ boolean gzip,
+ int N,
+ int startRow,
+ boolean hasHeader,
+ boolean seriesTimeColumn) {
+ // N<0 means read all lines
+
+ ArrayList<Point> ts = new ArrayList<>();
+ double max = Double.MIN_VALUE;
+ double min = Double.MAX_VALUE;
+
+ try {
+ if (gzip) {
+ inputStream = new GZIPInputStream(inputStream);
+ }
+ Reader decoder = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
+ BufferedReader bufferedReader = new BufferedReader(decoder);
+
+ String line;
+ if (hasHeader) {
+ bufferedReader.readLine();
+ }
+ int startCnt = 0;
+ while (startCnt < startRow && (line = bufferedReader.readLine()) != null) {
+ startCnt++;
+ }
+ if (startCnt < startRow) {
+ throw new IOException("not enough rows!");
+ }
+ int cnt = 0;
+ while ((cnt < N || N < 0) && (line = bufferedReader.readLine()) != null) {
+ String[] elements = line.split(delimiter);
+ long timestamp;
+ if (!seriesTimeColumn) {
+ timestamp = (long) Double.parseDouble(elements[0]);
+ } else {
+ timestamp = cnt + 1;
+ }
+ double value = Double.parseDouble(elements[1]);
+ ts.add(new Point(timestamp, value));
+
+ max = Math.max(max, value);
+ min = Math.min(min, value);
+ cnt++;
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return new TimeSeries(ts, max - min);
+ }
+}
(iotdb) 11/12: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0c93bdc7c1fed26dd1fee7c8ec0f56b7c34e1576
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jul 17 04:54:11 2024 +0800
add
---
.../iotdb/db/query/simpiece/MySample_fsw.java | 24 ++++++++++--------
.../db/query/simpiece/MySample_shrinkingcone.java | 25 +++++++++++--------
.../simpiece/MySample_shrinkingcone_full.java | 2 +-
.../iotdb/db/query/simpiece/MySample_simpiece.java | 29 ++++++++++++----------
4 files changed, 44 insertions(+), 36 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
index ed4eefe0b50..da43f5b824e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
@@ -32,18 +32,20 @@ public class MySample_fsw {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
String[] datasetNameList =
- new String[] {
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3"
+ new String[]{
+ "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3",
+ "NISTPV-Ground-2015-WindSpeed_ms"
};
- int[] noutList = new int[] {100};
- double[] r = new double[] {0.1, 0.5, 1.3};
- double[] epsilonList = new double[] {9.999999E-4, 284.4034399986267, 6.428162097930908};
- for (int y = 0; y < datasetNameList.length; y++) {
+ int[] noutList = new int[]{100};
+ double[] r = new double[]{0.1, 0.5, 1.3, 0};
+ int[] NList = new int[]{2500000, 2500000, 2500000, 500000};
+ double[] epsilonList = new double[]{9.999999E-4, 284.4034399986267, 6.428162097930908,10.814893245697021};
+ for (int y = 3; y < datasetNameList.length; y++) {
String datasetName = datasetNameList[y];
- int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
- int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
+ int start = (int) (10000000 / 2 - NList[y] * r[y]); // 从0开始计数
+ int end = (int) (10000000 / 2 + NList[y] * (1 - r[y]));
int N = end - start;
// int start = 0;
// int end = 10000;
@@ -56,7 +58,7 @@ public class MySample_fsw {
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
inputStream, delimiter, false, N, start, hasHeader, false);
- // double epsilon = getFSWParam(nout, ts, 1e-6);
+// double epsilon = getFSWParam(nout, ts, 1e-6);
double epsilon = epsilonList[y];
List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
System.out.println(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
index d09641736fd..1b167064f62 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
@@ -32,18 +32,21 @@ public class MySample_shrinkingcone {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
String[] datasetNameList =
- new String[] {
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3"
+ new String[]{
+ "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3",
+ "NISTPV-Ground-2015-WindSpeed_ms"
};
- int[] noutList = new int[] {100};
- double[] r = new double[] {0.1, 0.5, 1.3};
- double[] epsilonList = new double[] {0.001, 408.55843019485474, 7.996772289276123};
- for (int y = 0; y < datasetNameList.length; y++) {
+ int[] noutList = new int[]{100};
+ double[] r = new double[]{0.1, 0.5, 1.3, 0};
+ int[] NList = new int[]{2500000, 2500000, 2500000, 500000};
+ double[] epsilonList = new double[]{0.001, 408.55843019485474, 7.996772289276123,
+ 14.692305088043213};
+ for (int y = 3; y < datasetNameList.length; y++) {
String datasetName = datasetNameList[y];
- int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
- int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
+ int start = (int) (10000000 / 2 - NList[y] * r[y]); // 从0开始计数
+ int end = (int) (10000000 / 2 + NList[y] * (1 - r[y]));
int N = end - start;
// int start = 0;
// int end = 10000;
@@ -56,7 +59,7 @@ public class MySample_shrinkingcone {
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
inputStream, delimiter, false, N, start, hasHeader, false);
- // double epsilon = getSCParam(nout, ts, 1e-6);
+// double epsilon = getSCParam(nout, ts, 1e-6);
double epsilon = epsilonList[y];
List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
System.out.println(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java
index d0c7aeef63d..6cb8dd16a92 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java
@@ -92,7 +92,7 @@ public class MySample_shrinkingcone_full {
int end = 1000_0000;
int N = end - start;
boolean hasHeader = false;
- try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
String delimiter = ",";
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
index 9d7afe6fca8..8792c3a327e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
@@ -33,18 +33,21 @@ public class MySample_simpiece {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
String[] datasetNameList =
- new String[] {
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3"
+ new String[]{
+ "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3",
+ "NISTPV-Ground-2015-WindSpeed_ms"
};
- int[] noutList = new int[] {100};
- double[] r = new double[] {0.1, 0.5, 1.3};
- double[] epsilonList = new double[] {0.0009999, 316.5642651891633, 9.186667042922977};
- for (int y = 0; y < datasetNameList.length; y++) {
+ int[] noutList = new int[]{100};
+ double[] r = new double[]{0.1, 0.5, 1.3, 0};
+ double[] epsilonList = new double[]{0.0009999, 316.5642651891633, 9.186667042922977,
+ 11.162719900131227};
+ int[] NList = new int[]{2500000, 2500000, 2500000, 500000};
+ for (int y = 3; y < datasetNameList.length; y++) {
String datasetName = datasetNameList[y];
- int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
- int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
+ int start = (int) (10000000 / 2 - NList[y] * r[y]); // 从0开始计数
+ int end = (int) (10000000 / 2 + NList[y] * (1 - r[y]));
int N = end - start;
// int start = 0;
// int end = 10000;
@@ -58,7 +61,7 @@ public class MySample_simpiece {
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
inputStream, delimiter, false, N, start, hasHeader, false);
- // double epsilon = getSimPieceParam(nout, ts, 1e-6);
+// double epsilon = getSimPieceParam(nout, ts, 1e-6);
double epsilon = epsilonList[y];
SimPiece simPiece = new SimPiece(ts.data, epsilon);
System.out.println(
@@ -83,7 +86,7 @@ public class MySample_simpiece {
// end point of this segment
double v =
(segments.get(i + 1).getInitTimestamp() - segments.get(i).getInitTimestamp())
- * segments.get(i).getA()
+ * segments.get(i).getA()
+ segments.get(i).getB();
writer.println(segments.get(i + 1).getInitTimestamp() + "," + v);
}
@@ -94,7 +97,7 @@ public class MySample_simpiece {
+ segments.get(segments.size() - 1).getB());
double v =
(simPiece.lastTimeStamp - segments.get(segments.size() - 1).getInitTimestamp())
- * segments.get(segments.size() - 1).getA()
+ * segments.get(segments.size() - 1).getA()
+ segments.get(segments.size() - 1).getB();
writer.println(simPiece.lastTimeStamp + "," + v);
}
(iotdb) 08/12: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 742a1b885b65592b245960c4b24de9c2a5ed9680
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jul 15 02:14:43 2024 +0800
add
---
.../resources/conf/iotdb-engine.properties | 2 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 3 +
.../groupby/LocalGroupByExecutorTri_Uniform.java | 396 +++++++++++++++++++++
3 files changed, 400 insertions(+), 1 deletion(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 7cc6cfcc97a..e0d4d6e20d4 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -19,7 +19,7 @@
####################
### enable Tri
####################
-# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece, SC, FSW
+# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece, SC, FSW, Uniform
enable_Tri=MinMax
# segment error threshold for SimPiece, SC, FSW
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 5818d4b6a04..df2869e776f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -444,6 +444,9 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
} else if (CONFIG.getEnableTri().equals("FSW")) {
return new LocalGroupByExecutorTri_FSW(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
+ } else if (CONFIG.getEnableTri().equals("Uniform")) {
+ return new LocalGroupByExecutorTri_Uniform(
+ path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
} else {
logger.info("No matched enable_tri!");
return new LocalGroupByExecutor(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Uniform.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Uniform.java
new file mode 100644
index 00000000000..1145376b544
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Uniform.java
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.dataset.groupby;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.query.aggregation.AggregateResult;
+import org.apache.iotdb.db.query.aggregation.impl.MinValueAggrResult;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
+import org.apache.iotdb.db.query.reader.series.SeriesReader;
+import org.apache.iotdb.db.utils.FileLoaderUtils;
+import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.DoubleStatistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.FloatStatistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.IntegerStatistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.LongStatistics;
+import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+import org.apache.iotdb.tsfile.read.common.ChunkSuit4Tri;
+import org.apache.iotdb.tsfile.read.common.IOMonitor2;
+import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+import org.apache.iotdb.tsfile.utils.Pair;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Set;
+
+public class LocalGroupByExecutorTri_Uniform implements GroupByExecutor {
+
+ private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
+
+ private static final Logger M4_CHUNK_METADATA = LoggerFactory.getLogger("M4_CHUNK_METADATA");
+
+ // Aggregate result buffer of this path
+ private final List<AggregateResult> results = new ArrayList<>();
+
+ private List<ChunkSuit4Tri> currentChunkList;
+ private final List<ChunkSuit4Tri> futureChunkList = new ArrayList<>();
+
+ private Filter timeFilter;
+
+ private final int N1;
+
+ public LocalGroupByExecutorTri_Uniform(
+ PartialPath path,
+ Set<String> allSensors,
+ TSDataType dataType,
+ QueryContext context,
+ Filter timeFilter,
+ TsFileFilter fileFilter,
+ boolean ascending)
+ throws StorageEngineException, QueryProcessException {
+ // long start = System.nanoTime();
+
+ // get all data sources
+ QueryDataSource queryDataSource =
+ QueryResourceManager.getInstance().getQueryDataSource(path, context, this.timeFilter);
+
+ // update filter by TTL
+ this.timeFilter = queryDataSource.updateFilterUsingTTL(timeFilter);
+
+ SeriesReader seriesReader =
+ new SeriesReader(
+ path,
+ allSensors,
+ // fix bug: here use the aggregation type as the series data type,
+ // not using pageReader.getAllSatisfiedPageData is ok
+ dataType,
+ context,
+ queryDataSource,
+ timeFilter,
+ null,
+ fileFilter,
+ ascending);
+
+ GroupByFilter groupByFilter = (GroupByFilter) timeFilter;
+ long startTime = groupByFilter.getStartTime();
+ long endTime = groupByFilter.getEndTime();
+ long interval = groupByFilter.getInterval();
+ N1 = (int) Math.floor((endTime * 1.0 - startTime) / interval); // 分桶数
+
+ try {
+ // : this might be bad to load all chunk metadata at first
+ futureChunkList.addAll(seriesReader.getAllChunkMetadatas4Tri());
+ // order futureChunkList by chunk startTime
+ futureChunkList.sort(
+ new Comparator<ChunkSuit4Tri>() {
+ public int compare(ChunkSuit4Tri o1, ChunkSuit4Tri o2) {
+ return ((Comparable) (o1.chunkMetadata.getStartTime()))
+ .compareTo(o2.chunkMetadata.getStartTime());
+ }
+ });
+
+ if (M4_CHUNK_METADATA.isDebugEnabled()) {
+ if (timeFilter instanceof GroupByFilter) {
+ M4_CHUNK_METADATA.debug(
+ "M4_QUERY_PARAM,{},{},{}",
+ ((GroupByFilter) timeFilter).getStartTime(),
+ ((GroupByFilter) timeFilter).getEndTime(),
+ ((GroupByFilter) timeFilter).getInterval());
+ }
+ for (ChunkSuit4Tri ChunkSuit4Tri : futureChunkList) {
+ Statistics statistics = ChunkSuit4Tri.chunkMetadata.getStatistics();
+ long FP_t = statistics.getStartTime();
+ long LP_t = statistics.getEndTime();
+ long BP_t = statistics.getBottomTimestamp();
+ long TP_t = statistics.getTopTimestamp();
+ switch (statistics.getType()) {
+ case INT32:
+ int FP_v_int = ((IntegerStatistics) statistics).getFirstValue();
+ int LP_v_int = ((IntegerStatistics) statistics).getLastValue();
+ int BP_v_int = ((IntegerStatistics) statistics).getMinValue();
+ int TP_v_int = ((IntegerStatistics) statistics).getMaxValue();
+ M4_CHUNK_METADATA.debug(
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}",
+ FP_t,
+ LP_t,
+ BP_t,
+ TP_t,
+ FP_v_int,
+ LP_v_int,
+ BP_v_int,
+ TP_v_int,
+ ChunkSuit4Tri.chunkMetadata.getVersion(),
+ ChunkSuit4Tri.chunkMetadata.getOffsetOfChunkHeader(),
+ statistics.getCount());
+ break;
+ case INT64:
+ long FP_v_long = ((LongStatistics) statistics).getFirstValue();
+ long LP_v_long = ((LongStatistics) statistics).getLastValue();
+ long BP_v_long = ((LongStatistics) statistics).getMinValue();
+ long TP_v_long = ((LongStatistics) statistics).getMaxValue();
+ M4_CHUNK_METADATA.debug(
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}",
+ FP_t,
+ LP_t,
+ BP_t,
+ TP_t,
+ FP_v_long,
+ LP_v_long,
+ BP_v_long,
+ TP_v_long,
+ ChunkSuit4Tri.chunkMetadata.getVersion(),
+ ChunkSuit4Tri.chunkMetadata.getOffsetOfChunkHeader(),
+ statistics.getCount());
+ break;
+ case FLOAT:
+ float FP_v_float = ((FloatStatistics) statistics).getFirstValue();
+ float LP_v_float = ((FloatStatistics) statistics).getLastValue();
+ float BP_v_float = ((FloatStatistics) statistics).getMinValue();
+ float TP_v_float = ((FloatStatistics) statistics).getMaxValue();
+ M4_CHUNK_METADATA.debug(
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}",
+ FP_t,
+ LP_t,
+ BP_t,
+ TP_t,
+ FP_v_float,
+ LP_v_float,
+ BP_v_float,
+ TP_v_float,
+ ChunkSuit4Tri.chunkMetadata.getVersion(),
+ ChunkSuit4Tri.chunkMetadata.getOffsetOfChunkHeader(),
+ statistics.getCount());
+ break;
+ case DOUBLE:
+ double FP_v_double = ((DoubleStatistics) statistics).getFirstValue();
+ double LP_v_double = ((DoubleStatistics) statistics).getLastValue();
+ double BP_v_double = ((DoubleStatistics) statistics).getMinValue();
+ double TP_v_double = ((DoubleStatistics) statistics).getMaxValue();
+ M4_CHUNK_METADATA.debug(
+ "M4_CHUNK_METADATA,{},{},{},{},{},{},{},{},{},{},{}",
+ FP_t,
+ LP_t,
+ BP_t,
+ TP_t,
+ FP_v_double,
+ LP_v_double,
+ BP_v_double,
+ TP_v_double,
+ ChunkSuit4Tri.chunkMetadata.getVersion(),
+ ChunkSuit4Tri.chunkMetadata.getOffsetOfChunkHeader(),
+ statistics.getCount());
+ break;
+ default:
+ throw new QueryProcessException("unsupported data type!");
+ }
+ }
+ }
+
+ } catch (IOException e) {
+ throw new QueryProcessException(e.getMessage());
+ }
+
+ // IOMonitor2.addMeasure(Operation.M4_LSM_INIT_LOAD_ALL_CHUNKMETADATAS, System.nanoTime() -
+ // start);
+ }
+
+ @Override
+ public void addAggregateResult(AggregateResult aggrResult) {
+ results.add(aggrResult);
+ }
+
+ private void getCurrentChunkListFromFutureChunkList(long curStartTime, long curEndTime) {
+ // IOMonitor2.M4_LSM_status = Operation.M4_LSM_MERGE_M4_TIME_SPAN;
+
+ // empty currentChunkList
+ currentChunkList = new ArrayList<>();
+
+ // iterate futureChunkList
+ ListIterator<ChunkSuit4Tri> itr = futureChunkList.listIterator();
+ while (itr.hasNext()) {
+ ChunkSuit4Tri chunkSuit4Tri = itr.next();
+ ChunkMetadata chunkMetadata = chunkSuit4Tri.chunkMetadata;
+ long chunkMinTime = chunkMetadata.getStartTime();
+ long chunkMaxTime = chunkMetadata.getEndTime();
+ if (chunkMaxTime < curStartTime) {
+ // the chunk falls on the left side of the current M4 interval Ii
+ itr.remove();
+ } else if (chunkMinTime >= curEndTime) {
+ // the chunk falls on the right side of the current M4 interval Ii,
+ // and since futureChunkList is ordered by the startTime of chunkMetadata,
+ // the loop can be terminated early.
+ break;
+ } else if (chunkMaxTime < curEndTime) {
+ // this chunk is not related to buckets later
+ currentChunkList.add(chunkSuit4Tri);
+ itr.remove();
+ } else {
+ // this chunk is overlapped with the right border of the current bucket
+ currentChunkList.add(chunkSuit4Tri);
+ // still keep it in the futureChunkList
+ }
+ }
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(
+ long curStartTime, long curEndTime, long startTime, long endTime, long interval)
+ throws IOException {
+ StringBuilder series = new StringBuilder();
+
+ // clear result cache
+ for (AggregateResult result : results) {
+ result.reset();
+ }
+
+ series.append(CONFIG.getP1v()).append("[").append(CONFIG.getP1t()).append("]").append(",");
+
+ // Assume no empty buckets
+ for (int b = 0; b < N1; b++) {
+ long localCurStartTime = startTime + (b) * interval;
+ long localCurEndTime = startTime + (b + 1) * interval;
+
+ getCurrentChunkListFromFutureChunkList(localCurStartTime, localCurEndTime);
+
+ if (currentChunkList.size() == 0) {
+ // minValue[bottomTime],maxValue[topTime],firstValue[firstTime],lastValue[lastTime]
+ series.append("null[null],null[null],null[null],null[null],");
+ continue;
+ }
+
+ calculateFirst(currentChunkList, localCurStartTime, localCurEndTime, series);
+ }
+
+ series.append(CONFIG.getPnv()).append("[").append(CONFIG.getPnt()).append("]").append(",");
+
+ MinValueAggrResult minValueAggrResult = (MinValueAggrResult) results.get(0);
+ minValueAggrResult.updateResult(new MinMaxInfo<>(series.toString(), 0));
+
+ return results;
+ }
+
+ private void calculateFirst(
+ List<ChunkSuit4Tri> currentChunkList,
+ long curStartTime,
+ long curEndTime,
+ StringBuilder series)
+ throws IOException {
+ long firstTime = -1;
+ double firstValue = 0;
+
+ for (ChunkSuit4Tri chunkSuit4Tri : currentChunkList) {
+
+ Statistics statistics = chunkSuit4Tri.chunkMetadata.getStatistics();
+
+ if (canUseStatistics(chunkSuit4Tri, curStartTime, curEndTime)) {
+ // update FP
+ if (firstTime < 0) {
+ firstTime = statistics.getStartTime();
+ firstValue = (double) statistics.getFirstValue();
+ }
+ } else { // cannot use statistics directly
+ // 1. load page data if it hasn't been loaded
+ TSDataType dataType = chunkSuit4Tri.chunkMetadata.getDataType();
+ if (dataType != TSDataType.DOUBLE) {
+ throw new UnSupportedDataTypeException(String.valueOf(dataType));
+ }
+ if (chunkSuit4Tri.pageReader == null) {
+ chunkSuit4Tri.pageReader =
+ FileLoaderUtils.loadPageReaderList4CPV(chunkSuit4Tri.chunkMetadata, this.timeFilter);
+ // ATTENTION: YOU HAVE TO ENSURE THAT THERE IS ONLY ONE PAGE IN A CHUNK,
+ // BECAUSE THE WHOLE IMPLEMENTATION IS BASED ON THIS ASSUMPTION.
+ // OTHERWISE, PAGEREADER IS FOR THE FIRST PAGE IN THE CHUNK WHILE
+ // STEPREGRESS IS FOR THE LAST PAGE IN THE CHUNK (THE MERGE OF STEPREGRESS IS
+ // ASSIGN DIRECTLY), WHICH WILL INTRODUCE BUGS!
+ }
+
+ int count = chunkSuit4Tri.chunkMetadata.getStatistics().getCount();
+ PageReader pageReader = chunkSuit4Tri.pageReader;
+ int i;
+ for (i = chunkSuit4Tri.lastReadPos; i < count; i++) {
+ IOMonitor2.DCP_D_getAllSatisfiedPageData_traversedPointNum++;
+ long timestamp = pageReader.timeBuffer.getLong(i * 8);
+ if (timestamp < curStartTime) {
+ // 2. read from lastReadPos until the first point fallen within this bucket (if it
+ // exists)
+ continue;
+ } else if (timestamp >= curEndTime) {
+ // 3. traverse until the first point fallen right this bucket, also remember to update
+ // lastReadPos
+ chunkSuit4Tri.lastReadPos = i;
+ break;
+ } else {
+ // 4. update MinMax by traversing points fallen within this bucket
+ ByteBuffer valueBuffer = pageReader.valueBuffer;
+ double v = valueBuffer.getDouble(pageReader.timeBufferLength + i * 8);
+ if (firstTime < 0) {
+ firstTime = timestamp;
+ firstValue = v;
+ break;
+ }
+ }
+ }
+ }
+ }
+ if (firstTime >= 0) {
+ // minValue[bottomTime],maxValue[topTime],firstValue[firstTime],lastValue[lastTime]
+ series.append(firstValue).append("[").append(firstTime).append("]").append(",");
+ } else {
+ // empty bucket although statistics cover
+ // minValue[bottomTime],maxValue[topTime],firstValue[firstTime],lastValue[lastTime]
+ series.append("null[null],");
+ }
+ }
+
+ public boolean canUseStatistics(ChunkSuit4Tri chunkSuit4Tri, long curStartTime, long curEndTime) {
+ return false;
+ }
+
+ @Override
+ public Pair<Long, Object> peekNextNotNullValue(long nextStartTime, long nextEndTime)
+ throws IOException {
+ throw new IOException("no implemented");
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(long curStartTime, long curEndTime)
+ throws IOException, QueryProcessException {
+ throw new IOException("no implemented");
+ }
+}
(iotdb) 03/12: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 603dc93b930dc42dfd7c531e584aab93909cc595
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jul 14 01:23:48 2024 +0800
add
---
.../db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 322c1f51076..5453d6be36a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -274,6 +274,7 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
int cnt = 0;
for (int j = nextBucket * rps; j < (nextBucket + 1) * rps; j++) {
if (times.get(j) != null) {
+ IOMonitor2.DCP_D_getAllSatisfiedPageData_traversedPointNum++;
rt += times.get(j);
rv += (double) values.get(j);
cnt++;
@@ -290,6 +291,7 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
double select_v = -1;
for (int j = currentBucket * rps; j < (currentBucket + 1) * rps; j++) {
if (times.get(j) != null) {
+ IOMonitor2.DCP_D_getAllSatisfiedPageData_traversedPointNum++;
long t = times.get(j);
double v = values.get(j);
double area = IOMonitor2.calculateTri(lt, lv, t, v, rt, rv);
@@ -315,6 +317,7 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
double select_v = -1;
for (int j = currentBucket * rps; j < (currentBucket + 1) * rps; j++) {
if (times.get(j) != null) {
+ IOMonitor2.DCP_D_getAllSatisfiedPageData_traversedPointNum++;
long t = times.get(j);
double v = values.get(j);
double area = IOMonitor2.calculateTri(lt, lv, t, v, pnt, pnv); // 全局尾点作为右边固定点
(iotdb) 09/12: Add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 35bd7926d72a552c0872bacd3fe2f21d0fb92514
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jul 15 03:28:45 2024 +0800
Add
---
.../resources/conf/iotdb-engine.properties | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 26 ++-
.../query/simpiece/MySample_fsw_full_serial.java | 198 ++++++++++++++++++++
.../MySample_shrinkingcone_full_serial.java | 195 ++++++++++++++++++++
.../simpiece/MySample_simpiece_full_serial.java | 199 +++++++++++++++++++++
.../apache/iotdb/db/integration/tri/MyTest_M4.java | 3 +
7 files changed, 616 insertions(+), 9 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index e0d4d6e20d4..b7a8c2130b6 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -20,7 +20,7 @@
### enable Tri
####################
# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece, SC, FSW, Uniform
-enable_Tri=MinMax
+enable_Tri=""
# segment error threshold for SimPiece, SC, FSW
epsilon=100
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index e78243dd9f1..1bd01c3a5f5 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -433,7 +433,7 @@ public class IoTDBConfig {
private boolean enableCPV = false;
- private String enableTri = "ILTS"; // MinMax, MinMaxLTTB, M4, LTTB, ILTS
+ private String enableTri = ""; // MinMax, MinMaxLTTB, M4, LTTB, ILTS
private double epsilon = 100; // for SimPiece
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index df2869e776f..97da407a4a3 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -134,17 +134,29 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
@Override
public RowRecord nextWithoutConstraint() throws IOException {
- if (CONFIG.getEnableTri().equals("MinMax")
- || CONFIG.getEnableTri().equals("M4")
- || CONFIG.getEnableTri().equals("LTTB")
- || CONFIG.getEnableTri().equals("ILTS")
- || CONFIG.getEnableTri().equals("SimPiece")) {
- return nextWithoutConstraintTri_allInOne();
- } else if (CONFIG.getEnableTri().equals("MinMaxLTTB")) {
+ if (CONFIG.getEnableTri().equals("MinMaxLTTB")) {
return nextWithoutConstraintTri_MinMaxLTTB();
+ } else if (!CONFIG.getEnableTri().equals("")) {
+ return nextWithoutConstraintTri_allInOne();
} else {
return nextWithoutConstraint_raw();
}
+
+ // if (CONFIG.getEnableTri().equals("MinMax")
+ // || CONFIG.getEnableTri().equals("M4")
+ // || CONFIG.getEnableTri().equals("LTTB")
+ // || CONFIG.getEnableTri().equals("ILTS")
+ // || CONFIG.getEnableTri().equals("SimPiece")
+ // || CONFIG.getEnableTri().equals("SC")
+ // || CONFIG.getEnableTri().equals("FSW")
+ // || CONFIG.getEnableTri().equals("Uniform")
+ // ) {
+ // return nextWithoutConstraintTri_allInOne();
+ // } else if (CONFIG.getEnableTri().equals("MinMaxLTTB")) {
+ // return nextWithoutConstraintTri_MinMaxLTTB();
+ // } else {
+ // return nextWithoutConstraint_raw();
+ // }
}
public RowRecord nextWithoutConstraintTri_allInOne() throws IOException {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full_serial.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full_serial.java
new file mode 100644
index 00000000000..b51c8dddcb2
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full_serial.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.List;
+
+public class MySample_fsw_full_serial {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ String[] datasetNameList =
+ new String[] {
+ "NISTPV-Ground-2015-WindSpeed_ms",
+ "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3"
+ };
+ int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
+
+ double[][] epsilonArray = {
+ {
+ 9.618271350860596,
+ 9.101282596588135,
+ 8.991526126861572,
+ 8.62817907333374,
+ 8.374999523162842,
+ 8.202636241912842,
+ 7.999906063079834,
+ 7.947059154510498,
+ 7.858933925628662
+ },
+ {
+ 9.999275207519531E-4,
+ 9.989738464355469E-4,
+ 9.980201721191406E-4,
+ 9.970664978027344E-4,
+ 9.961128234863281E-4,
+ 9.951591491699219E-4,
+ 9.942054748535156E-4,
+ 9.932518005371094E-4,
+ 9.922981262207031E-4
+ },
+ {
+ 440.0235962867737,
+ 432.0247492790222,
+ 423.55674982070923,
+ 414.65001153945923,
+ 405.77113008499146,
+ 401.21820974349976,
+ 396.7935194969177,
+ 393.2065939903259,
+ 392.30953645706177
+ },
+ {
+ 9.295397281646729,
+ 8.096678256988525,
+ 7.536666393280029,
+ 6.961357593536377,
+ 6.473535060882568,
+ 6.064479351043701,
+ 5.743286609649658,
+ 5.5219340324401855,
+ 5.185042858123779
+ }
+ };
+
+ // double[][] epsilonArray = new double[datasetNameList.length][];
+ // for (int i = 0; i < datasetNameList.length; i++) {
+ // epsilonArray[i] = new double[noutList.length];
+ // }
+
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ int start = 0;
+ int end = 1000_0000;
+ int N = end - start;
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ TimeSeries ts =
+ TimeSeriesReader.getMyTimeSeries(
+ inputStream, delimiter, false, N, start, hasHeader, true);
+ for (int x = 0; x < noutList.length; x++) {
+ int nout = noutList[x];
+ // double epsilon = getFSWParam(nout, ts, 1e-6);
+ // epsilonArray[y][x] = epsilon;
+
+ double epsilon = epsilonArray[y][x];
+
+ List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
+ System.out.println(
+ datasetName
+ + ": n="
+ + N
+ + ",m="
+ + nout
+ + ",epsilon="
+ + epsilon
+ + ",actual m="
+ + reducedPoints.size());
+ try (PrintWriter writer =
+ new PrintWriter(
+ new FileWriter(
+ datasetName
+ + "-"
+ + N
+ + "-"
+ + nout
+ + "-"
+ + reducedPoints.size()
+ + "-fsw.csv"))) {
+ for (Point p : reducedPoints) {
+ writer.println(p.getTimestamp() + "," + p.getValue());
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (int i = 0; i < epsilonArray.length; i++) { // 遍历行
+ for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列
+ System.out.print(epsilonArray[i][j] + ",");
+ }
+ System.out.println();
+ }
+ }
+
+ public static double getFSWParam(int nout, TimeSeries ts, double accuracy) throws IOException {
+ double epsilon = 1;
+ boolean directLess = false;
+ boolean directMore = false;
+ while (true) {
+ List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
+ if (reducedPoints.size() > nout) {
+ if (directMore) {
+ break;
+ }
+ if (!directLess) {
+ directLess = true;
+ }
+ epsilon *= 2;
+ } else {
+ if (directLess) {
+ break;
+ }
+ if (!directMore) {
+ directMore = true;
+ }
+ epsilon /= 2;
+ }
+ }
+ double left = 0;
+ double right = 0;
+ if (directLess) {
+ left = epsilon / 2;
+ right = epsilon;
+ }
+ if (directMore) {
+ left = epsilon;
+ right = epsilon * 2;
+ }
+ while (Math.abs(right - left) > accuracy) {
+ double mid = (left + right) / 2;
+ List<Point> reducedPoints = FSW.reducePoints(ts.data, mid);
+ if (reducedPoints.size() > nout) {
+ left = mid;
+ } else {
+ right = mid;
+ }
+ }
+ return (left + right) / 2;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full_serial.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full_serial.java
new file mode 100644
index 00000000000..c2705919029
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full_serial.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.List;
+
+public class MySample_shrinkingcone_full_serial {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ String[] datasetNameList =
+ new String[] {
+ "NISTPV-Ground-2015-WindSpeed_ms",
+ "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3"
+ };
+ int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
+
+ // double[] epsilonList = new double[]{0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001,
+ // 0.001,
+ // 0.0009999999999999999};
+
+ double[][] epsilonArray = {
+ {
+ 14.388325214385986,
+ 13.768231868743896,
+ 13.343345165252686,
+ 13.091249942779541,
+ 12.69767427444458,
+ 12.391706943511963,
+ 12.167190074920654,
+ 11.972727298736572,
+ 11.756273746490479
+ },
+ {0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.0009999999999999999},
+ {
+ 680.2196469306946,
+ 665.2842917442322,
+ 650.2610821723938,
+ 636.7945942878723,
+ 620.9016032218933,
+ 604.7394433021545,
+ 592.9867577552795,
+ 578.6471419334412,
+ 570.5106825828552
+ },
+ {
+ 11.259880542755127,
+ 10.37297010421753,
+ 9.402754306793213,
+ 8.867334842681885,
+ 8.384315967559814,
+ 7.915340900421143,
+ 7.39516019821167,
+ 6.892223834991455,
+ 6.5178914070129395
+ }
+ };
+
+ // double[][] epsilonArray = new double[datasetNameList.length][];
+ // for (int i = 0; i < datasetNameList.length; i++) {
+ // epsilonArray[i] = new double[noutList.length];
+ // }
+
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ int start = 0;
+ int end = 1000_0000;
+ int N = end - start;
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ TimeSeries ts =
+ TimeSeriesReader.getMyTimeSeries(
+ inputStream, delimiter, false, N, start, hasHeader, true);
+ for (int x = 0; x < noutList.length; x++) {
+ int nout = noutList[x];
+
+ // double epsilon = getSCParam(nout, ts, 1e-6);
+ // epsilonArray[y][x] = epsilon;
+
+ double epsilon = epsilonArray[y][x];
+
+ // double epsilon = epsilonList[x]; // 单独给Qloss手调
+
+ List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
+ System.out.println(
+ datasetName
+ + ": n="
+ + N
+ + ",m="
+ + nout
+ + ",epsilon="
+ + epsilon
+ + ",actual m="
+ + reducedPoints.size());
+ try (PrintWriter writer =
+ new PrintWriter(
+ new FileWriter(
+ datasetName
+ + "-"
+ + N
+ + "-"
+ + nout
+ + "-"
+ + reducedPoints.size()
+ + "-sc.csv"))) {
+ for (Point p : reducedPoints) {
+ writer.println(p.getTimestamp() + "," + p.getValue());
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (int i = 0; i < epsilonArray.length; i++) { // 遍历行
+ for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列
+ System.out.print(epsilonArray[i][j] + ",");
+ }
+ System.out.println();
+ }
+ }
+
+ public static double getSCParam(int nout, TimeSeries ts, double accuracy) throws IOException {
+ double epsilon = 1;
+ boolean directLess = false;
+ boolean directMore = false;
+ while (true) {
+ List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
+ if (reducedPoints.size() > nout) {
+ if (directMore) {
+ break;
+ }
+ if (!directLess) {
+ directLess = true;
+ }
+ epsilon *= 2;
+ } else {
+ if (directLess) {
+ break;
+ }
+ if (!directMore) {
+ directMore = true;
+ }
+ epsilon /= 2;
+ }
+ }
+ double left = 0;
+ double right = 0;
+ if (directLess) {
+ left = epsilon / 2;
+ right = epsilon;
+ }
+ if (directMore) {
+ left = epsilon;
+ right = epsilon * 2;
+ }
+ while (Math.abs(right - left) > accuracy) {
+ double mid = (left + right) / 2;
+ List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, mid);
+ if (reducedPoints.size() > nout) {
+ left = mid;
+ } else {
+ right = mid;
+ }
+ }
+ return (left + right) / 2;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full_serial.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full_serial.java
new file mode 100644
index 00000000000..4dd8c88eaf7
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full_serial.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Comparator;
+import java.util.List;
+
+public class MySample_simpiece_full_serial {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ String[] datasetNameList =
+ new String[] {
+ "NISTPV-Ground-2015-WindSpeed_ms",
+ "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3"
+ };
+ int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
+
+ double[][] epsilonArray = {
+ {
+ 10.470000203704831,
+ 9.973635465240477,
+ 9.571347576141356,
+ 9.339938166046142,
+ 9.208912633514405,
+ 9.135163892364506,
+ 8.744444478607178,
+ 8.61042620162964,
+ 8.422101323699948
+ },
+ {
+ 9.987499999999999E-4,
+ 9.974999999999997E-4,
+ 9.784999999999998E-4,
+ 9.4425E-4,
+ 9.255E-4,
+ 8.8375E-4,
+ 8.4775E-4,
+ 7.865E-4,
+ 7.3825E-4
+ },
+ {
+ 502.0505219552564,
+ 493.3274919509581,
+ 485.91779255051347,
+ 479.26180967157245,
+ 463.5216509033579,
+ 446.88677012006065,
+ 441.2913785749938,
+ 435.45897277584766,
+ 426.0481592410648
+ },
+ {
+ 13.678959740142822,
+ 12.275238493499756,
+ 11.224719192962645,
+ 10.475999729766848,
+ 9.741776887359617,
+ 9.125823136444094,
+ 8.625197060394289,
+ 8.105121862335203,
+ 7.743280215911865
+ }
+ };
+
+ // double[][] epsilonArray = new double[datasetNameList.length][];
+ // for (int i = 0; i < datasetNameList.length; i++) {
+ // epsilonArray[i] = new double[noutList.length];
+ // }
+
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ int start = 0;
+ int end = 1000_0000;
+ int N = end - start;
+ // apply Sim-Piece on the input file, outputting nout points saved in csvFile
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ TimeSeries ts =
+ TimeSeriesReader.getMyTimeSeries(
+ inputStream, delimiter, false, N, start, hasHeader, true);
+ for (int x = 0; x < noutList.length; x++) {
+ int nout = noutList[x];
+
+ // double epsilon = getSimPieceParam(nout, ts, 1e-6);
+ // epsilonArray[y][x] = epsilon;
+
+ double epsilon = epsilonArray[y][x];
+
+ SimPiece simPiece = new SimPiece(ts.data, epsilon);
+ System.out.println(
+ datasetName
+ + ": n="
+ + N
+ + ",m="
+ + nout
+ + ",epsilon="
+ + epsilon
+ + ",actual m="
+ + simPiece.segments.size() * 2);
+ List<SimPieceSegment> segments = simPiece.segments;
+ segments.sort(Comparator.comparingLong(SimPieceSegment::getInitTimestamp));
+ try (PrintWriter writer =
+ new PrintWriter(
+ new FileWriter(
+ datasetName
+ + "-"
+ + N
+ + "-"
+ + nout
+ + "-"
+ + segments.size() * 2
+ + "-simpiece.csv"))) {
+ for (int i = 0; i < segments.size() - 1; i++) {
+ // start point of this segment
+ writer.println(segments.get(i).getInitTimestamp() + "," + segments.get(i).getB());
+ // end point of this segment
+ double v =
+ (segments.get(i + 1).getInitTimestamp() - segments.get(i).getInitTimestamp())
+ * segments.get(i).getA()
+ + segments.get(i).getB();
+ writer.println(segments.get(i + 1).getInitTimestamp() + "," + v);
+ }
+ // the two end points of the last segment
+ writer.println(
+ segments.get(segments.size() - 1).getInitTimestamp()
+ + ","
+ + segments.get(segments.size() - 1).getB());
+ double v =
+ (simPiece.lastTimeStamp - segments.get(segments.size() - 1).getInitTimestamp())
+ * segments.get(segments.size() - 1).getA()
+ + segments.get(segments.size() - 1).getB();
+ writer.println(simPiece.lastTimeStamp + "," + v);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (int i = 0; i < epsilonArray.length; i++) { // 遍历行
+ for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列
+ System.out.print(epsilonArray[i][j] + ",");
+ }
+ System.out.println();
+ }
+ }
+
+ public static double getSimPieceParam(int nout, TimeSeries ts, double accuracy)
+ throws IOException {
+ double epsilon = ts.range * 0.001;
+ while (true) {
+ SimPiece simPiece = new SimPiece(ts.data, epsilon);
+ if (simPiece.segments.size() * 2 > nout) { // note *2 for disjoint
+ epsilon *= 2;
+ } else {
+ break;
+ }
+ }
+ double left = epsilon / 2;
+ double right = epsilon;
+ while (Math.abs(right - left) > accuracy) {
+ double mid = (left + right) / 2;
+ SimPiece simPiece = new SimPiece(ts.data, mid);
+ if (simPiece.segments.size() * 2 > nout) { // note *2 for disjoint
+ left = mid;
+ } else {
+ right = mid;
+ }
+ }
+ return (left + right) / 2;
+ }
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_M4.java b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_M4.java
index 96edde2c040..ff0a3fc7afa 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_M4.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_M4.java
@@ -72,6 +72,9 @@ public class MyTest_M4 {
config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
config.setEnableTri("M4");
+ // config.setEnableTri("SC");
+ // config.setEpsilon(0.1);
+
config.setP1t(0);
config.setP1v(0);
config.setPnt(200);
(iotdb) 04/12: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6e5f82d318bdedee8a0aaf3475ce8c4553c948fa
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jul 14 10:11:23 2024 +0800
add
---
.../resources/conf/iotdb-engine.properties | 2 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 3 -
.../groupby/LocalGroupByExecutorTri_Visval.java | 161 ------------------
.../iotdb/db/query/simpiece/MySample_visval.java | 92 ----------
.../org/apache/iotdb/db/query/simpiece/Visval.java | 189 ---------------------
5 files changed, 1 insertion(+), 446 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index f26d8c0385a..18bcfd54fa5 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -19,7 +19,7 @@
####################
### enable Tri
####################
-# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece, SC, Visval
+# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece, SC
enable_Tri=MinMax
# SimPiece segment error threshold
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 5453d6be36a..6a4f559c775 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -441,9 +441,6 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
} else if (CONFIG.getEnableTri().equals("SC")) {
return new LocalGroupByExecutorTri_SC(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
- } else if (CONFIG.getEnableTri().equals("Visval")) {
- return new LocalGroupByExecutorTri_Visval(
- path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
} else {
logger.info("No matched enable_tri!");
return new LocalGroupByExecutor(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java
deleted file mode 100644
index 0091e370620..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.query.dataset.groupby;
-
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
-import org.apache.iotdb.db.exception.StorageEngineException;
-import org.apache.iotdb.db.exception.query.QueryProcessException;
-import org.apache.iotdb.db.metadata.PartialPath;
-import org.apache.iotdb.db.query.aggregation.AggregateResult;
-import org.apache.iotdb.db.query.aggregation.impl.MinValueAggrResult;
-import org.apache.iotdb.db.query.context.QueryContext;
-import org.apache.iotdb.db.query.control.QueryResourceManager;
-import org.apache.iotdb.db.query.filter.TsFileFilter;
-import org.apache.iotdb.db.query.reader.series.SeriesReader;
-import org.apache.iotdb.db.query.simpiece.Point;
-import org.apache.iotdb.db.query.simpiece.TimeSeries;
-import org.apache.iotdb.db.query.simpiece.TimeSeriesReader;
-import org.apache.iotdb.db.query.simpiece.Visval;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
-import org.apache.iotdb.tsfile.read.common.ChunkSuit4Tri;
-import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
-import org.apache.iotdb.tsfile.read.filter.basic.Filter;
-import org.apache.iotdb.tsfile.utils.Pair;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Set;
-
-public class LocalGroupByExecutorTri_Visval implements GroupByExecutor {
-
- private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
-
- // Aggregate result buffer of this path
- private final List<AggregateResult> results = new ArrayList<>();
-
- TimeSeries timeSeries;
-
- int m;
-
- public LocalGroupByExecutorTri_Visval(
- PartialPath path,
- Set<String> allSensors,
- TSDataType dataType,
- QueryContext context,
- Filter timeFilter,
- TsFileFilter fileFilter,
- boolean ascending)
- throws StorageEngineException, QueryProcessException {
- // long start = System.nanoTime();
-
- // get all data sources
- QueryDataSource queryDataSource =
- QueryResourceManager.getInstance().getQueryDataSource(path, context, timeFilter);
-
- // update filter by TTL
- // this.timeFilter = queryDataSource.updateFilterUsingTTL(timeFilter);
-
- SeriesReader seriesReader =
- new SeriesReader(
- path,
- allSensors,
- // fix bug: here use the aggregation type as the series data type,
- // not using pageReader.getAllSatisfiedPageData is ok
- dataType,
- context,
- queryDataSource,
- timeFilter,
- null,
- fileFilter,
- ascending);
-
- try {
- // : this might be bad to load all chunk metadata at first
- List<ChunkSuit4Tri> futureChunkList = new ArrayList<>();
- futureChunkList.addAll(seriesReader.getAllChunkMetadatas4Tri());
- // order futureChunkList by chunk startTime
- futureChunkList.sort(
- new Comparator<ChunkSuit4Tri>() {
- public int compare(ChunkSuit4Tri o1, ChunkSuit4Tri o2) {
- return ((Comparable) (o1.chunkMetadata.getStartTime()))
- .compareTo(o2.chunkMetadata.getStartTime());
- }
- });
-
- GroupByFilter groupByFilter = (GroupByFilter) timeFilter;
- long startTime = groupByFilter.getStartTime();
- long endTime = groupByFilter.getEndTime();
- long interval = groupByFilter.getInterval();
- this.m = (int) ((endTime - startTime) / interval);
-
- timeSeries = TimeSeriesReader.getTimeSeriesFromTsFiles(futureChunkList, startTime, endTime);
-
- } catch (IOException e) {
- throw new QueryProcessException(e.getMessage());
- }
- }
-
- @Override
- public void addAggregateResult(AggregateResult aggrResult) {
- results.add(aggrResult);
- }
-
- @Override
- public List<AggregateResult> calcResult(
- long curStartTime, long curEndTime, long startTime, long endTime, long interval)
- throws IOException {
- // group by curStartTime and curEndTime are not used in Sim-Piece segmentation
-
- StringBuilder series = new StringBuilder();
-
- // clear result cache
- for (AggregateResult result : results) {
- result.reset();
- }
-
- // Visval
- List<Point> reducedPoints = Visval.reducePoints(timeSeries.data, m);
-
- for (Point p : reducedPoints) {
- series.append(p.getValue()).append("[").append(p.getTimestamp()).append("]").append(",");
- }
-
- MinValueAggrResult minValueAggrResult = (MinValueAggrResult) results.get(0);
- minValueAggrResult.updateResult(new MinMaxInfo<>(series.toString(), 0));
- return results;
- }
-
- @Override
- public Pair<Long, Object> peekNextNotNullValue(long nextStartTime, long nextEndTime)
- throws IOException {
- throw new IOException("no implemented");
- }
-
- @Override
- public List<AggregateResult> calcResult(long curStartTime, long curEndTime)
- throws IOException, QueryProcessException {
- throw new IOException("no implemented");
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java
deleted file mode 100644
index 7ca14859b37..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
-
-package org.apache.iotdb.db.query.simpiece;
-
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.io.PrintWriter;
-import java.util.List;
-
-public class MySample_visval {
-
- public static void main(String[] args) {
- String fileDir = "D:\\desktop\\NISTPV\\";
- boolean series = true; // 从1开始编号列而不是时间戳列
- String[] datasetNameList =
- new String[] {
- // "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- // "NISTPV-Ground-2015-RTD_C_3"
- };
-
- int[] noutList = new int[] {0};
-
- double[] r = new double[] {0.1, 0.5, 1.3};
- for (int y = 0; y < datasetNameList.length; y++) {
- String datasetName = datasetNameList[y];
- // int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
- // int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
- // int N = end - start; // -1 for all
-
- int start = 0;
- int end = 100;
- int N = end - start;
-
- for (int nout : noutList) {
- // apply Sim-Piece on the input file, outputting nout points saved in csvFile
- boolean hasHeader = false;
- try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
- String delimiter = ",";
- TimeSeries ts =
- TimeSeriesReader.getMyTimeSeries(
- inputStream, delimiter, false, N, start, hasHeader, series);
-
- List<Point> reducedPoints = Visval.reducePoints(ts.data, nout);
- try (PrintWriter writer =
- new PrintWriter(new FileWriter(datasetName + "-" + N + "-" + nout + "-visval.csv"))) {
- for (Point p : reducedPoints) {
- writer.println(p.getTimestamp() + "," + p.getValue());
- }
- }
- // List<Point> points = new ArrayList<>();
- // points.add(new Point(1, 10));
- // points.add(new Point(2, 20));
- // points.add(new Point(3, 15));
- // points.add(new Point(4, 10));
- // points.add(new Point(5, 30));
- // points.add(new Point(6, 25));
- // points.add(new Point(7, 20));
- // int m = 4;
- // List<Point> reducedPoints = Visval.reducePoints(points, m);
- // System.out.println("Reduced points:");
- // for (Point point : reducedPoints) {
- // System.out.println("Timestamp: " + point.getTimestamp() + ", Value: " +
- // point.getValue());
- // }
-
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
deleted file mode 100644
index 95002fd0d26..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.query.simpiece;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.IntStream;
-
-// Visvalingam-Whyatt method of poly-line vertex reduction
-//
-// Visvalingam, M and Whyatt J D (1993)
-// "Line Generalisation by Repeated Elimination of Points", Cartographic J., 30 (1), 46 - 51
-//
-// Described here:
-//
-// http://web.archive.org/web/20100428020453/http://www2.dcs.hull.ac.uk/CISRG/publications/DPs/DP10/DP10.html
-//
-// =========================================
-//
-// The MIT License (MIT)
-//
-// Copyright (c) 2014 Elliot Hallmark
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-//
-// ================================
-public class Visval {
-
- private static class Triangle implements Comparable<Triangle> {
-
- int index;
- double area;
-
- Triangle(int index, double area) {
- this.index = index;
- this.area = area;
- }
-
- @Override
- public int compareTo(Triangle other) {
- return Double.compare(this.area, other.area);
- }
- }
-
- public static List<Point> reducePoints(List<Point> points, int m) {
- if (points.size() <= m) {
- return points;
- }
-
- // areas and remainIdx: records the dominating areas and indexes of remaining points during
- // bottom-up elimination
- // through remainIdx we can know the adjacency of remaining points easily.
- List<Double> areas = triangleAreaList(points);
- List<Integer> remainIdx = new ArrayList<>();
- IntStream.range(0, points.size()).forEach(remainIdx::add);
-
- int minIndex = 0;
- for (int i = 1; i < areas.size(); i++) {
- if (areas.get(i) < areas.get(minIndex)) {
- minIndex = i;
- }
- }
- double this_area = areas.get(minIndex);
- areas.remove(minIndex);
- remainIdx.remove(minIndex);
-
- while (remainIdx.size() > m) {
- boolean skip =
- false; // false mean next round needs to argmin globally, otherwise use recentMinIdx
- int recentMinIdx = -1;
-
- // update right new triangle area
- double right_area = Double.POSITIVE_INFINITY;
- if (minIndex <= remainIdx.size() - 2) {
- // note that now i already pop out min_vert
- right_area =
- calculateTriangleArea(
- points.get(remainIdx.get(minIndex - 1)),
- points.get(remainIdx.get(minIndex)),
- points.get(remainIdx.get(minIndex + 1)));
- if (right_area <= this_area) {
- // so next round does not need argmin globally
- skip = true;
- recentMinIdx = minIndex; // note that now i already pop out min_vert
- }
- areas.set(minIndex, right_area);
- }
-
- // update left new triangle area
- if (minIndex >= 2) {
- // note that now i already pop out min_vert
- double left_area =
- calculateTriangleArea(
- points.get(remainIdx.get(minIndex - 2)),
- points.get(remainIdx.get(minIndex - 1)),
- points.get(remainIdx.get(minIndex)));
- if (left_area <= this_area) {
- if (skip) { // means right point area is smaller than this_area, then compare left and
- // right
- if (left_area <= right_area) {
- recentMinIdx = minIndex - 1;
- }
- // otherwise keep skip right point
- } else { // means right point area is larger than this_area, while left is smaller than
- skip = true; // so next round does not need argmin globally
- recentMinIdx = minIndex - 1;
- }
- }
- areas.set(minIndex - 1, left_area);
- }
-
- if (!skip) { // left and right new triangle both larger than this_area, needs to argmin
- // globally
- minIndex = 0;
- for (int i = 1; i < areas.size(); i++) {
- if (areas.get(i) < areas.get(minIndex)) {
- minIndex = i;
- }
- }
- } else {
- minIndex = recentMinIdx;
- }
- this_area = areas.get(minIndex);
- areas.remove(minIndex);
- remainIdx.remove(minIndex);
- }
-
- List<Point> result = new ArrayList<>();
- for (int i : remainIdx) {
- result.add(points.get(i));
- }
- return result;
- }
-
- private static List<Double> triangleAreaList(List<Point> points) {
- List<Double> result = new ArrayList<>();
- result.add(Double.POSITIVE_INFINITY); // first
- for (int i = 1; i < points.size() - 1; i++) {
- double area = calculateTriangleArea(points.get(i - 1), points.get(i), points.get(i + 1));
- result.add(area);
- }
- result.add(Double.POSITIVE_INFINITY); // last
- return result;
- }
-
- private static double calculateTriangleArea(Point a, Point b, Point c) {
- double x1 = a.getTimestamp();
- double y1 = a.getValue();
- double x2 = b.getTimestamp();
- double y2 = b.getValue();
- double x3 = c.getTimestamp();
- double y3 = c.getValue();
-
- return Math.abs((x1 * (y2 - y3) + x2 * (y3 - y1) + x3 * (y1 - y2)) / 2.0);
- }
-}
(iotdb) 07/12: Add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 33562646f9c53e92ec51358a21ad4fb15e046e84
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jul 15 00:54:32 2024 +0800
Add
---
.../iotdb/db/query/simpiece/MySample_fsw_full.java | 198 ++++++++++++++++++++
.../simpiece/MySample_shrinkingcone_full.java | 195 ++++++++++++++++++++
.../db/query/simpiece/MySample_simpiece_full.java | 199 +++++++++++++++++++++
3 files changed, 592 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full.java
new file mode 100644
index 00000000000..e4e581488dd
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.List;
+
+public class MySample_fsw_full {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ String[] datasetNameList =
+ new String[] {
+ "NISTPV-Ground-2015-WindSpeed_ms",
+ "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3"
+ };
+ int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
+
+ double[][] epsilonArray = {
+ {
+ 9.618271350860596,
+ 9.101282596588135,
+ 8.991526126861572,
+ 8.62817907333374,
+ 8.374999523162842,
+ 8.202636241912842,
+ 7.999906063079834,
+ 7.947059154510498,
+ 7.858933925628662
+ },
+ {
+ 9.999275207519531E-4,
+ 9.989738464355469E-4,
+ 9.980201721191406E-4,
+ 9.970664978027344E-4,
+ 9.961128234863281E-4,
+ 9.951591491699219E-4,
+ 9.942054748535156E-4,
+ 9.932518005371094E-4,
+ 9.922981262207031E-4
+ },
+ {
+ 440.0235962867737,
+ 432.0247492790222,
+ 423.55674982070923,
+ 414.65001153945923,
+ 405.77113008499146,
+ 401.21820974349976,
+ 396.7935194969177,
+ 393.2065939903259,
+ 392.30953645706177
+ },
+ {
+ 9.295397281646729,
+ 8.096678256988525,
+ 7.536666393280029,
+ 6.961357593536377,
+ 6.473535060882568,
+ 6.064479351043701,
+ 5.743286609649658,
+ 5.5219340324401855,
+ 5.185042858123779
+ }
+ };
+
+ // double[][] epsilonArray = new double[datasetNameList.length][];
+ // for (int i = 0; i < datasetNameList.length; i++) {
+ // epsilonArray[i] = new double[noutList.length];
+ // }
+
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ int start = 0;
+ int end = 1000_0000;
+ int N = end - start;
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ TimeSeries ts =
+ TimeSeriesReader.getMyTimeSeries(
+ inputStream, delimiter, false, N, start, hasHeader, false);
+ for (int x = 0; x < noutList.length; x++) {
+ int nout = noutList[x];
+ // double epsilon = getFSWParam(nout, ts, 1e-6);
+ // epsilonArray[y][x] = epsilon;
+
+ double epsilon = epsilonArray[y][x];
+
+ List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
+ System.out.println(
+ datasetName
+ + ": n="
+ + N
+ + ",m="
+ + nout
+ + ",epsilon="
+ + epsilon
+ + ",actual m="
+ + reducedPoints.size());
+ try (PrintWriter writer =
+ new PrintWriter(
+ new FileWriter(
+ datasetName
+ + "-"
+ + N
+ + "-"
+ + nout
+ + "-"
+ + reducedPoints.size()
+ + "-fsw.csv"))) {
+ for (Point p : reducedPoints) {
+ writer.println(p.getTimestamp() + "," + p.getValue());
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (int i = 0; i < epsilonArray.length; i++) { // 遍历行
+ for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列
+ System.out.print(epsilonArray[i][j] + ",");
+ }
+ System.out.println();
+ }
+ }
+
+ public static double getFSWParam(int nout, TimeSeries ts, double accuracy) throws IOException {
+ double epsilon = 1;
+ boolean directLess = false;
+ boolean directMore = false;
+ while (true) {
+ List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
+ if (reducedPoints.size() > nout) {
+ if (directMore) {
+ break;
+ }
+ if (!directLess) {
+ directLess = true;
+ }
+ epsilon *= 2;
+ } else {
+ if (directLess) {
+ break;
+ }
+ if (!directMore) {
+ directMore = true;
+ }
+ epsilon /= 2;
+ }
+ }
+ double left = 0;
+ double right = 0;
+ if (directLess) {
+ left = epsilon / 2;
+ right = epsilon;
+ }
+ if (directMore) {
+ left = epsilon;
+ right = epsilon * 2;
+ }
+ while (Math.abs(right - left) > accuracy) {
+ double mid = (left + right) / 2;
+ List<Point> reducedPoints = FSW.reducePoints(ts.data, mid);
+ if (reducedPoints.size() > nout) {
+ left = mid;
+ } else {
+ right = mid;
+ }
+ }
+ return (left + right) / 2;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java
new file mode 100644
index 00000000000..d0c7aeef63d
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.List;
+
+public class MySample_shrinkingcone_full {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ String[] datasetNameList =
+ new String[] {
+ "NISTPV-Ground-2015-WindSpeed_ms",
+ "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3"
+ };
+ int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
+
+ // double[] epsilonList = new double[]{0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001,
+ // 0.001,
+ // 0.0009999999999999999};
+
+ double[][] epsilonArray = {
+ {
+ 14.388325214385986,
+ 13.768231868743896,
+ 13.343345165252686,
+ 13.091249942779541,
+ 12.69767427444458,
+ 12.391706943511963,
+ 12.167190074920654,
+ 11.972727298736572,
+ 11.756273746490479
+ },
+ {0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.0009999999999999999},
+ {
+ 680.2196469306946,
+ 665.2842917442322,
+ 650.2610821723938,
+ 636.7945942878723,
+ 620.9016032218933,
+ 604.7394433021545,
+ 592.9867577552795,
+ 578.6471419334412,
+ 570.5106825828552
+ },
+ {
+ 11.259880542755127,
+ 10.37297010421753,
+ 9.402754306793213,
+ 8.867334842681885,
+ 8.384315967559814,
+ 7.915340900421143,
+ 7.39516019821167,
+ 6.892223834991455,
+ 6.5178914070129395
+ }
+ };
+
+ // double[][] epsilonArray = new double[datasetNameList.length][];
+ // for (int i = 0; i < datasetNameList.length; i++) {
+ // epsilonArray[i] = new double[noutList.length];
+ // }
+
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ int start = 0;
+ int end = 1000_0000;
+ int N = end - start;
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ TimeSeries ts =
+ TimeSeriesReader.getMyTimeSeries(
+ inputStream, delimiter, false, N, start, hasHeader, false);
+ for (int x = 0; x < noutList.length; x++) {
+ int nout = noutList[x];
+
+ // double epsilon = getSCParam(nout, ts, 1e-6);
+ // epsilonArray[y][x] = epsilon;
+
+ double epsilon = epsilonArray[y][x];
+
+ // double epsilon = epsilonList[x]; // 单独给Qloss手调
+
+ List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
+ System.out.println(
+ datasetName
+ + ": n="
+ + N
+ + ",m="
+ + nout
+ + ",epsilon="
+ + epsilon
+ + ",actual m="
+ + reducedPoints.size());
+ try (PrintWriter writer =
+ new PrintWriter(
+ new FileWriter(
+ datasetName
+ + "-"
+ + N
+ + "-"
+ + nout
+ + "-"
+ + reducedPoints.size()
+ + "-sc.csv"))) {
+ for (Point p : reducedPoints) {
+ writer.println(p.getTimestamp() + "," + p.getValue());
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (int i = 0; i < epsilonArray.length; i++) { // 遍历行
+ for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列
+ System.out.print(epsilonArray[i][j] + ",");
+ }
+ System.out.println();
+ }
+ }
+
+ public static double getSCParam(int nout, TimeSeries ts, double accuracy) throws IOException {
+ double epsilon = 1;
+ boolean directLess = false;
+ boolean directMore = false;
+ while (true) {
+ List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
+ if (reducedPoints.size() > nout) {
+ if (directMore) {
+ break;
+ }
+ if (!directLess) {
+ directLess = true;
+ }
+ epsilon *= 2;
+ } else {
+ if (directLess) {
+ break;
+ }
+ if (!directMore) {
+ directMore = true;
+ }
+ epsilon /= 2;
+ }
+ }
+ double left = 0;
+ double right = 0;
+ if (directLess) {
+ left = epsilon / 2;
+ right = epsilon;
+ }
+ if (directMore) {
+ left = epsilon;
+ right = epsilon * 2;
+ }
+ while (Math.abs(right - left) > accuracy) {
+ double mid = (left + right) / 2;
+ List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, mid);
+ if (reducedPoints.size() > nout) {
+ left = mid;
+ } else {
+ right = mid;
+ }
+ }
+ return (left + right) / 2;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full.java
new file mode 100644
index 00000000000..e7a2cd19c43
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Comparator;
+import java.util.List;
+
+public class MySample_simpiece_full {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ String[] datasetNameList =
+ new String[] {
+ "NISTPV-Ground-2015-WindSpeed_ms",
+ "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3"
+ };
+ int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
+
+ double[][] epsilonArray = {
+ {
+ 10.470000203704831,
+ 9.973635465240477,
+ 9.571347576141356,
+ 9.339938166046142,
+ 9.208912633514405,
+ 9.135163892364506,
+ 8.744444478607178,
+ 8.61042620162964,
+ 8.422101323699948
+ },
+ {
+ 9.987499999999999E-4,
+ 9.974999999999997E-4,
+ 9.784999999999998E-4,
+ 9.4425E-4,
+ 9.255E-4,
+ 8.8375E-4,
+ 8.4775E-4,
+ 7.865E-4,
+ 7.3825E-4
+ },
+ {
+ 502.0505219552564,
+ 493.3274919509581,
+ 485.91779255051347,
+ 479.26180967157245,
+ 463.5216509033579,
+ 446.88677012006065,
+ 441.2913785749938,
+ 435.45897277584766,
+ 426.0481592410648
+ },
+ {
+ 13.678959740142822,
+ 12.275238493499756,
+ 11.224719192962645,
+ 10.475999729766848,
+ 9.741776887359617,
+ 9.125823136444094,
+ 8.625197060394289,
+ 8.105121862335203,
+ 7.743280215911865
+ }
+ };
+
+ // double[][] epsilonArray = new double[datasetNameList.length][];
+ // for (int i = 0; i < datasetNameList.length; i++) {
+ // epsilonArray[i] = new double[noutList.length];
+ // }
+
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ int start = 0;
+ int end = 1000_0000;
+ int N = end - start;
+ // apply Sim-Piece on the input file, outputting nout points saved in csvFile
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ TimeSeries ts =
+ TimeSeriesReader.getMyTimeSeries(
+ inputStream, delimiter, false, N, start, hasHeader, false);
+ for (int x = 0; x < noutList.length; x++) {
+ int nout = noutList[x];
+
+ // double epsilon = getSimPieceParam(nout, ts, 1e-6);
+ // epsilonArray[y][x] = epsilon;
+
+ double epsilon = epsilonArray[y][x];
+
+ SimPiece simPiece = new SimPiece(ts.data, epsilon);
+ System.out.println(
+ datasetName
+ + ": n="
+ + N
+ + ",m="
+ + nout
+ + ",epsilon="
+ + epsilon
+ + ",actual m="
+ + simPiece.segments.size() * 2);
+ List<SimPieceSegment> segments = simPiece.segments;
+ segments.sort(Comparator.comparingLong(SimPieceSegment::getInitTimestamp));
+ try (PrintWriter writer =
+ new PrintWriter(
+ new FileWriter(
+ datasetName
+ + "-"
+ + N
+ + "-"
+ + nout
+ + "-"
+ + segments.size() * 2
+ + "-simpiece.csv"))) {
+ for (int i = 0; i < segments.size() - 1; i++) {
+ // start point of this segment
+ writer.println(segments.get(i).getInitTimestamp() + "," + segments.get(i).getB());
+ // end point of this segment
+ double v =
+ (segments.get(i + 1).getInitTimestamp() - segments.get(i).getInitTimestamp())
+ * segments.get(i).getA()
+ + segments.get(i).getB();
+ writer.println(segments.get(i + 1).getInitTimestamp() + "," + v);
+ }
+ // the two end points of the last segment
+ writer.println(
+ segments.get(segments.size() - 1).getInitTimestamp()
+ + ","
+ + segments.get(segments.size() - 1).getB());
+ double v =
+ (simPiece.lastTimeStamp - segments.get(segments.size() - 1).getInitTimestamp())
+ * segments.get(segments.size() - 1).getA()
+ + segments.get(segments.size() - 1).getB();
+ writer.println(simPiece.lastTimeStamp + "," + v);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ for (int i = 0; i < epsilonArray.length; i++) { // 遍历行
+ for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列
+ System.out.print(epsilonArray[i][j] + ",");
+ }
+ System.out.println();
+ }
+ }
+
+ public static double getSimPieceParam(int nout, TimeSeries ts, double accuracy)
+ throws IOException {
+ double epsilon = ts.range * 0.001;
+ while (true) {
+ SimPiece simPiece = new SimPiece(ts.data, epsilon);
+ if (simPiece.segments.size() * 2 > nout) { // note *2 for disjoint
+ epsilon *= 2;
+ } else {
+ break;
+ }
+ }
+ double left = epsilon / 2;
+ double right = epsilon;
+ while (Math.abs(right - left) > accuracy) {
+ double mid = (left + right) / 2;
+ SimPiece simPiece = new SimPiece(ts.data, mid);
+ if (simPiece.segments.size() * 2 > nout) { // note *2 for disjoint
+ left = mid;
+ } else {
+ right = mid;
+ }
+ }
+ return (left + right) / 2;
+ }
+}
(iotdb) 10/12: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 44b0c441c7dee1437fd62b5abf8a63ff1dcaf2c3
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jul 15 18:27:12 2024 +0800
add
---
.../resources/conf/iotdb-engine.properties | 2 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 3 +
.../groupby/LocalGroupByExecutorTri_Visval.java | 160 ++++++++++++++++
.../iotdb/db/query/simpiece/MySample_visval.java | 94 ++++++++++
.../iotdb/db/query/simpiece/TimeSeriesReader.java | 94 ++++++++++
.../org/apache/iotdb/db/query/simpiece/Visval.java | 111 +++++++++++
.../iotdb/db/query/simpiece/VisvalPoint.java | 13 ++
.../apache/iotdb/db/query/simpiece/Visvalold.java | 203 +++++++++++++++++++++
8 files changed, 679 insertions(+), 1 deletion(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index b7a8c2130b6..696856331b7 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -19,7 +19,7 @@
####################
### enable Tri
####################
-# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece, SC, FSW, Uniform
+# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece, SC, FSW, Uniform, Visval
enable_Tri=""
# segment error threshold for SimPiece, SC, FSW
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 97da407a4a3..32eccb2076b 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -459,6 +459,9 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
} else if (CONFIG.getEnableTri().equals("Uniform")) {
return new LocalGroupByExecutorTri_Uniform(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
+ } else if (CONFIG.getEnableTri().equals("Visval")) {
+ return new LocalGroupByExecutorTri_Visval(
+ path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
} else {
logger.info("No matched enable_tri!");
return new LocalGroupByExecutor(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java
new file mode 100644
index 00000000000..3cb801b70e7
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.dataset.groupby;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.query.aggregation.AggregateResult;
+import org.apache.iotdb.db.query.aggregation.impl.MinValueAggrResult;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
+import org.apache.iotdb.db.query.reader.series.SeriesReader;
+import org.apache.iotdb.db.query.simpiece.TimeSeriesReader;
+import org.apache.iotdb.db.query.simpiece.Visval;
+import org.apache.iotdb.db.query.simpiece.VisvalPoint;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
+import org.apache.iotdb.tsfile.read.common.ChunkSuit4Tri;
+import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.utils.Pair;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+public class LocalGroupByExecutorTri_Visval implements GroupByExecutor {
+
+ private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
+
+ // Aggregate result buffer of this path
+ private final List<AggregateResult> results = new ArrayList<>();
+
+ List<VisvalPoint> points;
+
+ private final int m;
+
+ public LocalGroupByExecutorTri_Visval(
+ PartialPath path,
+ Set<String> allSensors,
+ TSDataType dataType,
+ QueryContext context,
+ Filter timeFilter,
+ TsFileFilter fileFilter,
+ boolean ascending)
+ throws StorageEngineException, QueryProcessException {
+ // long start = System.nanoTime();
+
+ // get all data sources
+ QueryDataSource queryDataSource =
+ QueryResourceManager.getInstance().getQueryDataSource(path, context, timeFilter);
+
+ // update filter by TTL
+ // this.timeFilter = queryDataSource.updateFilterUsingTTL(timeFilter);
+
+ SeriesReader seriesReader =
+ new SeriesReader(
+ path,
+ allSensors,
+ // fix bug: here use the aggregation type as the series data type,
+ // not using pageReader.getAllSatisfiedPageData is ok
+ dataType,
+ context,
+ queryDataSource,
+ timeFilter,
+ null,
+ fileFilter,
+ ascending);
+
+ try {
+ // : this might be bad to load all chunk metadata at first
+ List<ChunkSuit4Tri> futureChunkList = new ArrayList<>();
+ futureChunkList.addAll(seriesReader.getAllChunkMetadatas4Tri());
+ // order futureChunkList by chunk startTime
+ futureChunkList.sort(
+ new Comparator<ChunkSuit4Tri>() {
+ public int compare(ChunkSuit4Tri o1, ChunkSuit4Tri o2) {
+ return ((Comparable) (o1.chunkMetadata.getStartTime()))
+ .compareTo(o2.chunkMetadata.getStartTime());
+ }
+ });
+
+ GroupByFilter groupByFilter = (GroupByFilter) timeFilter;
+ long startTime = groupByFilter.getStartTime();
+ long endTime = groupByFilter.getEndTime();
+ long interval = groupByFilter.getInterval();
+ m = (int) Math.floor((endTime * 1.0 - startTime) / interval);
+
+ points = TimeSeriesReader.getTimeSeriesFromTsFilesVisval(futureChunkList, startTime, endTime);
+
+ } catch (IOException e) {
+ throw new QueryProcessException(e.getMessage());
+ }
+ }
+
+ @Override
+ public void addAggregateResult(AggregateResult aggrResult) {
+ results.add(aggrResult);
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(
+ long curStartTime, long curEndTime, long startTime, long endTime, long interval)
+ throws IOException {
+ // group by curStartTime and curEndTime are not used in Sim-Piece segmentation
+
+ StringBuilder series = new StringBuilder();
+
+ // clear result cache
+ for (AggregateResult result : results) {
+ result.reset();
+ }
+
+ // Visval
+ List<VisvalPoint> reducedPoints = Visval.reducePoints(points, m);
+
+ for (VisvalPoint p : reducedPoints) {
+ series.append(p.y).append("[").append(p.x).append("]").append(",");
+ }
+
+ MinValueAggrResult minValueAggrResult = (MinValueAggrResult) results.get(0);
+ minValueAggrResult.updateResult(new MinMaxInfo<>(series.toString(), 0));
+ return results;
+ }
+
+ @Override
+ public Pair<Long, Object> peekNextNotNullValue(long nextStartTime, long nextEndTime)
+ throws IOException {
+ throw new IOException("no implemented");
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(long curStartTime, long curEndTime)
+ throws IOException, QueryProcessException {
+ throw new IOException("no implemented");
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java
new file mode 100644
index 00000000000..1ed02baba7c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java
@@ -0,0 +1,94 @@
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.util.List;
+
+public class MySample_visval {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ String[] datasetNameList =
+ new String[] {
+ "NISTPV-Ground-2015-Qloss_Ah",
+ // "NISTPV-Ground-2015-Pyra1_Wm2",
+ // "NISTPV-Ground-2015-RTD_C_3"
+ };
+
+ int[] noutList = new int[] {2400000};
+
+ double[] r = new double[] {0.1, 0.5, 1.3};
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
+ int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
+ int N = end - start; // -1 for all
+ //
+ // int start = 0;
+ // int end = 100000;
+ // int N = end - start;
+
+ for (int nout : noutList) {
+ // apply Sim-Piece on the input file, outputting nout points saved in csvFile
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ List<VisvalPoint> points =
+ TimeSeriesReader.getMyTimeSeriesVisval(
+ inputStream, delimiter, false, N, start, hasHeader, false);
+ System.out.println(points.size());
+ System.out.println(nout);
+
+ List<VisvalPoint> reducedPoints = Visval.reducePoints(points, nout);
+ try (PrintWriter writer =
+ new PrintWriter(
+ new FileWriter(
+ datasetName + "-" + N + "-" + reducedPoints.size() + "-visval.csv"))) {
+ for (VisvalPoint p : reducedPoints) {
+ writer.println(p.x + "," + p.y);
+ }
+ }
+ System.out.println(reducedPoints.size());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ // List<Point> points = new ArrayList<>();
+ // points.add(new Point(1, 10));
+ // points.add(new Point(2, 20));
+ // points.add(new Point(3, 15));
+ // points.add(new Point(4, 10));
+ // points.add(new Point(5, 30));
+ // points.add(new Point(6, 25));
+ // points.add(new Point(7, 20));
+ // int m = 4;
+ // List<Point> reducedPoints = Visval.reducePoints(points, m);
+ // System.out.println("Reduced points:");
+ // for (Point point : reducedPoints) {
+ // System.out.println("Timestamp: " + point.getTimestamp() + ", Value: " +
+ // point.getValue());
+ // }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeriesReader.java
index 129baeefeda..65b828cb1b1 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/TimeSeriesReader.java
@@ -80,6 +80,46 @@ public class TimeSeriesReader {
return new TimeSeries(ts, max - min);
}
+ public static List<VisvalPoint> getTimeSeriesFromTsFilesVisval(
+ List<ChunkSuit4Tri> chunkSuit4TriList, long startTime, long endTime) throws IOException {
+ // assume chunkSuit4TriList already sorted in increasing time order
+ ArrayList<VisvalPoint> ts = new ArrayList<>();
+ double max = Double.MIN_VALUE;
+ double min = Double.MAX_VALUE;
+
+ for (ChunkSuit4Tri chunkSuit4Tri : chunkSuit4TriList) {
+ ChunkMetadata chunkMetadata = chunkSuit4Tri.chunkMetadata;
+ long chunkMinTime = chunkMetadata.getStartTime();
+ long chunkMaxTime = chunkMetadata.getEndTime();
+ if (chunkMaxTime < startTime) {
+ continue;
+ } else if (chunkMinTime >= endTime) {
+ break;
+ } else {
+ PageReader pageReader =
+ FileLoaderUtils.loadPageReaderList4CPV(
+ chunkSuit4Tri.chunkMetadata,
+ null); // note do not assign to chunkSuit4Tri.pageReader
+ for (int j = 0; j < chunkSuit4Tri.chunkMetadata.getStatistics().getCount(); j++) {
+ IOMonitor2.DCP_D_getAllSatisfiedPageData_traversedPointNum++;
+ long timestamp = pageReader.timeBuffer.getLong(j * 8);
+ if (timestamp < startTime) {
+ continue;
+ } else if (timestamp >= endTime) {
+ break;
+ } else { // rightStartTime<=t<rightEndTime
+ ByteBuffer valueBuffer = pageReader.valueBuffer;
+ double value = valueBuffer.getDouble(pageReader.timeBufferLength + j * 8);
+ ts.add(new VisvalPoint(timestamp, value));
+ max = Math.max(max, value);
+ min = Math.min(min, value);
+ }
+ }
+ }
+ }
+ return ts;
+ }
+
public static TimeSeries getTimeSeries(InputStream inputStream, String delimiter, boolean gzip) {
ArrayList<Point> ts = new ArrayList<>();
double max = Double.MIN_VALUE;
@@ -163,4 +203,58 @@ public class TimeSeriesReader {
return new TimeSeries(ts, max - min);
}
+
+ public static List<VisvalPoint> getMyTimeSeriesVisval(
+ InputStream inputStream,
+ String delimiter,
+ boolean gzip,
+ int N,
+ int startRow,
+ boolean hasHeader,
+ boolean seriesTimeColumn) {
+ // N<0 means read all lines
+
+ ArrayList<VisvalPoint> ts = new ArrayList<>();
+ double max = Double.MIN_VALUE;
+ double min = Double.MAX_VALUE;
+
+ try {
+ if (gzip) {
+ inputStream = new GZIPInputStream(inputStream);
+ }
+ Reader decoder = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
+ BufferedReader bufferedReader = new BufferedReader(decoder);
+
+ String line;
+ if (hasHeader) {
+ bufferedReader.readLine();
+ }
+ int startCnt = 0;
+ while (startCnt < startRow && (line = bufferedReader.readLine()) != null) {
+ startCnt++;
+ }
+ if (startCnt < startRow) {
+ throw new IOException("not enough rows!");
+ }
+ int cnt = 0;
+ while ((cnt < N || N < 0) && (line = bufferedReader.readLine()) != null) {
+ String[] elements = line.split(delimiter);
+ long timestamp;
+ if (!seriesTimeColumn) {
+ timestamp = (long) Double.parseDouble(elements[0]);
+ } else {
+ timestamp = cnt + 1;
+ }
+ double value = Double.parseDouble(elements[1]);
+ ts.add(new VisvalPoint(timestamp, value));
+
+ max = Math.max(max, value);
+ min = Math.min(min, value);
+ cnt++;
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return ts;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
new file mode 100644
index 00000000000..d4b54d3adc1
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.util.List;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+public class Visval {
+
+ public static List<VisvalPoint> reducePoints(List<VisvalPoint> points, int targetCount) {
+ if (points.size() <= targetCount) {
+ return points;
+ }
+
+ ConcurrentSkipListMap<Double, VisvalPoint> minHeap = new ConcurrentSkipListMap<>();
+ // 初始化双向链表和优先队列
+ for (int i = 1; i < points.size() - 1; i++) {
+ VisvalPoint p = points.get(i);
+ p.prev = points.get(i - 1);
+ p.next = points.get(i + 1);
+ p.area = calculateArea(p.prev, p, p.next);
+ minHeap.put(p.area, p);
+ }
+
+ while (points.size() > targetCount) {
+ // 移除面积最小的点
+ VisvalPoint p = minHeap.pollFirstEntry().getValue();
+ if (p == null) {
+ return points;
+ }
+ // 移除点 p
+ VisvalPoint prev = p.prev;
+ VisvalPoint next = p.next;
+ prev.next = next;
+ next.prev = prev;
+
+ // 更新相邻点的三角形面积
+ if (prev.prev != null) {
+ minHeap.remove(prev.area);
+ prev.area = calculateArea(prev.prev, prev, next);
+ minHeap.put(prev.area, prev);
+ }
+ if (next.next != null) {
+ minHeap.remove(next.area);
+ next.area = calculateArea(prev, next, next.next);
+ minHeap.put(next.area, next);
+ }
+ points.remove(p);
+ }
+
+ return points;
+ }
+
+ private static double calculateArea(VisvalPoint a, VisvalPoint b, VisvalPoint c) {
+ // 计算三角形面积的函数
+ return Math.abs(a.x * (b.y - c.y) + b.x * (c.y - a.y) + c.x * (a.y - b.y)) / 2.0;
+ }
+
+ // public static void main(String[] args) {
+ // List<VisvalPoint> points = new ArrayList<>();
+ // List<Point> pointsold = new ArrayList<>();
+ // // 添加点到列表 points 100万数据
+ // Random rand = new Random();
+ //
+ // int targetCount = 10;
+ // for (int i = 0; i < 10000; i++) {
+ // int v = rand.nextInt(1000);
+ // points.add(new VisvalPoint(i, v));
+ // pointsold.add(new Point(i, v));
+ // }
+ //
+ //// int targetCount = 9000000;
+ //// for (int i = 0; i < 10000000; i++) {
+ //// points.add(new VisvalPoint(i, rand.nextInt(1000)));
+ //// }
+ //
+ // // 计算运行时间
+ // long startTime = System.currentTimeMillis();
+ //
+ // List<VisvalPoint> reducedPoints = Visval.reducePoints(points, targetCount);
+ // for (VisvalPoint point : reducedPoints) {
+ // System.out.println("Timestamp: " + point.x + ", Value: " + point.y);
+ // }
+ // List<Point> reducedPointsold = Visvalold.reducePoints(pointsold, targetCount);
+ // for (Point point : reducedPointsold) {
+ // System.out.println("Timestamp: " + point.getTimestamp() + ", Value: " +
+ // point.getValue());
+ // }
+ // // 输出结果
+ // System.out.println(reducedPoints.size());
+ // long endTime = System.currentTimeMillis();
+ // System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+ // }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalPoint.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalPoint.java
new file mode 100644
index 00000000000..ef29bc30c09
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalPoint.java
@@ -0,0 +1,13 @@
+package org.apache.iotdb.db.query.simpiece;
+
+public class VisvalPoint {
+
+ public double x, y;
+ public VisvalPoint prev, next; // 双向链表中的前驱和后继
+ public double area; // 与相邻点形成的三角形面积
+
+ VisvalPoint(double x, double y) {
+ this.x = x;
+ this.y = y;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visvalold.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visvalold.java
new file mode 100644
index 00000000000..2592d042c3c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visvalold.java
@@ -0,0 +1,203 @@
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.IntStream;
+
+// Visvalingam-Whyatt method of poly-line vertex reduction
+//
+// Visvalingam, M and Whyatt J D (1993)
+// "Line Generalisation by Repeated Elimination of Points", Cartographic J., 30 (1), 46 - 51
+//
+// Described here:
+//
+// http://web.archive.org/web/20100428020453/http://www2.dcs.hull.ac.uk/CISRG/publications/DPs/DP10/DP10.html
+//
+// =========================================
+//
+// The MIT License (MIT)
+//
+// Copyright (c) 2014 Elliot Hallmark
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// ================================
+public class Visvalold {
+
+ private static class Triangle implements Comparable<Triangle> {
+
+ int index;
+ double area;
+
+ Triangle(int index, double area) {
+ this.index = index;
+ this.area = area;
+ }
+
+ @Override
+ public int compareTo(Triangle other) {
+ return Double.compare(this.area, other.area);
+ }
+ }
+
+ public static List<Point> reducePoints(List<Point> points, int m) {
+ if (points.size() <= m) {
+ return points;
+ }
+
+ // areas and remainIdx: records the dominating areas and indexes of remaining points during
+ // bottom-up elimination
+ // through remainIdx we can know the adjacency of remaining points easily.
+ // TreeMap<Double, Integer> areasMap = new TreeMap<>();// for speedup
+ List<Double> areas = triangleAreaList(points);
+ List<Integer> remainIdx = new ArrayList<>();
+ IntStream.range(0, points.size()).forEach(remainIdx::add);
+
+ int minIndex = 0;
+ for (int i = 1; i < areas.size(); i++) {
+ if (areas.get(i) < areas.get(minIndex)) {
+ minIndex = i;
+ }
+ }
+ // AtomicInteger minIndexTmp = new AtomicInteger(0);
+ // IntStream.range(0, areas.size()).parallel().forEach(i -> {
+ // if (areas.get(i) < areas.get(minIndexTmp.get())) {
+ // minIndexTmp.set(i);
+ // }
+ // });
+ // int minIndex = minIndexTmp.get();
+ double this_area = areas.get(minIndex);
+ areas.remove(minIndex);
+ remainIdx.remove(minIndex);
+
+ while (remainIdx.size() > m) {
+ boolean skip =
+ false; // false mean next round needs to argmin globally, otherwise use recentMinIdx
+ int recentMinIdx = -1;
+
+ // update right new triangle area
+ double right_area = Double.POSITIVE_INFINITY;
+ if (minIndex <= remainIdx.size() - 2) {
+ // note that now i already pop out min_vert
+ right_area =
+ calculateTriangleArea(
+ points.get(remainIdx.get(minIndex - 1)),
+ points.get(remainIdx.get(minIndex)),
+ points.get(remainIdx.get(minIndex + 1)));
+ if (right_area <= this_area) {
+ // so next round does not need argmin globally
+ skip = true;
+ recentMinIdx = minIndex; // note that now i already pop out min_vert
+ }
+ areas.set(minIndex, right_area);
+ }
+
+ // update left new triangle area
+ if (minIndex >= 2) {
+ // note that now i already pop out min_vert
+ double left_area =
+ calculateTriangleArea(
+ points.get(remainIdx.get(minIndex - 2)),
+ points.get(remainIdx.get(minIndex - 1)),
+ points.get(remainIdx.get(minIndex)));
+ if (left_area <= this_area) {
+ if (skip) { // means right point area is smaller than this_area, then compare left and
+ // right
+ if (left_area <= right_area) {
+ recentMinIdx = minIndex - 1;
+ }
+ // otherwise keep skip right point
+ } else { // means right point area is larger than this_area, while left is smaller than
+ skip = true; // so next round does not need argmin globally
+ recentMinIdx = minIndex - 1;
+ }
+ }
+ areas.set(minIndex - 1, left_area);
+ }
+
+ if (!skip) { // left and right new triangle both larger than this_area, needs to argmin
+ // globally
+ minIndex = 0;
+ for (int i = 1; i < areas.size(); i++) {
+ if (areas.get(i) < areas.get(minIndex)) {
+ minIndex = i;
+ }
+ }
+ // AtomicInteger minIndexTmp2 = new AtomicInteger(0);
+ // IntStream.range(0, areas.size()).parallel().forEach(i -> {
+ // if (areas.get(i) < areas.get(minIndexTmp2.get())) {
+ // minIndexTmp2.set(i);
+ // }
+ // });
+ // minIndex = minIndexTmp2.get();
+ } else {
+ minIndex = recentMinIdx;
+ }
+ this_area = areas.get(minIndex);
+ areas.remove(minIndex);
+ remainIdx.remove(minIndex);
+ }
+
+ List<Point> result = new ArrayList<>();
+ for (int i : remainIdx) {
+ result.add(points.get(i));
+ }
+ return result;
+ }
+
+ private static List<Double> triangleAreaList(List<Point> points) {
+ List<Double> result = new ArrayList<>();
+ result.add(Double.POSITIVE_INFINITY); // first
+ for (int i = 1; i < points.size() - 1; i++) {
+ double area = calculateTriangleArea(points.get(i - 1), points.get(i), points.get(i + 1));
+ result.add(area);
+ }
+ result.add(Double.POSITIVE_INFINITY); // last
+ return result;
+ }
+
+ private static double calculateTriangleArea(Point a, Point b, Point c) {
+ double x1 = a.getTimestamp();
+ double y1 = a.getValue();
+ double x2 = b.getTimestamp();
+ double y2 = b.getValue();
+ double x3 = c.getTimestamp();
+ double y3 = c.getValue();
+
+ return Math.abs((x1 * (y2 - y3) + x2 * (y3 - y1) + x3 * (y1 - y2)) / 2.0);
+ }
+}
(iotdb) 06/12: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3f4796ae153362004a4e216d545ee145d085bb6c
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jul 14 18:06:19 2024 +0800
add
---
.../iotdb/db/query/simpiece/MySample_fsw.java | 88 +++++++++++++++++-----
.../db/query/simpiece/MySample_shrinkingcone.java | 85 +++++++++++++++++----
.../iotdb/db/query/simpiece/MySample_simpiece.java | 31 ++++++--
3 files changed, 163 insertions(+), 41 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
index 854962e96a6..ed4eefe0b50 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
@@ -23,6 +23,7 @@ package org.apache.iotdb.db.query.simpiece;
import java.io.FileInputStream;
import java.io.FileWriter;
+import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;
@@ -30,41 +31,48 @@ public class MySample_fsw {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
- boolean series = true; // 从1开始编号列而不是时间戳列
String[] datasetNameList =
new String[] {
- // "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Qloss_Ah",
"NISTPV-Ground-2015-Pyra1_Wm2",
- // "NISTPV-Ground-2015-RTD_C_3"
+ "NISTPV-Ground-2015-RTD_C_3"
};
-
- int[] noutList = new int[] {0};
-
+ int[] noutList = new int[] {100};
double[] r = new double[] {0.1, 0.5, 1.3};
+ double[] epsilonList = new double[] {9.999999E-4, 284.4034399986267, 6.428162097930908};
for (int y = 0; y < datasetNameList.length; y++) {
String datasetName = datasetNameList[y];
- // int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
- // int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
- // int N = end - start; // -1 for all
-
- int start = 0;
- int end = 10000;
+ int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
+ int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
int N = end - start;
+ // int start = 0;
+ // int end = 10000;
+ // int N = end - start;
for (int nout : noutList) {
- // apply Sim-Piece on the input file, outputting nout points saved in csvFile
boolean hasHeader = false;
try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
String delimiter = ",";
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
- inputStream, delimiter, false, N, start, hasHeader, series);
-
- double epsilon = 0.1;
+ inputStream, delimiter, false, N, start, hasHeader, false);
+ // double epsilon = getFSWParam(nout, ts, 1e-6);
+ double epsilon = epsilonList[y];
List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
+ System.out.println(
+ datasetName
+ + ": n="
+ + N
+ + ",m="
+ + nout
+ + ",epsilon="
+ + epsilon
+ + ",actual m="
+ + reducedPoints.size());
try (PrintWriter writer =
new PrintWriter(
- new FileWriter(datasetName + "-" + N + "-" + reducedPoints.size() + "-sc.csv"))) {
+ new FileWriter(
+ datasetName + "-" + N + "-" + reducedPoints.size() + "-fsw.csv"))) {
for (Point p : reducedPoints) {
writer.println(p.getTimestamp() + "," + p.getValue());
}
@@ -75,4 +83,50 @@ public class MySample_fsw {
}
}
}
+
+ public static double getFSWParam(int nout, TimeSeries ts, double accuracy) throws IOException {
+ double epsilon = 1;
+ boolean directLess = false;
+ boolean directMore = false;
+ while (true) {
+ List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
+ if (reducedPoints.size() > nout) {
+ if (directMore) {
+ break;
+ }
+ if (!directLess) {
+ directLess = true;
+ }
+ epsilon *= 2;
+ } else {
+ if (directLess) {
+ break;
+ }
+ if (!directMore) {
+ directMore = true;
+ }
+ epsilon /= 2;
+ }
+ }
+ double left = 0;
+ double right = 0;
+ if (directLess) {
+ left = epsilon / 2;
+ right = epsilon;
+ }
+ if (directMore) {
+ left = epsilon;
+ right = epsilon * 2;
+ }
+ while (Math.abs(right - left) > accuracy) {
+ double mid = (left + right) / 2;
+ List<Point> reducedPoints = FSW.reducePoints(ts.data, mid);
+ if (reducedPoints.size() > nout) {
+ left = mid;
+ } else {
+ right = mid;
+ }
+ }
+ return (left + right) / 2;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
index 9abb7b748c6..d09641736fd 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
@@ -23,6 +23,7 @@ package org.apache.iotdb.db.query.simpiece;
import java.io.FileInputStream;
import java.io.FileWriter;
+import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;
@@ -30,38 +31,44 @@ public class MySample_shrinkingcone {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
- boolean series = true; // 从1开始编号列而不是时间戳列
String[] datasetNameList =
new String[] {
- // "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Qloss_Ah",
"NISTPV-Ground-2015-Pyra1_Wm2",
- // "NISTPV-Ground-2015-RTD_C_3"
+ "NISTPV-Ground-2015-RTD_C_3"
};
-
- int[] noutList = new int[] {0};
-
+ int[] noutList = new int[] {100};
double[] r = new double[] {0.1, 0.5, 1.3};
+ double[] epsilonList = new double[] {0.001, 408.55843019485474, 7.996772289276123};
for (int y = 0; y < datasetNameList.length; y++) {
String datasetName = datasetNameList[y];
- // int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
- // int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
- // int N = end - start; // -1 for all
-
- int start = 0;
- int end = 100000;
+ int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
+ int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
int N = end - start;
+ // int start = 0;
+ // int end = 10000;
+ // int N = end - start;
for (int nout : noutList) {
- // apply Sim-Piece on the input file, outputting nout points saved in csvFile
boolean hasHeader = false;
try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
String delimiter = ",";
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
- inputStream, delimiter, false, N, start, hasHeader, series);
-
- double epsilon = 10;
+ inputStream, delimiter, false, N, start, hasHeader, false);
+ // double epsilon = getSCParam(nout, ts, 1e-6);
+ double epsilon = epsilonList[y];
List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
+ System.out.println(
+ datasetName
+ + ": n="
+ + N
+ + ",m="
+ + nout
+ + ",epsilon="
+ + epsilon
+ + ",actual m="
+ + reducedPoints.size());
try (PrintWriter writer =
new PrintWriter(
new FileWriter(datasetName + "-" + N + "-" + reducedPoints.size() + "-sc.csv"))) {
@@ -75,4 +82,50 @@ public class MySample_shrinkingcone {
}
}
}
+
+ public static double getSCParam(int nout, TimeSeries ts, double accuracy) throws IOException {
+ double epsilon = 1;
+ boolean directLess = false;
+ boolean directMore = false;
+ while (true) {
+ List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
+ if (reducedPoints.size() > nout) {
+ if (directMore) {
+ break;
+ }
+ if (!directLess) {
+ directLess = true;
+ }
+ epsilon *= 2;
+ } else {
+ if (directLess) {
+ break;
+ }
+ if (!directMore) {
+ directMore = true;
+ }
+ epsilon /= 2;
+ }
+ }
+ double left = 0;
+ double right = 0;
+ if (directLess) {
+ left = epsilon / 2;
+ right = epsilon;
+ }
+ if (directMore) {
+ left = epsilon;
+ right = epsilon * 2;
+ }
+ while (Math.abs(right - left) > accuracy) {
+ double mid = (left + right) / 2;
+ List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, mid);
+ if (reducedPoints.size() > nout) {
+ left = mid;
+ } else {
+ right = mid;
+ }
+ }
+ return (left + right) / 2;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
index 5d93d23bec7..9d7afe6fca8 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
@@ -32,20 +32,23 @@ public class MySample_simpiece {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
- boolean series = true; // 从1开始编号列而不是时间戳列
String[] datasetNameList =
new String[] {
"NISTPV-Ground-2015-Qloss_Ah",
- // "NISTPV-Ground-2015-Pyra1_Wm2",
- // "NISTPV-Ground-2015-RTD_C_3"
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ "NISTPV-Ground-2015-RTD_C_3"
};
int[] noutList = new int[] {100};
double[] r = new double[] {0.1, 0.5, 1.3};
+ double[] epsilonList = new double[] {0.0009999, 316.5642651891633, 9.186667042922977};
for (int y = 0; y < datasetNameList.length; y++) {
String datasetName = datasetNameList[y];
int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
- int N = end - start; // -1 for all
+ int N = end - start;
+ // int start = 0;
+ // int end = 10000;
+ // int N = end - start;
for (int nout : noutList) {
// apply Sim-Piece on the input file, outputting nout points saved in csvFile
@@ -54,14 +57,26 @@ public class MySample_simpiece {
String delimiter = ",";
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
- inputStream, delimiter, false, N, start, hasHeader, series);
- double epsilon = getSimPieceParam(nout, ts, 1e-6);
- System.out.println(datasetName + ": n=" + N + ",m=" + nout + ",epsilon=" + epsilon);
+ inputStream, delimiter, false, N, start, hasHeader, false);
+ // double epsilon = getSimPieceParam(nout, ts, 1e-6);
+ double epsilon = epsilonList[y];
SimPiece simPiece = new SimPiece(ts.data, epsilon);
+ System.out.println(
+ datasetName
+ + ": n="
+ + N
+ + ",m="
+ + nout
+ + ",epsilon="
+ + epsilon
+ + ",actual m="
+ + simPiece.segments.size() * 2);
List<SimPieceSegment> segments = simPiece.segments;
segments.sort(Comparator.comparingLong(SimPieceSegment::getInitTimestamp));
try (PrintWriter writer =
- new PrintWriter(new FileWriter(datasetName + "-" + N + "-" + nout + ".csv"))) {
+ new PrintWriter(
+ new FileWriter(
+ datasetName + "-" + N + "-" + segments.size() * 2 + "-simpiece.csv"))) {
for (int i = 0; i < segments.size() - 1; i++) {
// start point of this segment
writer.println(segments.get(i).getInitTimestamp() + "," + segments.get(i).getB());
(iotdb) 12/12: update readme
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ba493800bf842c9ccfd2c959124b48c59e78d114
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jul 21 23:45:32 2024 +0800
update readme
---
README.md | 7 +-
.../groupby/LocalGroupByExecutorTri_SimPiece.java | 6 -
.../org/apache/iotdb/db/query/simpiece/FSW.java | 3 -
.../iotdb/db/query/simpiece/MySample_fsw.java | 24 +--
.../iotdb/db/query/simpiece/MySample_fsw_full.java | 12 +-
.../query/simpiece/MySample_fsw_full_serial.java | 198 --------------------
.../db/query/simpiece/MySample_shrinkingcone.java | 25 +--
.../simpiece/MySample_shrinkingcone_full.java | 16 +-
.../MySample_shrinkingcone_full_serial.java | 195 --------------------
.../iotdb/db/query/simpiece/MySample_simpiece.java | 29 +--
.../db/query/simpiece/MySample_simpiece_full.java | 8 +-
.../simpiece/MySample_simpiece_full_serial.java | 199 --------------------
.../iotdb/db/query/simpiece/MySample_visval.java | 94 ----------
.../org/apache/iotdb/db/query/simpiece/Visval.java | 41 -----
.../iotdb/db/query/simpiece/VisvalPoint.java | 23 ++-
.../apache/iotdb/db/query/simpiece/Visvalold.java | 203 ---------------------
16 files changed, 59 insertions(+), 1024 deletions(-)
diff --git a/README.md b/README.md
index 4f288392135..f4b5b8681fc 100644
--- a/README.md
+++ b/README.md
@@ -23,8 +23,13 @@
- The code for ILTS and other baselines compared in experiments are available in this repository.
- ILTS is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_ILTS.java) with convex hull precomputation in [tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java](tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java).
- LTTB is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_LTTB.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_LTTB.java).
+ - MinMaxLTTB is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_MinMaxPreselection.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_MinMaxPreselection.java) and [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java).
+ - Visval is implemented in [server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java](server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java).
- MinMax is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_MinMax.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_MinMax.java).
- M4 is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_M4.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_M4.java).
- - MinMaxLTTB is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_MinMaxPreselection.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_MinMaxPreselection.java) and [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java).
+ - Uniform is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Uniform.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Uniform.java).
+ - FSW is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_FSW.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_FSW.java) and [server/src/main/java/org/apache/iotdb/db/query/simpiece/FSW.java](server/src/main/java/org/apache/iotdb/db/query/simpiece/FSW.java).
+ - SC is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SC.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SC.java) and [server/src/main/java/org/apache/iotdb/db/query/simpiece/ShrinkingCone.java](server/src/main/java/org/apache/iotdb/db/query/simpiece/ShrinkingCone.java).
+ - Sim-Piece is implemented in [server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SimPiece.java](server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SimPiece.java) and [server/src/main/java/org/apache/iotdb/db/query/simpiece/SimPiece.java](server/src/main/java/org/apache/iotdb/db/query/simpiece/SimPiece.java).
- Some integration tests for correctness are in [server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java](server/src/test/java/org/apache/iotdb/db/integration/tri/MyTest_ILTS.java).
- For the README of [Apache IoTDB](https://iotdb.apache.org/) itself, please see [README_IOTDB.md](README_IOTDB.md). To build this repository, run `mvn clean package -DskipTests -pl -distribution`.
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SimPiece.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SimPiece.java
index 0a9b4fbefa5..3412d022d09 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SimPiece.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SimPiece.java
@@ -133,9 +133,6 @@ public class LocalGroupByExecutorTri_SimPiece implements GroupByExecutor {
result.reset();
}
- //
- // series.append(CONFIG.getP1v()).append("[").append(CONFIG.getP1t()).append("]").append(",");
-
SimPiece simPiece = new SimPiece(timeSeries.data, epsilon);
List<SimPieceSegment> segments = simPiece.segments;
segments.sort(Comparator.comparingLong(SimPieceSegment::getInitTimestamp));
@@ -176,9 +173,6 @@ public class LocalGroupByExecutorTri_SimPiece implements GroupByExecutor {
.append("]")
.append(",");
- //
- // series.append(CONFIG.getPnv()).append("[").append(CONFIG.getPnt()).append("]").append(",");
-
MinValueAggrResult minValueAggrResult = (MinValueAggrResult) results.get(0);
minValueAggrResult.updateResult(new MinMaxInfo<>(series.toString(), 0));
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/FSW.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/FSW.java
index 4b2e0163e1a..057e7c29607 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/FSW.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/FSW.java
@@ -57,14 +57,12 @@ public class FSW {
((p_upper[i] - points.get(segmentPoint.get(seg_no)[1]).getValue())
/ (points.get(i).getTimestamp()
- points.get(segmentPoint.get(seg_no)[1]).getTimestamp())));
- // (i - segmentPoint.get(seg_no)[1])));
lowSlope =
Math.max(
lowSlope,
((p_lower[i] - points.get(segmentPoint.get(seg_no)[1]).getValue())
/ (points.get(i).getTimestamp()
- points.get(segmentPoint.get(seg_no)[1]).getTimestamp())));
- // (i - segmentPoint.get(seg_no)[1])));
if (upSlope < lowSlope) {
seg_no += 1;
segmentPoint.add(new int[] {seg_no, csp_id});
@@ -76,7 +74,6 @@ public class FSW {
(points.get(i).getValue() - points.get(segmentPoint.get(seg_no)[1]).getValue())
/ (points.get(i).getTimestamp()
- points.get(segmentPoint.get(seg_no)[1]).getTimestamp());
- // (i - segmentPoint.get(seg_no)[1]);
if (s >= lowSlope && s <= upSlope) {
csp_id = i;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
index da43f5b824e..358b1e2d891 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw.java
@@ -31,25 +31,17 @@ public class MySample_fsw {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
- String[] datasetNameList =
- new String[]{
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3",
- "NISTPV-Ground-2015-WindSpeed_ms"
- };
- int[] noutList = new int[]{100};
- double[] r = new double[]{0.1, 0.5, 1.3, 0};
- int[] NList = new int[]{2500000, 2500000, 2500000, 500000};
- double[] epsilonList = new double[]{9.999999E-4, 284.4034399986267, 6.428162097930908,10.814893245697021};
- for (int y = 3; y < datasetNameList.length; y++) {
+ String[] datasetNameList = new String[] {"Qloss", "Pyra1", "RTD", "WindSpeed"};
+ int[] noutList = new int[] {100};
+ double[] r = new double[] {0.1, 0.5, 1.3, 0};
+ int[] NList = new int[] {2500000, 2500000, 2500000, 500000};
+ double[] epsilonList =
+ new double[] {9.999999E-4, 284.4034399986267, 6.428162097930908, 10.814893245697021};
+ for (int y = 0; y < datasetNameList.length; y++) {
String datasetName = datasetNameList[y];
int start = (int) (10000000 / 2 - NList[y] * r[y]); // 从0开始计数
int end = (int) (10000000 / 2 + NList[y] * (1 - r[y]));
int N = end - start;
- // int start = 0;
- // int end = 10000;
- // int N = end - start;
for (int nout : noutList) {
boolean hasHeader = false;
@@ -58,7 +50,7 @@ public class MySample_fsw {
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
inputStream, delimiter, false, N, start, hasHeader, false);
-// double epsilon = getFSWParam(nout, ts, 1e-6);
+ // double epsilon = getFSWParam(nout, ts, 1e-6);
double epsilon = epsilonList[y];
List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
System.out.println(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full.java
index e4e581488dd..706e140a1fd 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full.java
@@ -31,13 +31,7 @@ public class MySample_fsw_full {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
- String[] datasetNameList =
- new String[] {
- "NISTPV-Ground-2015-WindSpeed_ms",
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3"
- };
+ String[] datasetNameList = new String[] {"WindSpeed", "Qloss", "Pyra1", "RTD"};
int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
double[][] epsilonArray = {
@@ -142,8 +136,8 @@ public class MySample_fsw_full {
}
}
- for (int i = 0; i < epsilonArray.length; i++) { // 遍历行
- for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列
+ for (int i = 0; i < epsilonArray.length; i++) {
+ for (int j = 0; j < epsilonArray[i].length; j++) {
System.out.print(epsilonArray[i][j] + ",");
}
System.out.println();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full_serial.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full_serial.java
deleted file mode 100644
index b51c8dddcb2..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_fsw_full_serial.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
-
-package org.apache.iotdb.db.query.simpiece;
-
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.List;
-
-public class MySample_fsw_full_serial {
-
- public static void main(String[] args) {
- String fileDir = "D:\\desktop\\NISTPV\\";
- String[] datasetNameList =
- new String[] {
- "NISTPV-Ground-2015-WindSpeed_ms",
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3"
- };
- int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
-
- double[][] epsilonArray = {
- {
- 9.618271350860596,
- 9.101282596588135,
- 8.991526126861572,
- 8.62817907333374,
- 8.374999523162842,
- 8.202636241912842,
- 7.999906063079834,
- 7.947059154510498,
- 7.858933925628662
- },
- {
- 9.999275207519531E-4,
- 9.989738464355469E-4,
- 9.980201721191406E-4,
- 9.970664978027344E-4,
- 9.961128234863281E-4,
- 9.951591491699219E-4,
- 9.942054748535156E-4,
- 9.932518005371094E-4,
- 9.922981262207031E-4
- },
- {
- 440.0235962867737,
- 432.0247492790222,
- 423.55674982070923,
- 414.65001153945923,
- 405.77113008499146,
- 401.21820974349976,
- 396.7935194969177,
- 393.2065939903259,
- 392.30953645706177
- },
- {
- 9.295397281646729,
- 8.096678256988525,
- 7.536666393280029,
- 6.961357593536377,
- 6.473535060882568,
- 6.064479351043701,
- 5.743286609649658,
- 5.5219340324401855,
- 5.185042858123779
- }
- };
-
- // double[][] epsilonArray = new double[datasetNameList.length][];
- // for (int i = 0; i < datasetNameList.length; i++) {
- // epsilonArray[i] = new double[noutList.length];
- // }
-
- for (int y = 0; y < datasetNameList.length; y++) {
- String datasetName = datasetNameList[y];
- int start = 0;
- int end = 1000_0000;
- int N = end - start;
- boolean hasHeader = false;
- try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
- String delimiter = ",";
- TimeSeries ts =
- TimeSeriesReader.getMyTimeSeries(
- inputStream, delimiter, false, N, start, hasHeader, true);
- for (int x = 0; x < noutList.length; x++) {
- int nout = noutList[x];
- // double epsilon = getFSWParam(nout, ts, 1e-6);
- // epsilonArray[y][x] = epsilon;
-
- double epsilon = epsilonArray[y][x];
-
- List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
- System.out.println(
- datasetName
- + ": n="
- + N
- + ",m="
- + nout
- + ",epsilon="
- + epsilon
- + ",actual m="
- + reducedPoints.size());
- try (PrintWriter writer =
- new PrintWriter(
- new FileWriter(
- datasetName
- + "-"
- + N
- + "-"
- + nout
- + "-"
- + reducedPoints.size()
- + "-fsw.csv"))) {
- for (Point p : reducedPoints) {
- writer.println(p.getTimestamp() + "," + p.getValue());
- }
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- for (int i = 0; i < epsilonArray.length; i++) { // 遍历行
- for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列
- System.out.print(epsilonArray[i][j] + ",");
- }
- System.out.println();
- }
- }
-
- public static double getFSWParam(int nout, TimeSeries ts, double accuracy) throws IOException {
- double epsilon = 1;
- boolean directLess = false;
- boolean directMore = false;
- while (true) {
- List<Point> reducedPoints = FSW.reducePoints(ts.data, epsilon);
- if (reducedPoints.size() > nout) {
- if (directMore) {
- break;
- }
- if (!directLess) {
- directLess = true;
- }
- epsilon *= 2;
- } else {
- if (directLess) {
- break;
- }
- if (!directMore) {
- directMore = true;
- }
- epsilon /= 2;
- }
- }
- double left = 0;
- double right = 0;
- if (directLess) {
- left = epsilon / 2;
- right = epsilon;
- }
- if (directMore) {
- left = epsilon;
- right = epsilon * 2;
- }
- while (Math.abs(right - left) > accuracy) {
- double mid = (left + right) / 2;
- List<Point> reducedPoints = FSW.reducePoints(ts.data, mid);
- if (reducedPoints.size() > nout) {
- left = mid;
- } else {
- right = mid;
- }
- }
- return (left + right) / 2;
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
index 1b167064f62..f2e88ca98f3 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
@@ -31,26 +31,17 @@ public class MySample_shrinkingcone {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
- String[] datasetNameList =
- new String[]{
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3",
- "NISTPV-Ground-2015-WindSpeed_ms"
- };
- int[] noutList = new int[]{100};
- double[] r = new double[]{0.1, 0.5, 1.3, 0};
- int[] NList = new int[]{2500000, 2500000, 2500000, 500000};
- double[] epsilonList = new double[]{0.001, 408.55843019485474, 7.996772289276123,
- 14.692305088043213};
- for (int y = 3; y < datasetNameList.length; y++) {
+ String[] datasetNameList = new String[] {"Qloss", "Pyra1", "RTD", "WindSpeed"};
+ int[] noutList = new int[] {100};
+ double[] r = new double[] {0.1, 0.5, 1.3, 0};
+ int[] NList = new int[] {2500000, 2500000, 2500000, 500000};
+ double[] epsilonList =
+ new double[] {0.001, 408.55843019485474, 7.996772289276123, 14.692305088043213};
+ for (int y = 0; y < datasetNameList.length; y++) {
String datasetName = datasetNameList[y];
int start = (int) (10000000 / 2 - NList[y] * r[y]); // 从0开始计数
int end = (int) (10000000 / 2 + NList[y] * (1 - r[y]));
int N = end - start;
- // int start = 0;
- // int end = 10000;
- // int N = end - start;
for (int nout : noutList) {
boolean hasHeader = false;
@@ -59,7 +50,7 @@ public class MySample_shrinkingcone {
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
inputStream, delimiter, false, N, start, hasHeader, false);
-// double epsilon = getSCParam(nout, ts, 1e-6);
+ // double epsilon = getSCParam(nout, ts, 1e-6);
double epsilon = epsilonList[y];
List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
System.out.println(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java
index 6cb8dd16a92..3ab602021f9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full.java
@@ -31,19 +31,9 @@ public class MySample_shrinkingcone_full {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
- String[] datasetNameList =
- new String[] {
- "NISTPV-Ground-2015-WindSpeed_ms",
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3"
- };
+ String[] datasetNameList = new String[] {"WindSpeed", "Qloss", "Pyra1", "RTD"};
int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
- // double[] epsilonList = new double[]{0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001,
- // 0.001,
- // 0.0009999999999999999};
-
double[][] epsilonArray = {
{
14.388325214385986,
@@ -92,7 +82,7 @@ public class MySample_shrinkingcone_full {
int end = 1000_0000;
int N = end - start;
boolean hasHeader = false;
- try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
String delimiter = ",";
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
@@ -105,8 +95,6 @@ public class MySample_shrinkingcone_full {
double epsilon = epsilonArray[y][x];
- // double epsilon = epsilonList[x]; // 单独给Qloss手调
-
List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
System.out.println(
datasetName
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full_serial.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full_serial.java
deleted file mode 100644
index c2705919029..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone_full_serial.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
-
-package org.apache.iotdb.db.query.simpiece;
-
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.List;
-
-public class MySample_shrinkingcone_full_serial {
-
- public static void main(String[] args) {
- String fileDir = "D:\\desktop\\NISTPV\\";
- String[] datasetNameList =
- new String[] {
- "NISTPV-Ground-2015-WindSpeed_ms",
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3"
- };
- int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
-
- // double[] epsilonList = new double[]{0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001,
- // 0.001,
- // 0.0009999999999999999};
-
- double[][] epsilonArray = {
- {
- 14.388325214385986,
- 13.768231868743896,
- 13.343345165252686,
- 13.091249942779541,
- 12.69767427444458,
- 12.391706943511963,
- 12.167190074920654,
- 11.972727298736572,
- 11.756273746490479
- },
- {0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.0009999999999999999},
- {
- 680.2196469306946,
- 665.2842917442322,
- 650.2610821723938,
- 636.7945942878723,
- 620.9016032218933,
- 604.7394433021545,
- 592.9867577552795,
- 578.6471419334412,
- 570.5106825828552
- },
- {
- 11.259880542755127,
- 10.37297010421753,
- 9.402754306793213,
- 8.867334842681885,
- 8.384315967559814,
- 7.915340900421143,
- 7.39516019821167,
- 6.892223834991455,
- 6.5178914070129395
- }
- };
-
- // double[][] epsilonArray = new double[datasetNameList.length][];
- // for (int i = 0; i < datasetNameList.length; i++) {
- // epsilonArray[i] = new double[noutList.length];
- // }
-
- for (int y = 0; y < datasetNameList.length; y++) {
- String datasetName = datasetNameList[y];
- int start = 0;
- int end = 1000_0000;
- int N = end - start;
- boolean hasHeader = false;
- try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
- String delimiter = ",";
- TimeSeries ts =
- TimeSeriesReader.getMyTimeSeries(
- inputStream, delimiter, false, N, start, hasHeader, true);
- for (int x = 0; x < noutList.length; x++) {
- int nout = noutList[x];
-
- // double epsilon = getSCParam(nout, ts, 1e-6);
- // epsilonArray[y][x] = epsilon;
-
- double epsilon = epsilonArray[y][x];
-
- // double epsilon = epsilonList[x]; // 单独给Qloss手调
-
- List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
- System.out.println(
- datasetName
- + ": n="
- + N
- + ",m="
- + nout
- + ",epsilon="
- + epsilon
- + ",actual m="
- + reducedPoints.size());
- try (PrintWriter writer =
- new PrintWriter(
- new FileWriter(
- datasetName
- + "-"
- + N
- + "-"
- + nout
- + "-"
- + reducedPoints.size()
- + "-sc.csv"))) {
- for (Point p : reducedPoints) {
- writer.println(p.getTimestamp() + "," + p.getValue());
- }
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- for (int i = 0; i < epsilonArray.length; i++) { // 遍历行
- for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列
- System.out.print(epsilonArray[i][j] + ",");
- }
- System.out.println();
- }
- }
-
- public static double getSCParam(int nout, TimeSeries ts, double accuracy) throws IOException {
- double epsilon = 1;
- boolean directLess = false;
- boolean directMore = false;
- while (true) {
- List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
- if (reducedPoints.size() > nout) {
- if (directMore) {
- break;
- }
- if (!directLess) {
- directLess = true;
- }
- epsilon *= 2;
- } else {
- if (directLess) {
- break;
- }
- if (!directMore) {
- directMore = true;
- }
- epsilon /= 2;
- }
- }
- double left = 0;
- double right = 0;
- if (directLess) {
- left = epsilon / 2;
- right = epsilon;
- }
- if (directMore) {
- left = epsilon;
- right = epsilon * 2;
- }
- while (Math.abs(right - left) > accuracy) {
- double mid = (left + right) / 2;
- List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, mid);
- if (reducedPoints.size() > nout) {
- left = mid;
- } else {
- right = mid;
- }
- }
- return (left + right) / 2;
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
index 8792c3a327e..c0f64f24a46 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
@@ -32,26 +32,17 @@ public class MySample_simpiece {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
- String[] datasetNameList =
- new String[]{
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3",
- "NISTPV-Ground-2015-WindSpeed_ms"
- };
- int[] noutList = new int[]{100};
- double[] r = new double[]{0.1, 0.5, 1.3, 0};
- double[] epsilonList = new double[]{0.0009999, 316.5642651891633, 9.186667042922977,
- 11.162719900131227};
- int[] NList = new int[]{2500000, 2500000, 2500000, 500000};
- for (int y = 3; y < datasetNameList.length; y++) {
+ String[] datasetNameList = new String[] {"Qloss", "Pyra1", "RTD", "WindSpeed"};
+ int[] noutList = new int[] {100};
+ double[] r = new double[] {0.1, 0.5, 1.3, 0};
+ double[] epsilonList =
+ new double[] {0.0009999, 316.5642651891633, 9.186667042922977, 11.162719900131227};
+ int[] NList = new int[] {2500000, 2500000, 2500000, 500000};
+ for (int y = 0; y < datasetNameList.length; y++) {
String datasetName = datasetNameList[y];
int start = (int) (10000000 / 2 - NList[y] * r[y]); // 从0开始计数
int end = (int) (10000000 / 2 + NList[y] * (1 - r[y]));
int N = end - start;
- // int start = 0;
- // int end = 10000;
- // int N = end - start;
for (int nout : noutList) {
// apply Sim-Piece on the input file, outputting nout points saved in csvFile
@@ -61,7 +52,7 @@ public class MySample_simpiece {
TimeSeries ts =
TimeSeriesReader.getMyTimeSeries(
inputStream, delimiter, false, N, start, hasHeader, false);
-// double epsilon = getSimPieceParam(nout, ts, 1e-6);
+ // double epsilon = getSimPieceParam(nout, ts, 1e-6);
double epsilon = epsilonList[y];
SimPiece simPiece = new SimPiece(ts.data, epsilon);
System.out.println(
@@ -86,7 +77,7 @@ public class MySample_simpiece {
// end point of this segment
double v =
(segments.get(i + 1).getInitTimestamp() - segments.get(i).getInitTimestamp())
- * segments.get(i).getA()
+ * segments.get(i).getA()
+ segments.get(i).getB();
writer.println(segments.get(i + 1).getInitTimestamp() + "," + v);
}
@@ -97,7 +88,7 @@ public class MySample_simpiece {
+ segments.get(segments.size() - 1).getB());
double v =
(simPiece.lastTimeStamp - segments.get(segments.size() - 1).getInitTimestamp())
- * segments.get(segments.size() - 1).getA()
+ * segments.get(segments.size() - 1).getA()
+ segments.get(segments.size() - 1).getB();
writer.println(simPiece.lastTimeStamp + "," + v);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full.java
index e7a2cd19c43..16876b2886b 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full.java
@@ -32,13 +32,7 @@ public class MySample_simpiece_full {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
- String[] datasetNameList =
- new String[] {
- "NISTPV-Ground-2015-WindSpeed_ms",
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3"
- };
+ String[] datasetNameList = new String[] {"WindSpeed", "Qloss", "Pyra1", "RTD"};
int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
double[][] epsilonArray = {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full_serial.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full_serial.java
deleted file mode 100644
index 4dd8c88eaf7..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece_full_serial.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
-
-package org.apache.iotdb.db.query.simpiece;
-
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.Comparator;
-import java.util.List;
-
-public class MySample_simpiece_full_serial {
-
- public static void main(String[] args) {
- String fileDir = "D:\\desktop\\NISTPV\\";
- String[] datasetNameList =
- new String[] {
- "NISTPV-Ground-2015-WindSpeed_ms",
- "NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3"
- };
- int[] noutList = new int[] {320, 360, 400, 440, 480, 520, 560, 600, 640};
-
- double[][] epsilonArray = {
- {
- 10.470000203704831,
- 9.973635465240477,
- 9.571347576141356,
- 9.339938166046142,
- 9.208912633514405,
- 9.135163892364506,
- 8.744444478607178,
- 8.61042620162964,
- 8.422101323699948
- },
- {
- 9.987499999999999E-4,
- 9.974999999999997E-4,
- 9.784999999999998E-4,
- 9.4425E-4,
- 9.255E-4,
- 8.8375E-4,
- 8.4775E-4,
- 7.865E-4,
- 7.3825E-4
- },
- {
- 502.0505219552564,
- 493.3274919509581,
- 485.91779255051347,
- 479.26180967157245,
- 463.5216509033579,
- 446.88677012006065,
- 441.2913785749938,
- 435.45897277584766,
- 426.0481592410648
- },
- {
- 13.678959740142822,
- 12.275238493499756,
- 11.224719192962645,
- 10.475999729766848,
- 9.741776887359617,
- 9.125823136444094,
- 8.625197060394289,
- 8.105121862335203,
- 7.743280215911865
- }
- };
-
- // double[][] epsilonArray = new double[datasetNameList.length][];
- // for (int i = 0; i < datasetNameList.length; i++) {
- // epsilonArray[i] = new double[noutList.length];
- // }
-
- for (int y = 0; y < datasetNameList.length; y++) {
- String datasetName = datasetNameList[y];
- int start = 0;
- int end = 1000_0000;
- int N = end - start;
- // apply Sim-Piece on the input file, outputting nout points saved in csvFile
- boolean hasHeader = false;
- try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
- String delimiter = ",";
- TimeSeries ts =
- TimeSeriesReader.getMyTimeSeries(
- inputStream, delimiter, false, N, start, hasHeader, true);
- for (int x = 0; x < noutList.length; x++) {
- int nout = noutList[x];
-
- // double epsilon = getSimPieceParam(nout, ts, 1e-6);
- // epsilonArray[y][x] = epsilon;
-
- double epsilon = epsilonArray[y][x];
-
- SimPiece simPiece = new SimPiece(ts.data, epsilon);
- System.out.println(
- datasetName
- + ": n="
- + N
- + ",m="
- + nout
- + ",epsilon="
- + epsilon
- + ",actual m="
- + simPiece.segments.size() * 2);
- List<SimPieceSegment> segments = simPiece.segments;
- segments.sort(Comparator.comparingLong(SimPieceSegment::getInitTimestamp));
- try (PrintWriter writer =
- new PrintWriter(
- new FileWriter(
- datasetName
- + "-"
- + N
- + "-"
- + nout
- + "-"
- + segments.size() * 2
- + "-simpiece.csv"))) {
- for (int i = 0; i < segments.size() - 1; i++) {
- // start point of this segment
- writer.println(segments.get(i).getInitTimestamp() + "," + segments.get(i).getB());
- // end point of this segment
- double v =
- (segments.get(i + 1).getInitTimestamp() - segments.get(i).getInitTimestamp())
- * segments.get(i).getA()
- + segments.get(i).getB();
- writer.println(segments.get(i + 1).getInitTimestamp() + "," + v);
- }
- // the two end points of the last segment
- writer.println(
- segments.get(segments.size() - 1).getInitTimestamp()
- + ","
- + segments.get(segments.size() - 1).getB());
- double v =
- (simPiece.lastTimeStamp - segments.get(segments.size() - 1).getInitTimestamp())
- * segments.get(segments.size() - 1).getA()
- + segments.get(segments.size() - 1).getB();
- writer.println(simPiece.lastTimeStamp + "," + v);
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- for (int i = 0; i < epsilonArray.length; i++) { // 遍历行
- for (int j = 0; j < epsilonArray[i].length; j++) { // 遍历列
- System.out.print(epsilonArray[i][j] + ",");
- }
- System.out.println();
- }
- }
-
- public static double getSimPieceParam(int nout, TimeSeries ts, double accuracy)
- throws IOException {
- double epsilon = ts.range * 0.001;
- while (true) {
- SimPiece simPiece = new SimPiece(ts.data, epsilon);
- if (simPiece.segments.size() * 2 > nout) { // note *2 for disjoint
- epsilon *= 2;
- } else {
- break;
- }
- }
- double left = epsilon / 2;
- double right = epsilon;
- while (Math.abs(right - left) > accuracy) {
- double mid = (left + right) / 2;
- SimPiece simPiece = new SimPiece(ts.data, mid);
- if (simPiece.segments.size() * 2 > nout) { // note *2 for disjoint
- left = mid;
- } else {
- right = mid;
- }
- }
- return (left + right) / 2;
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java
deleted file mode 100644
index 1ed02baba7c..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java
+++ /dev/null
@@ -1,94 +0,0 @@
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing,
-// * software distributed under the License is distributed on an
-// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// * KIND, either express or implied. See the License for the
-// * specific language governing permissions and limitations
-// * under the License.
-// */
-// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
-
-package org.apache.iotdb.db.query.simpiece;
-
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.io.PrintWriter;
-import java.util.List;
-
-public class MySample_visval {
-
- public static void main(String[] args) {
- String fileDir = "D:\\desktop\\NISTPV\\";
- String[] datasetNameList =
- new String[] {
- "NISTPV-Ground-2015-Qloss_Ah",
- // "NISTPV-Ground-2015-Pyra1_Wm2",
- // "NISTPV-Ground-2015-RTD_C_3"
- };
-
- int[] noutList = new int[] {2400000};
-
- double[] r = new double[] {0.1, 0.5, 1.3};
- for (int y = 0; y < datasetNameList.length; y++) {
- String datasetName = datasetNameList[y];
- int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
- int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
- int N = end - start; // -1 for all
- //
- // int start = 0;
- // int end = 100000;
- // int N = end - start;
-
- for (int nout : noutList) {
- // apply Sim-Piece on the input file, outputting nout points saved in csvFile
- boolean hasHeader = false;
- try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
- String delimiter = ",";
- List<VisvalPoint> points =
- TimeSeriesReader.getMyTimeSeriesVisval(
- inputStream, delimiter, false, N, start, hasHeader, false);
- System.out.println(points.size());
- System.out.println(nout);
-
- List<VisvalPoint> reducedPoints = Visval.reducePoints(points, nout);
- try (PrintWriter writer =
- new PrintWriter(
- new FileWriter(
- datasetName + "-" + N + "-" + reducedPoints.size() + "-visval.csv"))) {
- for (VisvalPoint p : reducedPoints) {
- writer.println(p.x + "," + p.y);
- }
- }
- System.out.println(reducedPoints.size());
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- // List<Point> points = new ArrayList<>();
- // points.add(new Point(1, 10));
- // points.add(new Point(2, 20));
- // points.add(new Point(3, 15));
- // points.add(new Point(4, 10));
- // points.add(new Point(5, 30));
- // points.add(new Point(6, 25));
- // points.add(new Point(7, 20));
- // int m = 4;
- // List<Point> reducedPoints = Visval.reducePoints(points, m);
- // System.out.println("Reduced points:");
- // for (Point point : reducedPoints) {
- // System.out.println("Timestamp: " + point.getTimestamp() + ", Value: " +
- // point.getValue());
- // }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
index d4b54d3adc1..62b3019edac 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
@@ -30,7 +30,6 @@ public class Visval {
}
ConcurrentSkipListMap<Double, VisvalPoint> minHeap = new ConcurrentSkipListMap<>();
- // 初始化双向链表和优先队列
for (int i = 1; i < points.size() - 1; i++) {
VisvalPoint p = points.get(i);
p.prev = points.get(i - 1);
@@ -40,18 +39,15 @@ public class Visval {
}
while (points.size() > targetCount) {
- // 移除面积最小的点
VisvalPoint p = minHeap.pollFirstEntry().getValue();
if (p == null) {
return points;
}
- // 移除点 p
VisvalPoint prev = p.prev;
VisvalPoint next = p.next;
prev.next = next;
next.prev = prev;
- // 更新相邻点的三角形面积
if (prev.prev != null) {
minHeap.remove(prev.area);
prev.area = calculateArea(prev.prev, prev, next);
@@ -69,43 +65,6 @@ public class Visval {
}
private static double calculateArea(VisvalPoint a, VisvalPoint b, VisvalPoint c) {
- // 计算三角形面积的函数
return Math.abs(a.x * (b.y - c.y) + b.x * (c.y - a.y) + c.x * (a.y - b.y)) / 2.0;
}
-
- // public static void main(String[] args) {
- // List<VisvalPoint> points = new ArrayList<>();
- // List<Point> pointsold = new ArrayList<>();
- // // 添加点到列表 points 100万数据
- // Random rand = new Random();
- //
- // int targetCount = 10;
- // for (int i = 0; i < 10000; i++) {
- // int v = rand.nextInt(1000);
- // points.add(new VisvalPoint(i, v));
- // pointsold.add(new Point(i, v));
- // }
- //
- //// int targetCount = 9000000;
- //// for (int i = 0; i < 10000000; i++) {
- //// points.add(new VisvalPoint(i, rand.nextInt(1000)));
- //// }
- //
- // // 计算运行时间
- // long startTime = System.currentTimeMillis();
- //
- // List<VisvalPoint> reducedPoints = Visval.reducePoints(points, targetCount);
- // for (VisvalPoint point : reducedPoints) {
- // System.out.println("Timestamp: " + point.x + ", Value: " + point.y);
- // }
- // List<Point> reducedPointsold = Visvalold.reducePoints(pointsold, targetCount);
- // for (Point point : reducedPointsold) {
- // System.out.println("Timestamp: " + point.getTimestamp() + ", Value: " +
- // point.getValue());
- // }
- // // 输出结果
- // System.out.println(reducedPoints.size());
- // long endTime = System.currentTimeMillis();
- // System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
- // }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalPoint.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalPoint.java
index ef29bc30c09..358c68d2cac 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalPoint.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalPoint.java
@@ -1,10 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.iotdb.db.query.simpiece;
public class VisvalPoint {
public double x, y;
- public VisvalPoint prev, next; // 双向链表中的前驱和后继
- public double area; // 与相邻点形成的三角形面积
+ public VisvalPoint prev, next;
+ public double area;
VisvalPoint(double x, double y) {
this.x = x;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visvalold.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visvalold.java
deleted file mode 100644
index 2592d042c3c..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visvalold.java
+++ /dev/null
@@ -1,203 +0,0 @@
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing,
-// * software distributed under the License is distributed on an
-// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// * KIND, either express or implied. See the License for the
-// * specific language governing permissions and limitations
-// * under the License.
-// */
-
-package org.apache.iotdb.db.query.simpiece;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.IntStream;
-
-// Visvalingam-Whyatt method of poly-line vertex reduction
-//
-// Visvalingam, M and Whyatt J D (1993)
-// "Line Generalisation by Repeated Elimination of Points", Cartographic J., 30 (1), 46 - 51
-//
-// Described here:
-//
-// http://web.archive.org/web/20100428020453/http://www2.dcs.hull.ac.uk/CISRG/publications/DPs/DP10/DP10.html
-//
-// =========================================
-//
-// The MIT License (MIT)
-//
-// Copyright (c) 2014 Elliot Hallmark
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-//
-// ================================
-public class Visvalold {
-
- private static class Triangle implements Comparable<Triangle> {
-
- int index;
- double area;
-
- Triangle(int index, double area) {
- this.index = index;
- this.area = area;
- }
-
- @Override
- public int compareTo(Triangle other) {
- return Double.compare(this.area, other.area);
- }
- }
-
- public static List<Point> reducePoints(List<Point> points, int m) {
- if (points.size() <= m) {
- return points;
- }
-
- // areas and remainIdx: records the dominating areas and indexes of remaining points during
- // bottom-up elimination
- // through remainIdx we can know the adjacency of remaining points easily.
- // TreeMap<Double, Integer> areasMap = new TreeMap<>();// for speedup
- List<Double> areas = triangleAreaList(points);
- List<Integer> remainIdx = new ArrayList<>();
- IntStream.range(0, points.size()).forEach(remainIdx::add);
-
- int minIndex = 0;
- for (int i = 1; i < areas.size(); i++) {
- if (areas.get(i) < areas.get(minIndex)) {
- minIndex = i;
- }
- }
- // AtomicInteger minIndexTmp = new AtomicInteger(0);
- // IntStream.range(0, areas.size()).parallel().forEach(i -> {
- // if (areas.get(i) < areas.get(minIndexTmp.get())) {
- // minIndexTmp.set(i);
- // }
- // });
- // int minIndex = minIndexTmp.get();
- double this_area = areas.get(minIndex);
- areas.remove(minIndex);
- remainIdx.remove(minIndex);
-
- while (remainIdx.size() > m) {
- boolean skip =
- false; // false mean next round needs to argmin globally, otherwise use recentMinIdx
- int recentMinIdx = -1;
-
- // update right new triangle area
- double right_area = Double.POSITIVE_INFINITY;
- if (minIndex <= remainIdx.size() - 2) {
- // note that now i already pop out min_vert
- right_area =
- calculateTriangleArea(
- points.get(remainIdx.get(minIndex - 1)),
- points.get(remainIdx.get(minIndex)),
- points.get(remainIdx.get(minIndex + 1)));
- if (right_area <= this_area) {
- // so next round does not need argmin globally
- skip = true;
- recentMinIdx = minIndex; // note that now i already pop out min_vert
- }
- areas.set(minIndex, right_area);
- }
-
- // update left new triangle area
- if (minIndex >= 2) {
- // note that now i already pop out min_vert
- double left_area =
- calculateTriangleArea(
- points.get(remainIdx.get(minIndex - 2)),
- points.get(remainIdx.get(minIndex - 1)),
- points.get(remainIdx.get(minIndex)));
- if (left_area <= this_area) {
- if (skip) { // means right point area is smaller than this_area, then compare left and
- // right
- if (left_area <= right_area) {
- recentMinIdx = minIndex - 1;
- }
- // otherwise keep skip right point
- } else { // means right point area is larger than this_area, while left is smaller than
- skip = true; // so next round does not need argmin globally
- recentMinIdx = minIndex - 1;
- }
- }
- areas.set(minIndex - 1, left_area);
- }
-
- if (!skip) { // left and right new triangle both larger than this_area, needs to argmin
- // globally
- minIndex = 0;
- for (int i = 1; i < areas.size(); i++) {
- if (areas.get(i) < areas.get(minIndex)) {
- minIndex = i;
- }
- }
- // AtomicInteger minIndexTmp2 = new AtomicInteger(0);
- // IntStream.range(0, areas.size()).parallel().forEach(i -> {
- // if (areas.get(i) < areas.get(minIndexTmp2.get())) {
- // minIndexTmp2.set(i);
- // }
- // });
- // minIndex = minIndexTmp2.get();
- } else {
- minIndex = recentMinIdx;
- }
- this_area = areas.get(minIndex);
- areas.remove(minIndex);
- remainIdx.remove(minIndex);
- }
-
- List<Point> result = new ArrayList<>();
- for (int i : remainIdx) {
- result.add(points.get(i));
- }
- return result;
- }
-
- private static List<Double> triangleAreaList(List<Point> points) {
- List<Double> result = new ArrayList<>();
- result.add(Double.POSITIVE_INFINITY); // first
- for (int i = 1; i < points.size() - 1; i++) {
- double area = calculateTriangleArea(points.get(i - 1), points.get(i), points.get(i + 1));
- result.add(area);
- }
- result.add(Double.POSITIVE_INFINITY); // last
- return result;
- }
-
- private static double calculateTriangleArea(Point a, Point b, Point c) {
- double x1 = a.getTimestamp();
- double y1 = a.getValue();
- double x2 = b.getTimestamp();
- double y2 = b.getValue();
- double x3 = c.getTimestamp();
- double y3 = c.getValue();
-
- return Math.abs((x1 * (y2 - y3) + x2 * (y3 - y1) + x3 * (y1 - y2)) / 2.0);
- }
-}
(iotdb) 02/12: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/LTS-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c77a57e5041861867e3320516993d6e66d99bebb
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jul 14 00:24:48 2024 +0800
add
---
.../resources/conf/iotdb-engine.properties | 2 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 6 +
.../groupby/LocalGroupByExecutorTri_SC.java | 163 ++++++++++++++++++
.../groupby/LocalGroupByExecutorTri_Visval.java | 161 ++++++++++++++++++
.../db/query/simpiece/MySample_shrinkingcone.java | 78 +++++++++
.../{MySample.java => MySample_simpiece.java} | 6 +-
.../iotdb/db/query/simpiece/MySample_visval.java | 92 ++++++++++
.../iotdb/db/query/simpiece/ShrinkingCone.java | 80 +++++++++
.../org/apache/iotdb/db/query/simpiece/Visval.java | 189 +++++++++++++++++++++
9 files changed, 773 insertions(+), 4 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 60c5df8bf15..f26d8c0385a 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -19,7 +19,7 @@
####################
### enable Tri
####################
-# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece
+# MinMax, MinMaxLTTB, M4, LTTB, ILTS, SimPiece, SC, Visval
enable_Tri=MinMax
# SimPiece segment error threshold
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
index 7bc5433d854..322c1f51076 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithoutValueFilterDataSet.java
@@ -435,6 +435,12 @@ public class GroupByWithoutValueFilterDataSet extends GroupByEngineDataSet {
} else if (CONFIG.getEnableTri().equals("SimPiece")) {
return new LocalGroupByExecutorTri_SimPiece(
path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
+ } else if (CONFIG.getEnableTri().equals("SC")) {
+ return new LocalGroupByExecutorTri_SC(
+ path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
+ } else if (CONFIG.getEnableTri().equals("Visval")) {
+ return new LocalGroupByExecutorTri_Visval(
+ path, allSensors, dataType, context, timeFilter, fileFilter, ascending);
} else {
logger.info("No matched enable_tri!");
return new LocalGroupByExecutor(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SC.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SC.java
new file mode 100644
index 00000000000..1cf596c7ff2
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_SC.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.dataset.groupby;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.query.aggregation.AggregateResult;
+import org.apache.iotdb.db.query.aggregation.impl.MinValueAggrResult;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
+import org.apache.iotdb.db.query.reader.series.SeriesReader;
+import org.apache.iotdb.db.query.simpiece.Point;
+import org.apache.iotdb.db.query.simpiece.ShrinkingCone;
+import org.apache.iotdb.db.query.simpiece.TimeSeries;
+import org.apache.iotdb.db.query.simpiece.TimeSeriesReader;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
+import org.apache.iotdb.tsfile.read.common.ChunkSuit4Tri;
+import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.utils.Pair;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+public class LocalGroupByExecutorTri_SC implements GroupByExecutor {
+
+ private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
+
+ // Aggregate result buffer of this path
+ private final List<AggregateResult> results = new ArrayList<>();
+
+ TimeSeries timeSeries;
+
+ double epsilon = CONFIG.getEpsilon();
+
+ public LocalGroupByExecutorTri_SC(
+ PartialPath path,
+ Set<String> allSensors,
+ TSDataType dataType,
+ QueryContext context,
+ Filter timeFilter,
+ TsFileFilter fileFilter,
+ boolean ascending)
+ throws StorageEngineException, QueryProcessException {
+ // long start = System.nanoTime();
+
+ // get all data sources
+ QueryDataSource queryDataSource =
+ QueryResourceManager.getInstance().getQueryDataSource(path, context, timeFilter);
+
+ // update filter by TTL
+ // this.timeFilter = queryDataSource.updateFilterUsingTTL(timeFilter);
+
+ SeriesReader seriesReader =
+ new SeriesReader(
+ path,
+ allSensors,
+ // fix bug: here use the aggregation type as the series data type,
+ // not using pageReader.getAllSatisfiedPageData is ok
+ dataType,
+ context,
+ queryDataSource,
+ timeFilter,
+ null,
+ fileFilter,
+ ascending);
+
+ try {
+ // : this might be bad to load all chunk metadata at first
+ List<ChunkSuit4Tri> futureChunkList = new ArrayList<>();
+ futureChunkList.addAll(seriesReader.getAllChunkMetadatas4Tri());
+ // order futureChunkList by chunk startTime
+ futureChunkList.sort(
+ new Comparator<ChunkSuit4Tri>() {
+ public int compare(ChunkSuit4Tri o1, ChunkSuit4Tri o2) {
+ return ((Comparable) (o1.chunkMetadata.getStartTime()))
+ .compareTo(o2.chunkMetadata.getStartTime());
+ }
+ });
+
+ GroupByFilter groupByFilter = (GroupByFilter) timeFilter;
+ long startTime = groupByFilter.getStartTime();
+ long endTime = groupByFilter.getEndTime();
+
+ timeSeries = TimeSeriesReader.getTimeSeriesFromTsFiles(futureChunkList, startTime, endTime);
+
+ } catch (IOException e) {
+ throw new QueryProcessException(e.getMessage());
+ }
+ }
+
+ @Override
+ public void addAggregateResult(AggregateResult aggrResult) {
+ results.add(aggrResult);
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(
+ long curStartTime, long curEndTime, long startTime, long endTime, long interval)
+ throws IOException {
+ // group by curStartTime and curEndTime are not used in Sim-Piece segmentation
+
+ StringBuilder series = new StringBuilder();
+
+ // clear result cache
+ for (AggregateResult result : results) {
+ result.reset();
+ }
+
+ // shrinking cone
+ int length = timeSeries.length();
+ List<Point> points = timeSeries.data;
+
+ // Shrinking cone
+ List<Point> reducedPoints = ShrinkingCone.reducePoints(timeSeries.data, epsilon);
+
+ for (Point p : reducedPoints) {
+ series.append(p.getValue()).append("[").append(p.getTimestamp()).append("]").append(",");
+ }
+
+ MinValueAggrResult minValueAggrResult = (MinValueAggrResult) results.get(0);
+ minValueAggrResult.updateResult(new MinMaxInfo<>(series.toString(), 0));
+ return results;
+ }
+
+ @Override
+ public Pair<Long, Object> peekNextNotNullValue(long nextStartTime, long nextEndTime)
+ throws IOException {
+ throw new IOException("no implemented");
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(long curStartTime, long curEndTime)
+ throws IOException, QueryProcessException {
+ throw new IOException("no implemented");
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java
new file mode 100644
index 00000000000..0091e370620
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.dataset.groupby;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.query.aggregation.AggregateResult;
+import org.apache.iotdb.db.query.aggregation.impl.MinValueAggrResult;
+import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.filter.TsFileFilter;
+import org.apache.iotdb.db.query.reader.series.SeriesReader;
+import org.apache.iotdb.db.query.simpiece.Point;
+import org.apache.iotdb.db.query.simpiece.TimeSeries;
+import org.apache.iotdb.db.query.simpiece.TimeSeriesReader;
+import org.apache.iotdb.db.query.simpiece.Visval;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
+import org.apache.iotdb.tsfile.read.common.ChunkSuit4Tri;
+import org.apache.iotdb.tsfile.read.filter.GroupByFilter;
+import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.apache.iotdb.tsfile.utils.Pair;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+
+public class LocalGroupByExecutorTri_Visval implements GroupByExecutor {
+
+ private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
+
+ // Aggregate result buffer of this path
+ private final List<AggregateResult> results = new ArrayList<>();
+
+ TimeSeries timeSeries;
+
+ int m;
+
+ public LocalGroupByExecutorTri_Visval(
+ PartialPath path,
+ Set<String> allSensors,
+ TSDataType dataType,
+ QueryContext context,
+ Filter timeFilter,
+ TsFileFilter fileFilter,
+ boolean ascending)
+ throws StorageEngineException, QueryProcessException {
+ // long start = System.nanoTime();
+
+ // get all data sources
+ QueryDataSource queryDataSource =
+ QueryResourceManager.getInstance().getQueryDataSource(path, context, timeFilter);
+
+ // update filter by TTL
+ // this.timeFilter = queryDataSource.updateFilterUsingTTL(timeFilter);
+
+ SeriesReader seriesReader =
+ new SeriesReader(
+ path,
+ allSensors,
+ // fix bug: here use the aggregation type as the series data type,
+ // not using pageReader.getAllSatisfiedPageData is ok
+ dataType,
+ context,
+ queryDataSource,
+ timeFilter,
+ null,
+ fileFilter,
+ ascending);
+
+ try {
+ // : this might be bad to load all chunk metadata at first
+ List<ChunkSuit4Tri> futureChunkList = new ArrayList<>();
+ futureChunkList.addAll(seriesReader.getAllChunkMetadatas4Tri());
+ // order futureChunkList by chunk startTime
+ futureChunkList.sort(
+ new Comparator<ChunkSuit4Tri>() {
+ public int compare(ChunkSuit4Tri o1, ChunkSuit4Tri o2) {
+ return ((Comparable) (o1.chunkMetadata.getStartTime()))
+ .compareTo(o2.chunkMetadata.getStartTime());
+ }
+ });
+
+ GroupByFilter groupByFilter = (GroupByFilter) timeFilter;
+ long startTime = groupByFilter.getStartTime();
+ long endTime = groupByFilter.getEndTime();
+ long interval = groupByFilter.getInterval();
+ this.m = (int) ((endTime - startTime) / interval);
+
+ timeSeries = TimeSeriesReader.getTimeSeriesFromTsFiles(futureChunkList, startTime, endTime);
+
+ } catch (IOException e) {
+ throw new QueryProcessException(e.getMessage());
+ }
+ }
+
+ @Override
+ public void addAggregateResult(AggregateResult aggrResult) {
+ results.add(aggrResult);
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(
+ long curStartTime, long curEndTime, long startTime, long endTime, long interval)
+ throws IOException {
+ // group by curStartTime and curEndTime are not used in Sim-Piece segmentation
+
+ StringBuilder series = new StringBuilder();
+
+ // clear result cache
+ for (AggregateResult result : results) {
+ result.reset();
+ }
+
+ // Visval
+ List<Point> reducedPoints = Visval.reducePoints(timeSeries.data, m);
+
+ for (Point p : reducedPoints) {
+ series.append(p.getValue()).append("[").append(p.getTimestamp()).append("]").append(",");
+ }
+
+ MinValueAggrResult minValueAggrResult = (MinValueAggrResult) results.get(0);
+ minValueAggrResult.updateResult(new MinMaxInfo<>(series.toString(), 0));
+ return results;
+ }
+
+ @Override
+ public Pair<Long, Object> peekNextNotNullValue(long nextStartTime, long nextEndTime)
+ throws IOException {
+ throw new IOException("no implemented");
+ }
+
+ @Override
+ public List<AggregateResult> calcResult(long curStartTime, long curEndTime)
+ throws IOException, QueryProcessException {
+ throw new IOException("no implemented");
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
new file mode 100644
index 00000000000..9abb7b748c6
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_shrinkingcone.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.util.List;
+
+public class MySample_shrinkingcone {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ boolean series = true; // 从1开始编号列而不是时间戳列
+ String[] datasetNameList =
+ new String[] {
+ // "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ // "NISTPV-Ground-2015-RTD_C_3"
+ };
+
+ int[] noutList = new int[] {0};
+
+ double[] r = new double[] {0.1, 0.5, 1.3};
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ // int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
+ // int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
+ // int N = end - start; // -1 for all
+
+ int start = 0;
+ int end = 100000;
+ int N = end - start;
+
+ for (int nout : noutList) {
+ // apply Sim-Piece on the input file, outputting nout points saved in csvFile
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ TimeSeries ts =
+ TimeSeriesReader.getMyTimeSeries(
+ inputStream, delimiter, false, N, start, hasHeader, series);
+
+ double epsilon = 10;
+ List<Point> reducedPoints = ShrinkingCone.reducePoints(ts.data, epsilon);
+ try (PrintWriter writer =
+ new PrintWriter(
+ new FileWriter(datasetName + "-" + N + "-" + reducedPoints.size() + "-sc.csv"))) {
+ for (Point p : reducedPoints) {
+ writer.println(p.getTimestamp() + "," + p.getValue());
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
similarity index 97%
rename from server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample.java
rename to server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
index a54d60ac1d1..5d93d23bec7 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_simpiece.java
@@ -28,7 +28,7 @@ import java.io.PrintWriter;
import java.util.Comparator;
import java.util.List;
-public class MySample {
+public class MySample_simpiece {
public static void main(String[] args) {
String fileDir = "D:\\desktop\\NISTPV\\";
@@ -36,8 +36,8 @@ public class MySample {
String[] datasetNameList =
new String[] {
"NISTPV-Ground-2015-Qloss_Ah",
- "NISTPV-Ground-2015-Pyra1_Wm2",
- "NISTPV-Ground-2015-RTD_C_3"
+ // "NISTPV-Ground-2015-Pyra1_Wm2",
+ // "NISTPV-Ground-2015-RTD_C_3"
};
int[] noutList = new int[] {100};
double[] r = new double[] {0.1, 0.5, 1.3};
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java
new file mode 100644
index 00000000000..7ca14859b37
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/MySample_visval.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Sim-Piece code forked from https://github.com/xkitsios/Sim-Piece.git
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.util.List;
+
+public class MySample_visval {
+
+ public static void main(String[] args) {
+ String fileDir = "D:\\desktop\\NISTPV\\";
+ boolean series = true; // 从1开始编号列而不是时间戳列
+ String[] datasetNameList =
+ new String[] {
+ // "NISTPV-Ground-2015-Qloss_Ah",
+ "NISTPV-Ground-2015-Pyra1_Wm2",
+ // "NISTPV-Ground-2015-RTD_C_3"
+ };
+
+ int[] noutList = new int[] {0};
+
+ double[] r = new double[] {0.1, 0.5, 1.3};
+ for (int y = 0; y < datasetNameList.length; y++) {
+ String datasetName = datasetNameList[y];
+ // int start = (int) (10000000 / 2 - 2500000 * r[y]); // 从0开始计数
+ // int end = (int) (10000000 / 2 + 2500000 * (1 - r[y]));
+ // int N = end - start; // -1 for all
+
+ int start = 0;
+ int end = 100;
+ int N = end - start;
+
+ for (int nout : noutList) {
+ // apply Sim-Piece on the input file, outputting nout points saved in csvFile
+ boolean hasHeader = false;
+ try (FileInputStream inputStream = new FileInputStream(fileDir + datasetName + ".csv")) {
+ String delimiter = ",";
+ TimeSeries ts =
+ TimeSeriesReader.getMyTimeSeries(
+ inputStream, delimiter, false, N, start, hasHeader, series);
+
+ List<Point> reducedPoints = Visval.reducePoints(ts.data, nout);
+ try (PrintWriter writer =
+ new PrintWriter(new FileWriter(datasetName + "-" + N + "-" + nout + "-visval.csv"))) {
+ for (Point p : reducedPoints) {
+ writer.println(p.getTimestamp() + "," + p.getValue());
+ }
+ }
+ // List<Point> points = new ArrayList<>();
+ // points.add(new Point(1, 10));
+ // points.add(new Point(2, 20));
+ // points.add(new Point(3, 15));
+ // points.add(new Point(4, 10));
+ // points.add(new Point(5, 30));
+ // points.add(new Point(6, 25));
+ // points.add(new Point(7, 20));
+ // int m = 4;
+ // List<Point> reducedPoints = Visval.reducePoints(points, m);
+ // System.out.println("Reduced points:");
+ // for (Point point : reducedPoints) {
+ // System.out.println("Timestamp: " + point.getTimestamp() + ", Value: " +
+ // point.getValue());
+ // }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/ShrinkingCone.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/ShrinkingCone.java
new file mode 100644
index 00000000000..a5977068a69
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/ShrinkingCone.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ShrinkingCone {
+
+ public static List<Point> reducePoints(List<Point> points, double epsilon) {
+ List<Point> result = new ArrayList<>();
+ int length = points.size();
+
+ result.add(points.get(0)); // first point
+
+ // Precompute upper and lower bounds
+ double[] p_upper = new double[length];
+ double[] p_lower = new double[length];
+ for (int j = 0; j < length; j++) {
+ double v = points.get(j).getValue();
+ p_upper[j] = v + epsilon;
+ p_lower[j] = v - epsilon;
+ }
+
+ // init the first segment
+ int sp = 0;
+ int i = 1;
+ double vsp = points.get(sp).getValue();
+ double dx = points.get(i).getTimestamp() - points.get(sp).getTimestamp();
+ double upSlope = (p_upper[i] - vsp) / dx;
+ double lowSlope = (p_lower[i] - vsp) / dx;
+
+ while (i < length - 1) {
+ i++;
+ vsp = points.get(sp).getValue(); // the value of the start point
+ dx = points.get(i).getTimestamp() - points.get(sp).getTimestamp(); // time distance
+ double upV = upSlope * dx + vsp; // the upper bound for the current point
+ double lowV = lowSlope * dx + vsp; // the lower bound for the current point
+
+ if (lowV <= points.get(i).getValue() && points.get(i).getValue() <= upV) {
+ // current point in the zone
+ // shrink the zone
+ upSlope = Math.min(upSlope, (p_upper[i] - vsp) / dx);
+ lowSlope = Math.max(lowSlope, (p_lower[i] - vsp) / dx);
+ } else {
+ // record segment point
+ result.add(points.get(i - 1));
+
+ // begin new segment
+ sp = i - 1; // joint style
+ vsp = points.get(sp).getValue(); // note sp has changed
+ dx = points.get(i).getTimestamp() - points.get(sp).getTimestamp(); // note sp has changed
+ upSlope = (p_upper[i] - vsp) / dx;
+ lowSlope = (p_lower[i] - vsp) / dx;
+ }
+ }
+
+ // write last point
+ result.add(points.get(points.size() - 1));
+
+ return result;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
new file mode 100644
index 00000000000..95002fd0d26
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.simpiece;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.IntStream;
+
+// Visvalingam-Whyatt method of poly-line vertex reduction
+//
+// Visvalingam, M and Whyatt J D (1993)
+// "Line Generalisation by Repeated Elimination of Points", Cartographic J., 30 (1), 46 - 51
+//
+// Described here:
+//
+// http://web.archive.org/web/20100428020453/http://www2.dcs.hull.ac.uk/CISRG/publications/DPs/DP10/DP10.html
+//
+// =========================================
+//
+// The MIT License (MIT)
+//
+// Copyright (c) 2014 Elliot Hallmark
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// ================================
+public class Visval {
+
+ private static class Triangle implements Comparable<Triangle> {
+
+ int index;
+ double area;
+
+ Triangle(int index, double area) {
+ this.index = index;
+ this.area = area;
+ }
+
+ @Override
+ public int compareTo(Triangle other) {
+ return Double.compare(this.area, other.area);
+ }
+ }
+
+ public static List<Point> reducePoints(List<Point> points, int m) {
+ if (points.size() <= m) {
+ return points;
+ }
+
+ // areas and remainIdx: records the dominating areas and indexes of remaining points during
+ // bottom-up elimination
+ // through remainIdx we can know the adjacency of remaining points easily.
+ List<Double> areas = triangleAreaList(points);
+ List<Integer> remainIdx = new ArrayList<>();
+ IntStream.range(0, points.size()).forEach(remainIdx::add);
+
+ int minIndex = 0;
+ for (int i = 1; i < areas.size(); i++) {
+ if (areas.get(i) < areas.get(minIndex)) {
+ minIndex = i;
+ }
+ }
+ double this_area = areas.get(minIndex);
+ areas.remove(minIndex);
+ remainIdx.remove(minIndex);
+
+ while (remainIdx.size() > m) {
+ boolean skip =
+ false; // false mean next round needs to argmin globally, otherwise use recentMinIdx
+ int recentMinIdx = -1;
+
+ // update right new triangle area
+ double right_area = Double.POSITIVE_INFINITY;
+ if (minIndex <= remainIdx.size() - 2) {
+ // note that now i already pop out min_vert
+ right_area =
+ calculateTriangleArea(
+ points.get(remainIdx.get(minIndex - 1)),
+ points.get(remainIdx.get(minIndex)),
+ points.get(remainIdx.get(minIndex + 1)));
+ if (right_area <= this_area) {
+ // so next round does not need argmin globally
+ skip = true;
+ recentMinIdx = minIndex; // note that now i already pop out min_vert
+ }
+ areas.set(minIndex, right_area);
+ }
+
+ // update left new triangle area
+ if (minIndex >= 2) {
+ // note that now i already pop out min_vert
+ double left_area =
+ calculateTriangleArea(
+ points.get(remainIdx.get(minIndex - 2)),
+ points.get(remainIdx.get(minIndex - 1)),
+ points.get(remainIdx.get(minIndex)));
+ if (left_area <= this_area) {
+ if (skip) { // means right point area is smaller than this_area, then compare left and
+ // right
+ if (left_area <= right_area) {
+ recentMinIdx = minIndex - 1;
+ }
+ // otherwise keep skip right point
+ } else { // means right point area is larger than this_area, while left is smaller than
+ skip = true; // so next round does not need argmin globally
+ recentMinIdx = minIndex - 1;
+ }
+ }
+ areas.set(minIndex - 1, left_area);
+ }
+
+ if (!skip) { // left and right new triangle both larger than this_area, needs to argmin
+ // globally
+ minIndex = 0;
+ for (int i = 1; i < areas.size(); i++) {
+ if (areas.get(i) < areas.get(minIndex)) {
+ minIndex = i;
+ }
+ }
+ } else {
+ minIndex = recentMinIdx;
+ }
+ this_area = areas.get(minIndex);
+ areas.remove(minIndex);
+ remainIdx.remove(minIndex);
+ }
+
+ List<Point> result = new ArrayList<>();
+ for (int i : remainIdx) {
+ result.add(points.get(i));
+ }
+ return result;
+ }
+
+ private static List<Double> triangleAreaList(List<Point> points) {
+ List<Double> result = new ArrayList<>();
+ result.add(Double.POSITIVE_INFINITY); // first
+ for (int i = 1; i < points.size() - 1; i++) {
+ double area = calculateTriangleArea(points.get(i - 1), points.get(i), points.get(i + 1));
+ result.add(area);
+ }
+ result.add(Double.POSITIVE_INFINITY); // last
+ return result;
+ }
+
+ private static double calculateTriangleArea(Point a, Point b, Point c) {
+ double x1 = a.getTimestamp();
+ double y1 = a.getValue();
+ double x2 = b.getTimestamp();
+ double y2 = b.getValue();
+ double x3 = c.getTimestamp();
+ double y3 = c.getValue();
+
+ return Math.abs((x1 * (y2 - y3) + x2 * (y3 - y1) + x3 * (y1 - y2)) / 2.0);
+ }
+}