You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by le...@apache.org on 2025/02/23 17:44:01 UTC
(iotdb) branch research/area-visualization created (now c35524ad1a6)
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a change to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at c35524ad1a6 add
This branch includes the following new commits:
new 584658ee74b disable license check
new b9109823f63 add standard implementation
new a74bbeefdea add
new a3702eef02a add
new 545bff4a975 format code
new 308affe7688 add
new 54b9ed1181a add
new 997e8c0687f fix
new 776f3845619 add
new 6857dc2e13d add ad
new 068c3e65882 add
new 85ef4103874 before
new 8dfbcb233e1 f
new b263ae6b181 de
new b8eb70f98c8 just array for e
new 65d259e5111 array
new cb9ca0b06f8 add
new 1f328cbe744 before
new 98cdae627c7 Add
new 33a64412236 ok
new c449be4b506 add
new aaa8825052f add m
new 5ee9e320f4c return bottom-up order
new d545f8db5f5 eBUG_build
new 4f8e6f3264b swab
new ec0a3cebce1 swab"
new dcb711d269a bottomUpL2;FSW
new 364f0d8d0b6 dp
new 2a22359d4bc add
new 5d649f12ea6 bottomUpYdiff
new eda41e9a27f add
new 30c63392fcf minmax
new e6bd68ed6a2 rdp
new 2270c6c291a before
new b74a25f14d2 fix
new 224f11a774a add
new 84b85948d46 ltd
new f2f408ba529 add
new 24d39c386f0 add
new dad9adc040a add
new a815c1137df add
new 8174cf4381e add
new 35b3b2c7a37 add
new c35524ad1a6 add
The 44 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 22/44: add m
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit aaa8825052f686955294ba4d5380b2cd39064433
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Jan 23 18:04:13 2025 +0800
add m
---
.../java/org/apache/iotdb/db/query/eBUG/Test1.java | 4 +-
.../java/org/apache/iotdb/db/query/eBUG/Test2.java | 41 +++++++++++++++++++
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 46 ++++++++++++++++++++--
3 files changed, 86 insertions(+), 5 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
index 426cfdc538d..f53f5719217 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
@@ -13,7 +13,7 @@ public class Test1 {
public static void main(String[] args) {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random(2);
+ Random rand = new Random(3);
int n = 10000;
int p = 10;
@@ -44,7 +44,7 @@ public class Test1 {
}
System.out.println("---------------------------------");
- int eParam = 0;
+ int eParam = 1;
long startTime = System.currentTimeMillis();
List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java
new file mode 100644
index 00000000000..589603525e5
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java
@@ -0,0 +1,41 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import io.jsonwebtoken.lang.Assert;
+
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
+
+public class Test2 {
+ // 用于测试在线采样
+ public static void main(String[] args) {
+ int n = 100000;
+ int eParam = 1;
+
+// int m = 100; // >2代表在线采样，<=2代表预计算
+ for (int m = 9900; m <= n; m++) {
+
+ Polyline polyline = new Polyline();
+ Random rand = new Random(2);
+ for (int i = 0; i < n; i++) {
+ double v = rand.nextInt(1000000);
+ polyline.addVertex(new Point(i, v));
+ }
+
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false, m);
+ long endTime = System.currentTimeMillis();
+ System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+
+ System.out.println(results.size());
+ Assert.isTrue(results.size() == m);
+ }
+// if (results.size() <= 100) {
+// System.out.println("+++++++++++++++++++");
+// for (Point point : results) {
+// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+// }
+// }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index e283789239a..b9147e4c574 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -56,10 +56,21 @@ public class eBUG {
}
public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
+ // precomputation mode
+ return buildEffectiveArea(lineToSimplify, e, debug, 0);
+ }
+
+ public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug, int m) {
if (e < 0) {
throw new IllegalArgumentException("Parameter e must be non-negative.");
}
+ if (m > 2) {
+ System.out.println("online sampling mode, returning " + m + " sampled points");
+ } else {
+ System.out.println("offline precomputation mode, returning each point with dominated significance");
+ }
+
List<Point> results = lineToSimplify.getVertices(); // 浅复制
// 存储的是点的引用，这样可以修改原来序列里点的淘汰状态
@@ -100,7 +111,13 @@ public class eBUG {
Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
double previousEA = -1;
- while (!triangleHeap.isEmpty()) {
+// while (!triangleHeap.isEmpty()) { // TODO 注意triangleHeap里装的是non-terminal point对应的三角形
+ // TODO 在线采样m个点，也就是说最后留下m-2个non-terminal point
+ // 注意：triangleHeap里装的包括了标记删除的点！所以triangleHeap.size()不是真正的留下的未被淘汰点数！
+ // 因此需要用fakeCnt来统计heap里的非真实点数，这样triangleHeap.size()-fakeCnt就是真正的留下的未被淘汰点数
+ int remainNonTerminalPoint = Math.max(0, m - 2);
+ int fakeCnt = 0;
+ while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
// 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
// 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
Triangle tri = triangleHeap.poll(); // O(logn)
@@ -108,6 +125,7 @@ public class eBUG {
// 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
// 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
if (tri.isDeleted) {
+ fakeCnt--; // 取出了一个被标记删除点
if (debug) {
System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
}
@@ -185,6 +203,7 @@ public class eBUG {
// 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
// 所以必须通过add来显式重新插入修改后的元素
triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
}
if (tri.next != null) {
@@ -232,12 +251,33 @@ public class eBUG {
// 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
// 所以必须通过add 来显式重新插入修改后的元素
triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
}
if (debug) {
System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
}
}
- return results; // 注意这就是lineToSimplify.getVertices()
+
+ if (m > 2) { // online sampling mode
+ // 把滞后的淘汰点施加上去，然后返回在线采样结果（也就是返回剩余未被淘汰的点）
+ for (Point p : laggedEliminatedPoints) {
+ p.markEliminated();
+ if (debug) {
+ System.out.println("apply lagged elimination of " + p);
+ }
+ }
+ List<Point> onlineSampled = new ArrayList<>();
+ Point start = lineToSimplify.get(0);
+ Point end = lineToSimplify.get(lineToSimplify.size() - 1);
+ while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ onlineSampled.add(start);
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ }
+ onlineSampled.add(end);
+ return onlineSampled;
+ } else { // offline precomputation mode, for precomputing the dominated significance of each point
+ return results; // 注意这就是lineToSimplify.getVertices()
+ }
}
public static void main(String[] args) {
@@ -262,7 +302,7 @@ public class eBUG {
}
long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ List<Point> results = buildEffectiveArea(polyline, eParam, false, 0);
long endTime = System.currentTimeMillis();
System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
(iotdb) 30/44: bottomUpYdiff
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5d649f12ea6b4d480cb93b3fbbd1a62ed830f783
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jan 28 02:11:14 2025 +0800
bottomUpYdiff
---
server/pom.xml | 4 +-
...ar => sample_BUYdiff-jar-with-dependencies.jar} | Bin 38716084 -> 38729082 bytes
server/sample_eBUG-jar-with-dependencies.jar | Bin 38716020 -> 38729070 bytes
server/sample_fsw-jar-with-dependencies.jar | Bin 38715995 -> 38729071 bytes
.../apache/iotdb/db/query/eBUG/BottomUpYdiff.java | 245 +++++++
.../java/org/apache/iotdb/db/query/eBUG/DNSL1.java | 154 ++---
.../java/org/apache/iotdb/db/query/eBUG/DP.java | 513 +++++++-------
.../java/org/apache/iotdb/db/query/eBUG/SWAB.java | 428 ++++++------
.../org/apache/iotdb/db/query/eBUG/Test10.java | 95 ++-
.../java/org/apache/iotdb/db/query/eBUG/Tool.java | 754 ++++++++++-----------
...e_bottomUpL2.java => sample_bottomUpYdiff.java} | 31 +-
11 files changed, 1228 insertions(+), 996 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index b0b6be3c539..8ea227c6cd3 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -283,12 +283,12 @@
<configuration>
<!-- <finalName>sample_fsw</finalName>-->
<finalName>sample_eBUG</finalName>
- <!-- <finalName>sample_BUL2</finalName>-->
+ <!-- <finalName>sample_BUYdiff</finalName>-->
<archive>
<manifest>
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
<mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpL2</mainClass>-->
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>-->
</manifest>
</archive>
<descriptorRefs>
diff --git a/server/sample_BUL2-jar-with-dependencies.jar b/server/sample_BUYdiff-jar-with-dependencies.jar
similarity index 96%
rename from server/sample_BUL2-jar-with-dependencies.jar
rename to server/sample_BUYdiff-jar-with-dependencies.jar
index d592875251c..d7360b7038a 100644
Binary files a/server/sample_BUL2-jar-with-dependencies.jar and b/server/sample_BUYdiff-jar-with-dependencies.jar differ
diff --git a/server/sample_eBUG-jar-with-dependencies.jar b/server/sample_eBUG-jar-with-dependencies.jar
index 0df972fa4a5..f442da1afb8 100644
Binary files a/server/sample_eBUG-jar-with-dependencies.jar and b/server/sample_eBUG-jar-with-dependencies.jar differ
diff --git a/server/sample_fsw-jar-with-dependencies.jar b/server/sample_fsw-jar-with-dependencies.jar
index c1981d2a984..3ca5857146d 100644
Binary files a/server/sample_fsw-jar-with-dependencies.jar and b/server/sample_fsw-jar-with-dependencies.jar differ
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/BottomUpYdiff.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/BottomUpYdiff.java
new file mode 100644
index 00000000000..6090415241f
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/BottomUpYdiff.java
@@ -0,0 +1,245 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.*;
+
+public class BottomUpYdiff {
+
+ public static List<Point> reducePoints(
+ List<Point> points, int m, DP.ERRORtype errorType, boolean debug) throws IOException {
+ if (m <= 2) {
+ throw new IOException("please make m>2");
+ }
+ // 直接控制采样点数
+ int total = points.size();
+ if (total < 3) {
+ return points; // 不足 3 个点无法形成三角形
+ }
+
+ // 每个triangle对应两个相邻的待合并的分段
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ points.get(0).prev = null;
+ points.get(0).next = points.get(1);
+ // points.get(0).index = 0;
+ points.get(total - 1).next = null;
+ points.get(total - 1).prev = points.get(total - 2);
+ // points.get(total - 1).index = points.size() - 1;
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+
+ double mc = DP.joint_segment_error(points, index1, index3, errorType);
+
+ triangles[i - 1] = new Triangle(index1, index2, index3, mc);
+
+ // 初始化点的状态 用于最后在线采样结果顺序输出未淘汰点
+ points.get(i).prev = points.get(i - 1);
+ points.get(i).next = points.get(i + 1);
+ // points.get(i).index = i; // for swab usage
+ }
+
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ }
+
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+
+ int remainNonTerminalPoint = m - 2;
+ int fakeCnt = 0;
+ // while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
+ while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll(); // O(logn)
+
+ // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ if (tri.isDeleted) {
+ fakeCnt--; // 取出了一个被标记删除点
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ continue;
+ }
+
+ // 真正的淘汰点
+ // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ if (debug) {
+ System.out.println("(1) eliminate " + points.get(tri.indices[1]));
+ }
+ points.get(tri.indices[1]).markEliminated();
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
+
+ Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+
+ // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // 前一个三角形连到后一个三角形
+ tri.prev.next =
+ tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ tri.prev.indices[2] = tri.indices[2];
+
+ double mc =
+ DP.joint_segment_error(points, tri.prev.indices[0], tri.prev.indices[2], errorType);
+
+ tri.prev.area = mc;
+ if (debug) {
+ System.out.println(
+ "(2) updating point on the left "
+ + points.get(tri.prev.indices[1])
+ + ", ranging ["
+ + tri.prev.indices[0]
+ + ","
+ + tri.prev.indices[2]
+ + "]");
+ }
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add来显式重新插入修改后的元素
+ triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+
+ if (tri.next != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+ tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+
+ Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
+
+ if (tri.prev != null) {
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
+ }
+
+ // 2. 处理pi被淘汰引起tri.next被更新的事情
+ tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ tri.next.indices[0] = tri.indices[0];
+
+ double mc =
+ DP.joint_segment_error(points, tri.next.indices[0], tri.next.indices[2], errorType);
+
+ tri.next.area = mc;
+ if (debug) {
+ System.out.println(
+ "(3) updating point on the right "
+ + points.get(tri.next.indices[1])
+ + ", ranging ["
+ + tri.next.indices[0]
+ + ","
+ + tri.next.indices[2]
+ + "]");
+ }
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ }
+
+ List<Point> onlineSampled = new ArrayList<>();
+ Point start = points.get(0);
+ Point end = points.get(points.size() - 1);
+ while (start
+ != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ }
+ onlineSampled.add(end);
+ return onlineSampled;
+ }
+
+ public static void main(String[] args) throws IOException {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 1000_00;
+ // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < points.size(); i++) {
+ Point point = points.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ int m = 100;
+
+ long startTime = System.currentTimeMillis();
+ List<Point> sampled = reducePoints(points, m, DP.ERRORtype.L_infy, false);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ System.out.println(sampled.size());
+
+ // startTime = System.currentTimeMillis();
+ //// List<Point> sampled2 = eBUG.buildEffectiveArea(points, 100000000, false, m);
+ // List<Point> sampled2 = SWAB.seg_bottomUp_m_withTimestamps(points, m, null, false);
+ // endTime = System.currentTimeMillis();
+ // System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ //// for (Point p : sampled2) {
+ //// System.out.println(p);
+ //// }
+ // System.out.println(sampled2.size());
+ // for (int i = 0; i < sampled2.size(); i++) {
+ // if (sampled.get(i).x != sampled2.get(i).x) {
+ // throw new IOException("wrong!");
+ // }
+ // }
+
+ // try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // // 写入字符串
+ // for (int i = 0; i < sampled.size(); i++) {
+ // writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ // }
+ //
+ // } catch (FileNotFoundException e) {
+ // e.printStackTrace();
+ // }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
index df2c09d35ab..be4ffe2d6f1 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
@@ -9,89 +9,91 @@ import java.util.Random;
import static org.apache.iotdb.db.query.eBUG.DP.dynamic_programming;
public class DNSL1 {
- // 默认使用linear interpolation连接分段首尾点来近似
- // 默认使用L1 error
- // 使用divide and conquer算法
- public static List<Point> reducePoints(List<Point> points, int m, boolean debug) {
- int n = points.size();
- int k = m - 1; // 分段数
- int intervalPts = (int) Math.pow((double) n / k, (double) 2 / 3); // divide batch点数
+ // 默认使用linear interpolation连接分段首尾点来近似
+ // 默认使用L1 error
+ // 使用divide and conquer算法
+ public static List<Point> reducePoints(List<Point> points, int m, boolean debug) {
+ int n = points.size();
+ int k = m - 1; // 分段数
+ int intervalPts = (int) Math.pow((double) n / k, (double) 2 / 3); // divide batch点数
- if (debug) {
- System.out.println("interval point length=" + intervalPts);
- }
+ if (debug) {
+ System.out.println("interval point length=" + intervalPts);
+ }
- // divide into intervalPts equal-length intervals
- List<Point> allSampledPoints = new ArrayList<>();
- for (int start = 0; start < n; start += intervalPts) {
- int end = Math.min(start + intervalPts, n);
- List<Point> batch = points.subList(start, end); // 左闭右开
- if (debug) {
- System.out.println("Processing batch: [" + start + "," + end + ")");
- }
- List<Point> sampledForBatch = dynamic_programming(batch, k, DP.ERROR.L1, false);
- allSampledPoints.addAll(sampledForBatch); // 把上一步的结果加到大的容器里
- }
+ // divide into intervalPts equal-length intervals
+ List<Point> allSampledPoints = new ArrayList<>();
+ for (int start = 0; start < n; start += intervalPts) {
+ int end = Math.min(start + intervalPts, n);
+ List<Point> batch = points.subList(start, end); // 左闭右开
+ if (debug) {
+ System.out.println("Processing batch: [" + start + "," + end + ")");
+ }
+ List<Point> sampledForBatch = dynamic_programming(batch, k, DP.ERRORtype.L1, false);
+ allSampledPoints.addAll(sampledForBatch); // 把上一步的结果加到大的容器里
+ }
- // 在大的容器上最后执行DP.dynamic_programming得到最后的m个采样点
- if (debug) {
- System.out.println("number of points from batch sampling: " + allSampledPoints.size());
- }
- List<Point> finalSampledPoints = dynamic_programming(allSampledPoints, k, DP.ERROR.L1, false);
- if (debug) {
- System.out.println("result point number: " + finalSampledPoints.size());
- }
- return finalSampledPoints;
+ // 在大的容器上最后执行DP.dynamic_programming得到最后的m个采样点
+ if (debug) {
+ System.out.println("number of points from batch sampling: " + allSampledPoints.size());
+ }
+ List<Point> finalSampledPoints =
+ dynamic_programming(allSampledPoints, k, DP.ERRORtype.L1, false);
+ if (debug) {
+ System.out.println("result point number: " + finalSampledPoints.size());
}
+ return finalSampledPoints;
+ }
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "raw.csv";
- boolean hasHeader = true;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 1000;
-// List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
- List<Point> points = polyline.getVertices();
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "raw.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 1000;
+ // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
- // 写入每个点的数据
- for (Point point : points) {
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(points.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
-// int m = (int) Math.pow(points.size(), 0.5);
- int m = 10;
+ // int m = (int) Math.pow(points.size(), 0.5);
+ int m = 10;
-// long startTime = System.currentTimeMillis();
-// List<Point> sampled = dynamic_programming(points, m - 1, DP.ERROR.area, false);
-// long endTime = System.currentTimeMillis();
-// System.out.println("Time taken: " + (endTime - startTime) + "ms");
-// for (Point p : sampled) {
-// System.out.println(p);
-// }
-// System.out.println(sampled.size());
+ // long startTime = System.currentTimeMillis();
+ // List<Point> sampled = dynamic_programming(points, m - 1, DP.ERROR.area, false);
+ // long endTime = System.currentTimeMillis();
+ // System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ // System.out.println(sampled.size());
- long startTime2 = System.currentTimeMillis();
-// System.out.println(points.size() * m / (Math.pow(points.size() * 1.0 / (m - 1), 2 / 3.0)));
-// System.out.println(10000000 * (2 / (Math.pow(10000000 * 1.0 / (2 - 1), 2 / 3.0))));
- List<Point> sampled2 = reducePoints(points, m, true);
- long endTime2 = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime2 - startTime2) + "ms");
-// for (Point p : sampled2) {
-// System.out.println(p);
-// }
-// System.out.println(sampled2.size());
- }
+ long startTime2 = System.currentTimeMillis();
+ // System.out.println(points.size() * m / (Math.pow(points.size() * 1.0 / (m - 1), 2 /
+ // 3.0)));
+ // System.out.println(10000000 * (2 / (Math.pow(10000000 * 1.0 / (2 - 1), 2 / 3.0))));
+ List<Point> sampled2 = reducePoints(points, m, true);
+ long endTime2 = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime2 - startTime2) + "ms");
+ // for (Point p : sampled2) {
+ // System.out.println(p);
+ // }
+ // System.out.println(sampled2.size());
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java
index 62c81cd10ea..1cd10c19fcc 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java
@@ -10,284 +10,305 @@ import java.util.*;
public class DP { // Dynamic-Programming
- // Enum to represent error types
- public enum ERROR {
- L1, L2, L_infy, area
- }
-
- // precomputing error measures of all possible segments, e.g., ad2 table
- public static double[][] prepareCostTable(List<Point> points, ERROR errorType, boolean debug) {
- int N = points.size();
- double[][] dists = new double[N][N]; // 默认0，对角线元素已经都是0了
- // 一个 double[47700][47700] 矩阵大约需要 16.94 GB 的内存。
-
- // 外层循环i：range长度=right boundary-left boundary
- // 内层循环j: 矩阵逐行
- for (int i = 1; i < N; i++) { // TODO
- for (int j = 0; j < N - i; j++) {
- // j = left boundary, r = right boundary, 左闭右闭
- int r = i + j; // r<N
- if (debug) {
- System.out.println(">>>> i=" + i + ", j=" + j + ", r=" + r);
- }
+ // Enum to represent error types
+ public enum ERRORtype {
+ L1,
+ L2,
+ L_infy,
+ area
+ }
- // lx=j,rx=r, 从lx=j到rx=r（左闭右闭）的linear interpolation（即连接lx和rx点）的errorType类型的误差
- double mc = joint_segment_error(points, j, r, errorType);
- dists[j][r] = mc;
- if (debug) {
- System.out.println("dists=");
- Tool.printArray(dists);
- System.out.println("---------------------");
- }
- }
- }
- return dists;
- }
+ // precomputing error measures of all possible segments, e.g., ad2 table
+ public static double[][] prepareCostTable(
+ List<Point> points, ERRORtype errorType, boolean debug) {
+ int N = points.size();
+ double[][] dists = new double[N][N]; // 默认0，对角线元素已经都是0了
+ // 一个 double[47700][47700] 矩阵大约需要 16.94 GB 的内存。
- // 注意k是分段数，不是采点数
- public static List<Point> dynamic_programming(List<Point> points, int k, ERROR errorType, boolean debug) {
- int N = points.size();
-
- // 预计算dists（即ad2 table），dists[i][j]代表对子序列i:j左闭右闭直接连接pi和pj的近似误差
- double[][] dists = prepareCostTable(points, errorType, debug);
+ // 外层循环i：range长度=right boundary-left boundary
+ // 内层循环j: 矩阵逐行
+ for (int i = 1; i < N; i++) { // TODO
+ for (int j = 0; j < N - i; j++) {
+ // j = left boundary, r = right boundary, 左闭右闭
+ int r = i + j; // r<N
if (debug) {
- Tool.printArray(dists);
- System.out.println("----------------------------------");
+ System.out.println(">>>> i=" + i + ", j=" + j + ", r=" + r);
}
- // 创建dp table。kSegDist[i][j]代表把子序列T[0:j]左闭右闭分成i段(java从0开始计数)的最小误差
- double[][] kSegDist = new double[k][N];
- // Initialize the case k=1 directly from the pre-computed distances
- // 注意java从0开始，所以是0 index第一行代表分段数1
- System.arraycopy(dists[0], 0, kSegDist[0], 0, kSegDist[0].length);
+ // lx=j,rx=r, 从lx=j到rx=r（左闭右闭）的linear interpolation（即连接lx和rx点）的errorType类型的误差
+ double mc = joint_segment_error(points, j, r, errorType);
+ dists[j][r] = mc;
if (debug) {
- System.out.println("k_seg_dist=");
- Tool.printArray(kSegDist);
- }
-
- // 创建path table。记录子问题的解。kSegPath[i][j]代表把T[0:j]分成i段这个问题的最优子问题的位置
- // 子问题是把T[0:kSegPath[i][j]]分成i-1段，以及最后一段T[kSegPath[i][j]:end]
- int[][] kSegPath = new int[k][N];
- // 第一行只分一段，所以不管终点是哪个（列），左边的闭合起点都是0
- // 虽然java数组默认初始是0，但是这里还是显式赋值
- Arrays.fill(kSegPath[0], 0);
- if (debug) {
- System.out.println("k_seg_path=");
- Tool.printArray(kSegPath);
+ System.out.println("dists=");
+ Tool.printArray(dists);
+ System.out.println("---------------------");
}
+ }
+ }
+ return dists;
+ }
- // 外层循环i：分段数，注意python从0开始，所以实际含义是2~k个分段数
- for (int i = 1; i < k; i++) {
- // 内层循环j：从第一个点开始到j终点（闭合）的序列
- // 所以含义是：找到从第一个点开始到j终点（闭合）的序列的分成(i+1)段的最佳分段方案（误差最小）
- for (int j = 0; j < N; j++) {
- // 动态规划
- // 注意linear interpolation不需要单点成为一个分段的情况
- // TODO 误差类型
- // TODO ghost修复
- // 从0:j的序列分成i段的问题：遍历所有可能的子问题组合解
- if (debug) {
- System.out.println(">>>分段数i+1" + (i + 1) + ",end pos j=" + j);
- }
- double[] choices = new double[j + 1]; // java从0开始
- int bestIndex = -1;
- double bestVal = Double.MAX_VALUE; // 越小越好
- for (int xtmp = 0; xtmp < j + 1; xtmp++) {
- if (errorType == ERROR.L_infy) {
- choices[xtmp] = Math.max(kSegDist[i - 1][xtmp], dists[xtmp][j]);
- } else {
- choices[xtmp] = kSegDist[i - 1][xtmp] + dists[xtmp][j];
- }
- if (choices[xtmp] < bestVal) {
- bestVal = choices[xtmp];
- bestIndex = xtmp;
- }
- }
- if (debug) {
- for (int xtmp = 0; xtmp < j + 1; xtmp++) { // 遍历从 0 到 j 的每个元素
- if (errorType == ERROR.L_infy) {
- System.out.printf(" max((k_seg_dist[%d, %d] = %f), (dists[%d, %d] = %f)) --> %f%n",
- i - 1, xtmp, kSegDist[i - 1][xtmp],
- xtmp, j, dists[xtmp][j],
- Math.max(kSegDist[i - 1][xtmp], dists[xtmp][j]));
- } else {
- System.out.printf(" (k_seg_dist[%d, %d] = %f) + (dists[%d, %d] = %f) --> %f%n",
- i - 1, xtmp, kSegDist[i - 1][xtmp],
- xtmp, j, dists[xtmp][j],
- kSegDist[i - 1][xtmp] + dists[xtmp][j]);
- }
- }
- }
+ // 注意k是分段数，不是采点数
+ public static List<Point> dynamic_programming(
+ List<Point> points, int k, ERRORtype errorType, boolean debug) {
+ int N = points.size();
- // Store the sub-problem solution
- kSegPath[i][j] = bestIndex;
- kSegDist[i][j] = bestVal;
+ // 预计算dists（即ad2 table），dists[i][j]代表对子序列i:j左闭右闭直接连接pi和pj的近似误差
+ double[][] dists = prepareCostTable(points, errorType, debug);
+ if (debug) {
+ Tool.printArray(dists);
+ System.out.println("----------------------------------");
+ }
- if (debug) {
- System.out.println("kSegDist[" + i + "][" + j + "] = " + bestVal);
- System.out.println("kSegDist=");
- Tool.printArray(kSegDist);
- System.out.println("kSegPath=");
- Tool.printArray(kSegPath);
- }
- }
- }
+ // 创建dp table。kSegDist[i][j]代表把子序列T[0:j]左闭右闭分成i段(java从0开始计数)的最小误差
+ double[][] kSegDist = new double[k][N];
+ // Initialize the case k=1 directly from the pre-computed distances
+ // 注意java从0开始，所以是0 index第一行代表分段数1
+ System.arraycopy(dists[0], 0, kSegDist[0], 0, kSegDist[0].length);
+ if (debug) {
+ System.out.println("k_seg_dist=");
+ Tool.printArray(kSegDist);
+ }
- // 开始回溯构建采样结果
- List<Point> sDs = new ArrayList<>(); // k+1个采样点
- List<Integer> sDs_idx = new ArrayList<>();
- int rhs = N - 1;
- sDs.add(points.get(rhs)); // 全局尾点
- sDs_idx.add(rhs);
+ // 创建path table。记录子问题的解。kSegPath[i][j]代表把T[0:j]分成i段这个问题的最优子问题的位置
+ // 子问题是把T[0:kSegPath[i][j]]分成i-1段，以及最后一段T[kSegPath[i][j]:end]
+ int[][] kSegPath = new int[k][N];
+ // 第一行只分一段，所以不管终点是哪个（列），左边的闭合起点都是0
+ // 虽然java数组默认初始是0，但是这里还是显式赋值
+ Arrays.fill(kSegPath[0], 0);
+ if (debug) {
+ System.out.println("k_seg_path=");
+ Tool.printArray(kSegPath);
+ }
- for (int i = k - 1; i >= 0; i--) {
- int lhs = kSegPath[i][rhs];
- sDs.add(points.get(lhs));
- sDs_idx.add(lhs);
- rhs = lhs;
+ // 外层循环i：分段数，注意python从0开始，所以实际含义是2~k个分段数
+ for (int i = 1; i < k; i++) {
+ // 内层循环j：从第一个点开始到j终点（闭合）的序列
+ // 所以含义是：找到从第一个点开始到j终点（闭合）的序列的分成(i+1)段的最佳分段方案（误差最小）
+ for (int j = 0; j < N; j++) {
+ // 动态规划
+ // 注意linear interpolation不需要单点成为一个分段的情况
+ // TODO 误差类型
+ // TODO ghost修复
+ // 从0:j的序列分成i段的问题：遍历所有可能的子问题组合解
+ if (debug) {
+ System.out.println(">>>分段数i+1" + (i + 1) + ",end pos j=" + j);
+ }
+ double[] choices = new double[j + 1]; // java从0开始
+ int bestIndex = -1;
+ double bestVal = Double.MAX_VALUE; // 越小越好
+ for (int xtmp = 0; xtmp < j + 1; xtmp++) {
+ if (errorType == ERRORtype.L_infy) {
+ choices[xtmp] = Math.max(kSegDist[i - 1][xtmp], dists[xtmp][j]);
+ } else {
+ choices[xtmp] = kSegDist[i - 1][xtmp] + dists[xtmp][j];
+ }
+ if (choices[xtmp] < bestVal) {
+ bestVal = choices[xtmp];
+ bestIndex = xtmp;
+ }
+ }
+ if (debug) {
+ for (int xtmp = 0; xtmp < j + 1; xtmp++) { // 遍历从 0 到 j 的每个元素
+ if (errorType == ERRORtype.L_infy) {
+ System.out.printf(
+ " max((k_seg_dist[%d, %d] = %f), (dists[%d, %d] = %f)) --> %f%n",
+ i - 1,
+ xtmp,
+ kSegDist[i - 1][xtmp],
+ xtmp,
+ j,
+ dists[xtmp][j],
+ Math.max(kSegDist[i - 1][xtmp], dists[xtmp][j]));
+ } else {
+ System.out.printf(
+ " (k_seg_dist[%d, %d] = %f) + (dists[%d, %d] = %f) --> %f%n",
+ i - 1,
+ xtmp,
+ kSegDist[i - 1][xtmp],
+ xtmp,
+ j,
+ dists[xtmp][j],
+ kSegDist[i - 1][xtmp] + dists[xtmp][j]);
+ }
+ }
}
- // 反转列表
- Collections.reverse(sDs);
- Collections.reverse(sDs_idx);
+ // Store the sub-problem solution
+ kSegPath[i][j] = bestIndex;
+ kSegDist[i][j] = bestVal;
if (debug) {
- System.out.println(sDs);
- System.out.println(">>>>>ad2[][]=");
- Tool.printArray(dists);
- System.out.println(">>>>>dp[][]=");
- Tool.printTransposeArray(kSegDist);
- System.out.println(">>>>>path[][]=");
- Tool.printTransposeArray(kSegPath);
- System.out.println(error(points, sDs_idx, errorType));
+ System.out.println("kSegDist[" + i + "][" + j + "] = " + bestVal);
+ System.out.println("kSegDist=");
+ Tool.printArray(kSegDist);
+ System.out.println("kSegPath=");
+ Tool.printArray(kSegPath);
}
+ }
+ }
+
+ // 开始回溯构建采样结果
+ List<Point> sDs = new ArrayList<>(); // k+1个采样点
+ List<Integer> sDs_idx = new ArrayList<>();
+ int rhs = N - 1;
+ sDs.add(points.get(rhs)); // 全局尾点
+ sDs_idx.add(rhs);
- return sDs;
+ for (int i = k - 1; i >= 0; i--) {
+ int lhs = kSegPath[i][rhs];
+ sDs.add(points.get(lhs));
+ sDs_idx.add(lhs);
+ rhs = lhs;
}
-// // Helper method to get index of minimum value
-// private static int getIndexOfMin(double[] array) {
-// double minVal = array[0];
-// int minIndex = 0;
-// for (int i = 1; i < array.length; i++) {
-// if (array[i] < minVal) {
-// minVal = array[i];
-// minIndex = i;
-// }
-// }
-// return minIndex;
-// }
+ // 反转列表
+ Collections.reverse(sDs);
+ Collections.reverse(sDs_idx);
- // Method to calculate joint segment error based on error type
- private static double joint_segment_error(List<Point> points, int lx, int rx, ERROR errorType) {
- // 默认joint=true, residual=true，即使用linear interpolation近似分段
- // lx~rx 左闭右闭
- if (lx == rx) {
- return 0;
- }
+ if (debug) {
+ System.out.println(sDs);
+ System.out.println(">>>>>ad2[][]=");
+ Tool.printArray(dists);
+ System.out.println(">>>>>dp[][]=");
+ Tool.printTransposeArray(kSegDist);
+ System.out.println(">>>>>path[][]=");
+ Tool.printTransposeArray(kSegPath);
+ System.out.println(error(points, sDs_idx, errorType));
+ }
- if (errorType != ERROR.area) {
- double x1 = points.get(lx).x;
- double y1 = points.get(lx).y;
- double x2 = points.get(rx).x;
- double y2 = points.get(rx).y;
+ return sDs;
+ }
- // linear interpolation:
- double k = (y2 - y1) / (x2 - x1);
- double b = (y1 * x2 - y2 * x1) / (x2 - x1);
+ // // Helper method to get index of minimum value
+ // private static int getIndexOfMin(double[] array) {
+ // double minVal = array[0];
+ // int minIndex = 0;
+ // for (int i = 1; i < array.length; i++) {
+ // if (array[i] < minVal) {
+ // minVal = array[i];
+ // minIndex = i;
+ // }
+ // }
+ // return minIndex;
+ // }
- double tmp = 0;
- if (errorType == ERROR.L2) {
- for (int i = lx; i <= rx; i++) {
- double e = (k * points.get(i).x + b - points.get(i).y) * (k * points.get(i).x + b - points.get(i).y);
- tmp += e;
- }
- } else if (errorType == ERROR.L1) {
- for (int i = lx; i <= rx; i++) {
- double e = Math.abs(k * points.get(i).x + b - points.get(i).y);
- tmp += e;
- }
- } else if (errorType == ERROR.L_infy) {
- for (int i = lx; i <= rx; i++) {
- double e = Math.abs(k * points.get(i).x + b - points.get(i).y); // 注意绝对值
- if (e > tmp) {
- tmp = e;
- }
- }
- }
- return tmp;
- } else { // AD
- List<Point> linearInterpolation = new ArrayList<>();
- linearInterpolation.add(points.get(lx));
- linearInterpolation.add(points.get(rx));
- return Tool.total_areal_displacement(points.subList(lx, rx + 1), // 注意lx,rx左闭右闭
- linearInterpolation, false);
- }
+ // Method to calculate joint segment error based on error type
+ public static double joint_segment_error(
+ List<Point> points, int lx, int rx, ERRORtype errorType) {
+ // 默认joint=true, residual=true，即使用linear interpolation近似分段
+ // lx~rx 左闭右闭
+ if (lx == rx) {
+ return 0;
}
- public static double error(List<Point> points, List<Integer> sampledIdx, ERROR errorType) {
- double res = 0;
- for (int i = 0; i < sampledIdx.size() - 1; i++) {
- int lx = sampledIdx.get(i);
- int rx = sampledIdx.get(i + 1);
- double e = joint_segment_error(points, lx, rx, errorType);
- if (errorType == ERROR.L_infy) {
- res = Math.max(res, e);
- } else {
- res += e;
- }
+ if (errorType != ERRORtype.area) {
+ double x1 = points.get(lx).x;
+ double y1 = points.get(lx).y;
+ double x2 = points.get(rx).x;
+ double y2 = points.get(rx).y;
+
+ // linear interpolation:
+ double k = (y2 - y1) / (x2 - x1);
+ double b = (y1 * x2 - y2 * x1) / (x2 - x1);
+
+ double tmp = 0;
+ if (errorType == ERRORtype.L2) {
+ for (int i = lx; i <= rx; i++) {
+ double e =
+ (k * points.get(i).x + b - points.get(i).y)
+ * (k * points.get(i).x + b - points.get(i).y);
+ tmp += e;
}
- return res;
+ } else if (errorType == ERRORtype.L1) {
+ for (int i = lx; i <= rx; i++) {
+ double e = Math.abs(k * points.get(i).x + b - points.get(i).y);
+ tmp += e;
+ }
+ } else if (errorType == ERRORtype.L_infy) {
+ for (int i = lx; i <= rx; i++) {
+ double e = Math.abs(k * points.get(i).x + b - points.get(i).y); // 注意绝对值
+ if (e > tmp) {
+ tmp = e;
+ }
+ }
+ }
+ return tmp;
+ } else { // AD
+ List<Point> linearInterpolation = new ArrayList<>();
+ linearInterpolation.add(points.get(lx));
+ linearInterpolation.add(points.get(rx));
+ return Tool.total_areal_displacement(
+ points.subList(lx, rx + 1), // 注意lx,rx左闭右闭
+ linearInterpolation,
+ false);
}
+ }
- // Example usage
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "D:\\LabSync\\iotdb\\我的Gitbook基地\\RUI Lei gitbook\\士论\\49. visval改进\\notebook\\raw.csv";
- boolean hasHeader = true;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 100;
-// List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
- List<Point> points = polyline.getVertices();
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+ public static double error(List<Point> points, List<Integer> sampledIdx, ERRORtype errorType) {
+ double res = 0;
+ for (int i = 0; i < sampledIdx.size() - 1; i++) {
+ int lx = sampledIdx.get(i);
+ int rx = sampledIdx.get(i + 1);
+ double e = joint_segment_error(points, lx, rx, errorType);
+ if (errorType == ERRORtype.L_infy) {
+ res = Math.max(res, e);
+ } else {
+ res += e;
+ }
+ }
+ return res;
+ }
- // 写入每个点的数据
- for (Point point : points) {
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(points.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ // Example usage
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input =
+ "D:\\LabSync\\iotdb\\我的Gitbook基地\\RUI Lei gitbook\\士论\\49. visval改进\\notebook\\raw.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 100;
+ // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
- long startTime = System.currentTimeMillis();
-// double[][] ad2 = prepareKSegments(points, ERROR.L1, false);
- int m = 10;
- List<Point> sampled = dynamic_programming(points, m - 1, ERROR.area, false);
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
- for (Point p : sampled) {
- System.out.println(p);
- }
- System.out.println(sampled.size());
+ long startTime = System.currentTimeMillis();
+ // double[][] ad2 = prepareKSegments(points, ERROR.L1, false);
+ int m = 10;
+ List<Point> sampled = dynamic_programming(points, m - 1, ERRORtype.area, false);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
-// try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
-// // 写入字符串
-// for (int i = 0; i < sampled.size(); i++) {
-// writer.println(sampled.get(i).x + "," + sampled.get(i).y);
-// }
-//
-// } catch (FileNotFoundException e) {
-// e.printStackTrace();
-// }
+ for (Point p : sampled) {
+ System.out.println(p);
}
+ System.out.println(sampled.size());
+
+ // try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // // 写入字符串
+ // for (int i = 0; i < sampled.size(); i++) {
+ // writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ // }
+ //
+ // } catch (FileNotFoundException e) {
+ // e.printStackTrace();
+ // }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
index f6d5e2fc6a3..5c29355fec1 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
@@ -1,6 +1,6 @@
package org.apache.iotdb.db.query.eBUG;
-import java.io.*;
+import java.io.IOException;
import java.util.*;
public class SWAB {
@@ -593,242 +593,194 @@ public class SWAB {
return segTs;
}
- public static List<Point> seg_bottomUp_m_withTimestamps(
- List<Point> points, int m, Object[] prefixSum, boolean debug) throws IOException {
- if (m <= 2) {
- throw new IOException("please make m>2");
- }
- // 直接控制采样点数
- int total = points.size();
- if (total < 3) {
- return points; // 不足 3 个点无法形成三角形
- }
-
- // 每个triangle对应两个相邻的待合并的分段
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- points.get(0).prev = null;
- points.get(0).next = points.get(1);
- // points.get(0).index = 0;
- points.get(total - 1).next = null;
- points.get(total - 1).prev = points.get(total - 2);
- // points.get(total - 1).index = points.size() - 1;
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- double mc;
- if (prefixSum == null) {
- mc = myLA_withTimestamps(points, index1, index3);
- } else {
- mc = myLA_withTimestamps(points, prefixSum, index1, index3);
- }
- triangles[i - 1] = new Triangle(index1, index2, index3, mc);
-
- // 初始化点的状态 用于最后在线采样结果顺序输出未淘汰点
- points.get(i).prev = points.get(i - 1);
- points.get(i).next = points.get(i + 1);
- // points.get(i).index = i; // for swab usage
- }
-
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
- }
-
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap =
- new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
-
- int remainNonTerminalPoint = m - 2;
- int fakeCnt = 0;
- // while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
- while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll(); // O(logn)
-
- // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
- // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
- if (tri.isDeleted) {
- fakeCnt--; // 取出了一个被标记删除点
- if (debug) {
- System.out.println(
- ">>>bottom-up, remaining "
- + triangleHeap.size()
- + " triangles (including those marked for removal)");
- }
- continue;
- }
-
- // 真正的淘汰点
- // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
- if (debug) {
- System.out.println("(1) eliminate " + points.get(tri.indices[1]));
- }
- points.get(tri.indices[1]).markEliminated();
-
- // 更新相邻三角形
- if (tri.prev != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-
- Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
- // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
-
- // 2. 处理pi被淘汰引起tri.prev被更新的事情
- // 前一个三角形连到后一个三角形
- tri.prev.next =
- tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
- tri.prev.indices[2] = tri.indices[2];
-
- double mc;
- if (prefixSum == null) {
- mc = myLA_withTimestamps(points, tri.prev.indices[0], tri.prev.indices[2]);
- } else {
- mc = myLA_withTimestamps(points, prefixSum, tri.prev.indices[0], tri.prev.indices[2]);
- }
- tri.prev.area = mc;
- if (debug) {
- System.out.println(
- "(2) updating point on the left "
- + points.get(tri.prev.indices[1])
- + ", ranging ["
- + tri.prev.indices[0]
- + ","
- + tri.prev.indices[2]
- + "]");
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add来显式重新插入修改后的元素
- triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
-
- if (tri.next != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.next被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
- tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
-
- Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
- // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
-
- if (tri.prev != null) {
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
- }
-
- // 2. 处理pi被淘汰引起tri.next被更新的事情
- tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
- tri.next.indices[0] = tri.indices[0];
-
- double mc;
- if (prefixSum == null) {
- mc = myLA_withTimestamps(points, tri.next.indices[0], tri.next.indices[2]);
- } else {
- mc = myLA_withTimestamps(points, prefixSum, tri.next.indices[0], tri.next.indices[2]);
- }
- tri.next.area = mc;
- if (debug) {
- System.out.println(
- "(3) updating point on the right "
- + points.get(tri.next.indices[1])
- + ", ranging ["
- + tri.next.indices[0]
- + ","
- + tri.next.indices[2]
- + "]");
- }
+ // public static List<Point> seg_bottomUp_m_withTimestamps(
+ // List<Point> points, int m, Object[] prefixSum, boolean debug) throws IOException {
+ // if (m <= 2) {
+ // throw new IOException("please make m>2");
+ // }
+ // // 直接控制采样点数
+ // int total = points.size();
+ // if (total < 3) {
+ // return points; // 不足 3 个点无法形成三角形
+ // }
+ //
+ // // 每个triangle对应两个相邻的待合并的分段
+ // int nTriangles = total - 2;
+ // Triangle[] triangles = new Triangle[nTriangles];
+ //
+ // // 创建所有三角形并计算初始面积
+ // points.get(0).prev = null;
+ // points.get(0).next = points.get(1);
+ // // points.get(0).index = 0;
+ // points.get(total - 1).next = null;
+ // points.get(total - 1).prev = points.get(total - 2);
+ // // points.get(total - 1).index = points.size() - 1;
+ // for (int i = 1; i < total - 1; i++) {
+ // int index1 = i - 1, index2 = i, index3 = i + 1;
+ // double mc;
+ // if (prefixSum == null) {
+ // mc = myLA_withTimestamps(points, index1, index3);
+ // } else {
+ // mc = myLA_withTimestamps(points, prefixSum, index1, index3);
+ // }
+ // triangles[i - 1] = new Triangle(index1, index2, index3, mc);
+ //
+ // // 初始化点的状态 用于最后在线采样结果顺序输出未淘汰点
+ // points.get(i).prev = points.get(i - 1);
+ // points.get(i).next = points.get(i + 1);
+ // // points.get(i).index = i; // for swab usage
+ // }
+ //
+ // // 设置三角形的前后关系
+ // for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ // triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ // triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ // }
+ //
+ // // 使用优先队列构建 minHeap
+ // PriorityQueue<Triangle> triangleHeap =
+ // new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ // Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+ //
+ // int remainNonTerminalPoint = m - 2;
+ // int fakeCnt = 0;
+ // // while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { //
+ // TODO
+ // while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
+ // // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ // Triangle tri = triangleHeap.poll(); // O(logn)
+ //
+ // // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ // if (tri.isDeleted) {
+ // fakeCnt--; // 取出了一个被标记删除点
+ // if (debug) {
+ // System.out.println(
+ // ">>>bottom-up, remaining "
+ // + triangleHeap.size()
+ // + " triangles (including those marked for removal)");
+ // }
+ // continue;
+ // }
+ //
+ // // 真正的淘汰点
+ // // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ // if (debug) {
+ // System.out.println("(1) eliminate " + points.get(tri.indices[1]));
+ // }
+ // points.get(tri.indices[1]).markEliminated();
+ //
+ // // 更新相邻三角形
+ // if (tri.prev != null) {
+ // // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+ //
+ // // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n)
+ // complexity!
+ // tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap
+ // poll到它就跳过就可以
+ //
+ // Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ // // tri.prev = newPre; // can omit, because already done by new
+ // Triangle(tri.prev)
+ //
+ // // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // // 前一个三角形连到后一个三角形
+ // tri.prev.next =
+ // tri.next; // ATTENTION!!!:
+ // 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ // tri.prev.indices[2] = tri.indices[2];
+ //
+ // double mc;
+ // if (prefixSum == null) {
+ // mc = myLA_withTimestamps(points, tri.prev.indices[0], tri.prev.indices[2]);
+ // } else {
+ // mc = myLA_withTimestamps(points, prefixSum, tri.prev.indices[0],
+ // tri.prev.indices[2]);
+ // }
+ // tri.prev.area = mc;
+ // if (debug) {
+ // System.out.println(
+ // "(2) updating point on the left "
+ // + points.get(tri.prev.indices[1])
+ // + ", ranging ["
+ // + tri.prev.indices[0]
+ // + ","
+ // + tri.prev.indices[2]
+ // + "]");
+ // }
+ //
+ // // 重新加入堆
+ // // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // // 所以必须通过add来显式重新插入修改后的元素
+ // triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ // fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ // }
+ //
+ // if (tri.next != null) {
+ // // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+ //
+ // // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n)
+ // complexity
+ // tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+ //
+ // Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ // // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
+ //
+ // if (tri.prev != null) {
+ // tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
+ // }
+ //
+ // // 2. 处理pi被淘汰引起tri.next被更新的事情
+ // tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ // tri.next.indices[0] = tri.indices[0];
+ //
+ // double mc;
+ // if (prefixSum == null) {
+ // mc = myLA_withTimestamps(points, tri.next.indices[0], tri.next.indices[2]);
+ // } else {
+ // mc = myLA_withTimestamps(points, prefixSum, tri.next.indices[0],
+ // tri.next.indices[2]);
+ // }
+ // tri.next.area = mc;
+ // if (debug) {
+ // System.out.println(
+ // "(3) updating point on the right "
+ // + points.get(tri.next.indices[1])
+ // + ", ranging ["
+ // + tri.next.indices[0]
+ // + ","
+ // + tri.next.indices[2]
+ // + "]");
+ // }
+ //
+ // // 重新加入堆
+ // // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // // 所以必须通过add 来显式重新插入修改后的元素
+ // triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ // fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ // }
+ // if (debug) {
+ // System.out.println(
+ // ">>>bottom-up, remaining "
+ // + triangleHeap.size()
+ // + " triangles (including those marked for removal)");
+ // }
+ // }
+ //
+ // List<Point> onlineSampled = new ArrayList<>();
+ // Point start = points.get(0);
+ // Point end = points.get(points.size() - 1);
+ // while (start
+ // != end) { //
+ // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ // onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ // start = start.next; // when e=0, only traversing three points pa pi pb
+ // }
+ // onlineSampled.add(end);
+ // return onlineSampled;
+ // }
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
- triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
- if (debug) {
- System.out.println(
- ">>>bottom-up, remaining "
- + triangleHeap.size()
- + " triangles (including those marked for removal)");
- }
- }
-
- List<Point> onlineSampled = new ArrayList<>();
- Point start = points.get(0);
- Point end = points.get(points.size() - 1);
- while (start
- != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
- start = start.next; // when e=0, only traversing three points pa pi pb
- }
- onlineSampled.add(end);
- return onlineSampled;
- }
-
- public static void main(String[] args) throws IOException {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = false;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 2000;
- // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
- List<Point> points = polyline.getVertices();
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < points.size(); i++) {
- Point point = points.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(points.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
-
- // long startTime = System.currentTimeMillis();
- Object[] prefixSum = prefixSum(polyline.getVertices());
- // long endTime = System.currentTimeMillis();
- // System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) +
- // "ms");
-
- int m = 100;
-
- // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
- long startTime = System.currentTimeMillis();
- List<Point> sampled = seg_bottomUp_m_withTimestamps(points, m, null, false);
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime - startTime) + "ms");
-
- // for (Point p : sampled) {
- // System.out.println(p);
- // }
- System.out.println(sampled.size());
-
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < sampled.size(); i++) {
- writer.println(sampled.get(i).x + "," + sampled.get(i).y);
- }
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test10.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test10.java
index 4dba96d0e3e..31c1cff7352 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test10.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test10.java
@@ -8,55 +8,54 @@ import java.util.Random;
import static org.apache.iotdb.db.query.eBUG.DP.dynamic_programming;
public class Test10 {
- // 用于验证java DP正确性
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "raw.csv";
- boolean hasHeader = true;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 100;
-// List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
- List<Point> points = polyline.getVertices();
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (Point point : points) {
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(points.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
-
-
- long startTime = System.currentTimeMillis();
-// double[][] ad2 = prepareKSegments(points, ERROR.L1, false);
- int m = 10;
- List<Point> sampled = dynamic_programming(points, m - 1, DP.ERROR.area, false);
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ // 用于验证java DP正确性
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "raw.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 100;
+ // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
- for (Point p : sampled) {
- System.out.println(p);
- }
- System.out.println(sampled.size());
+ long startTime = System.currentTimeMillis();
+ // double[][] ad2 = prepareKSegments(points, ERROR.L1, false);
+ int m = 10;
+ List<Point> sampled = dynamic_programming(points, m - 1, DP.ERRORtype.area, false);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
-// try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
-// // 写入字符串
-// for (int i = 0; i < sampled.size(); i++) {
-// writer.println(sampled.get(i).x + "," + sampled.get(i).y);
-// }
-//
-// } catch (FileNotFoundException e) {
-// e.printStackTrace();
-// }
+ for (Point p : sampled) {
+ System.out.println(p);
}
+ System.out.println(sampled.size());
+
+ // try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // // 写入字符串
+ // for (int i = 0; i < sampled.size(); i++) {
+ // writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ // }
+ //
+ // } catch (FileNotFoundException e) {
+ // e.printStackTrace();
+ // }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
index 9a54f835ec3..f2e2c0d68d9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
@@ -12,406 +12,406 @@ import java.util.stream.Stream;
public class Tool {
- public static void printArray(double[][] array) {
- // 遍历每一行
- for (double[] row : array) {
- for (double value : row) {
- // 格式化输出：%10.2f 表示右对齐，占10个字符，保留2位小数
- System.out.printf("%10.2f ", value);
- }
- System.out.println(); // 换行
- }
+ public static void printArray(double[][] array) {
+ // 遍历每一行
+ for (double[] row : array) {
+ for (double value : row) {
+ // 格式化输出：%10.2f 表示右对齐，占10个字符，保留2位小数
+ System.out.printf("%10.2f ", value);
+ }
+ System.out.println(); // 换行
}
-
- public static void printTransposeArray(double[][] array) {
- // 遍历每一列
- for (int i = 0; i < array[0].length; i++) {
- for (int j = 0; j < array.length; j++) {
- // 格式化输出：%10.2f 表示右对齐，占10个字符，保留2位小数
- System.out.printf("%10.2f ", array[j][i]);
- }
- System.out.println(); // 换行
- }
+ }
+
+ public static void printTransposeArray(double[][] array) {
+ // 遍历每一列
+ for (int i = 0; i < array[0].length; i++) {
+ for (int j = 0; j < array.length; j++) {
+ // 格式化输出：%10.2f 表示右对齐，占10个字符，保留2位小数
+ System.out.printf("%10.2f ", array[j][i]);
+ }
+ System.out.println(); // 换行
}
-
- public static void printArray(int[][] array) {
- // 遍历每一行
- for (int[] row : array) {
- for (int value : row) {
- System.out.printf("%10d ", value);
- }
- System.out.println(); // 换行
- }
+ }
+
+ public static void printArray(int[][] array) {
+ // 遍历每一行
+ for (int[] row : array) {
+ for (int value : row) {
+ System.out.printf("%10d ", value);
+ }
+ System.out.println(); // 换行
}
-
- public static void printTransposeArray(int[][] array) {
- // 遍历每一列
- for (int i = 0; i < array[0].length; i++) {
- for (int j = 0; j < array.length; j++) {
- // 格式化输出：%10.2f 表示右对齐，占10个字符，保留2位小数
- System.out.printf("%10d ", array[j][i]);
- }
- System.out.println(); // 换行
- }
+ }
+
+ public static void printTransposeArray(int[][] array) {
+ // 遍历每一列
+ for (int i = 0; i < array[0].length; i++) {
+ for (int j = 0; j < array.length; j++) {
+ // 格式化输出：%10.2f 表示右对齐，占10个字符，保留2位小数
+ System.out.printf("%10d ", array[j][i]);
+ }
+ System.out.println(); // 换行
}
-
- // Assumed interface for the simplification function
- interface SimplifyFunction {
- List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs)
- throws IOException;
+ }
+
+ // Assumed interface for the simplification function
+ interface SimplifyFunction {
+ List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs)
+ throws IOException;
+ }
+
+ // Method to generate the output file name based on input path
+ public static String generateOutputFileName(
+ String inputFilePath, String outDir, String customize) {
+ // Get the file name without extension
+ Path path = Paths.get(inputFilePath);
+ String fileNameWithoutExtension = path.getFileName().toString();
+ String name = fileNameWithoutExtension.substring(0, fileNameWithoutExtension.lastIndexOf('.'));
+
+ // Get the file extension
+ String extension =
+ path.getFileName().toString().substring(fileNameWithoutExtension.lastIndexOf('.'));
+
+ // Create output file path by appending '-ds' before the extension
+ // String outputFile = name + "-ds-e" + e + extension;
+ String outputFile = name + customize + extension;
+
+ if (outDir == null) { // 表示使用input文件所在的文件夹
+ // Handle path compatibility for different operating systems (Linux/Windows)
+ return path.getParent().resolve(outputFile).toString();
+ } else {
+ Path out = Paths.get(outDir);
+ return out.resolve(outputFile).toString();
}
-
- // Method to generate the output file name based on input path
- public static String generateOutputFileName(
- String inputFilePath, String outDir, String customize) {
- // Get the file name without extension
- Path path = Paths.get(inputFilePath);
- String fileNameWithoutExtension = path.getFileName().toString();
- String name = fileNameWithoutExtension.substring(0, fileNameWithoutExtension.lastIndexOf('.'));
-
- // Get the file extension
- String extension =
- path.getFileName().toString().substring(fileNameWithoutExtension.lastIndexOf('.'));
-
- // Create output file path by appending '-ds' before the extension
- // String outputFile = name + "-ds-e" + e + extension;
- String outputFile = name + customize + extension;
-
- if (outDir == null) { // 表示使用input文件所在的文件夹
- // Handle path compatibility for different operating systems (Linux/Windows)
- return path.getParent().resolve(outputFile).toString();
- } else {
- Path out = Paths.get(outDir);
- return out.resolve(outputFile).toString();
+ }
+
+ public static double getParam(
+ List<Point> points,
+ int m,
+ SimplifyFunction simplifyFunc,
+ double tolerantPts,
+ Object... kwargs)
+ throws IOException {
+ // double x = 1; 不要从1开始，因为对于swab来说使用如此小的maxError会很慢，宁可让maxError从大到小
+ double x = 10;
+ boolean directLess = false;
+ boolean directMore = false;
+
+ int iterNum = 0;
+ int maxIterNum = 50;
+
+ // First binary search loop to find the initial range
+ double base = 2;
+ while (true) {
+ // System.out.println("x=" + x);
+ iterNum++;
+ if (iterNum > maxIterNum) {
+ // System.out.println("reach maxIterNum1");
+ break; // Avoid infinite loops for special cases
+ }
+ List<Point> res = simplifyFunc.reducePoints(points, x, kwargs);
+ int length = res.size();
+ // System.out.println(length);
+
+ if (Math.abs(length - m) <= tolerantPts) {
+ return x; // Return the found parameter
+ }
+
+ if (length > m) {
+ if (directMore) {
+ break; // Reach the first more
}
- }
-
- public static double getParam(
- List<Point> points,
- int m,
- SimplifyFunction simplifyFunc,
- double tolerantPts,
- Object... kwargs)
- throws IOException {
- // double x = 1; 不要从1开始，因为对于swab来说使用如此小的maxError会很慢，宁可让maxError从大到小
- double x = 10;
- boolean directLess = false;
- boolean directMore = false;
-
- int iterNum = 0;
- int maxIterNum = 50;
-
- // First binary search loop to find the initial range
- double base = 2;
- while (true) {
- // System.out.println("x=" + x);
- iterNum++;
- if (iterNum > maxIterNum) {
- // System.out.println("reach maxIterNum1");
- break; // Avoid infinite loops for special cases
- }
- List<Point> res = simplifyFunc.reducePoints(points, x, kwargs);
- int length = res.size();
- // System.out.println(length);
-
- if (Math.abs(length - m) <= tolerantPts) {
- return x; // Return the found parameter
- }
-
- if (length > m) {
- if (directMore) {
- break; // Reach the first more
- }
- if (!directLess) {
- directLess = true;
- }
-
- x *= base; // Need less points, increase x
- } else {
- if (directLess) {
- break; // Reach the first less
- }
- if (!directMore) {
- directMore = true;
- }
-
- x /= base; // Need more points, decrease x
- }
+ if (!directLess) {
+ directLess = true;
}
- // Determine the initial left and right bounds
- double left = (directLess) ? x / base : x;
- double right = (directMore) ? x * base : x;
-
- // Refine the range with binary search
- iterNum = 0;
- while (true) {
- iterNum++;
- if (iterNum > maxIterNum) {
- // System.out.println("reach maxIterNum");
- break; // Avoid infinite loops
- }
-
- double mid = (left + right) / 2;
-
- List<Point> res = simplifyFunc.reducePoints(points, mid, kwargs);
- int length = res.size();
- // System.out.println(length);
-
- if (Math.abs(length - m) <= tolerantPts) {
- return mid; // Return the refined parameter
- }
-
- if (length > m) {
- left = mid; // Need less, narrow range to the left
- } else {
- right = mid; // Need more, narrow range to the right
- }
- // System.out.println(left + "," + right);
+ x *= base; // Need less points, increase x
+ } else {
+ if (directLess) {
+ break; // Reach the first less
}
-
- return (left + right) / 2; // Return the average of left and right as the final parameter
- }
-
- public static List<Point> readFromFile(
- String input, boolean hasHeader, int timeIdx, int valueIdx, int N) {
- List<Point> res = new ArrayList<>();
-
- // Using Files.lines() for memory-efficient line-by-line reading
- try (Stream<String> lines = Files.lines(Paths.get(input))) {
- Iterator<String> iterator = lines.iterator();
- int lineNumber = 0;
-
- // Skip the header line if necessary
- if (hasHeader && iterator.hasNext()) {
- iterator.next(); // Skip the header
- }
-
- // Process each line
- while (iterator.hasNext()) {
- lineNumber++;
- String line = iterator.next();
- String[] columns = line.split(",");
-
- if (columns.length > Math.max(timeIdx, valueIdx)) {
- double time;
- if (timeIdx < 0) {
- time = lineNumber;
- } else {
- time = Double.parseDouble(columns[timeIdx]);
- }
- double value = Double.parseDouble(columns[valueIdx]);
- res.add(new Point(time, value));
- // System.out.println("Line " + lineNumber + " - Time: " + time + ",
- // Value: " + value);
- } else {
- System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
- }
- if (N > 0 && lineNumber >= N) {
- // N>0控制点数，否则N<=0表示读全部数据
- break;
- }
- }
- } catch (IOException e) {
- e.printStackTrace();
+ if (!directMore) {
+ directMore = true;
}
- return res;
- }
-
- // 计算三角形面积
- public static double triArea(Point d0, Point d1, Point d2) {
- double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
- return (dArea > 0.0) ? dArea : -dArea; // abs
- }
- // 计算简单多边形（边不交叉）的面积（鞋带公式）
- public static double calculatePolygonArea(List<Point> points) {
- // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
- int n = points.size();
- double area = 0;
- for (int i = 0; i < n; i++) {
- int next = (i + 1) % n;
- area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
- }
- return Math.abs(area) / 2.0;
+ x /= base; // Need more points, decrease x
+ }
}
- // 计算两个向量的叉积
- public static double crossProduct(double x1, double y1, double x2, double y2) {
- // >0: (x2,y2)在(x1,y1)的逆时针方向
- // <0: (x2,y2)在(x1,y1)的顺时针方向
- // =0: 平行或共线
- return x1 * y2 - y1 * x2;
+ // Determine the initial left and right bounds
+ double left = (directLess) ? x / base : x;
+ double right = (directMore) ? x * base : x;
+
+ // Refine the range with binary search
+ iterNum = 0;
+ while (true) {
+ iterNum++;
+ if (iterNum > maxIterNum) {
+ // System.out.println("reach maxIterNum");
+ break; // Avoid infinite loops
+ }
+
+ double mid = (left + right) / 2;
+
+ List<Point> res = simplifyFunc.reducePoints(points, mid, kwargs);
+ int length = res.size();
+ // System.out.println(length);
+
+ if (Math.abs(length - m) <= tolerantPts) {
+ return mid; // Return the refined parameter
+ }
+
+ if (length > m) {
+ left = mid; // Need less, narrow range to the left
+ } else {
+ right = mid; // Need more, narrow range to the right
+ }
+ // System.out.println(left + "," + right);
}
- // 判断两条线段是否相交并计算交点
- // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
- public static Object[] lineIntersection(Point[] L1, Point[] L2) {
- double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
- double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
-
- // 判断是否相交（叉积）
- double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
- double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
- double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
- double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
-
- // 如果叉积条件满足，表示有交点
- // d1*d2<0意味着P3、P4分别在L12的两边
- // d3*d4<0意味着P1、P2分别在L34的两边
- // 两个同时满足说明有交点
- if (d1 * d2 < 0 && d3 * d4 < 0) {
- double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
- double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
- double x = x1 + t1 * (x2 - x1);
- double y = y1 + t1 * (y2 - y1);
- return new Object[]{true, new Point(x, y)};
- }
-
- // 检查是否起点或终点重合
- if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
- return new Object[]{true, new Point(x1, y1)};
+ return (left + right) / 2; // Return the average of left and right as the final parameter
+ }
+
+ public static List<Point> readFromFile(
+ String input, boolean hasHeader, int timeIdx, int valueIdx, int N) {
+ List<Point> res = new ArrayList<>();
+
+ // Using Files.lines() for memory-efficient line-by-line reading
+ try (Stream<String> lines = Files.lines(Paths.get(input))) {
+ Iterator<String> iterator = lines.iterator();
+ int lineNumber = 0;
+
+ // Skip the header line if necessary
+ if (hasHeader && iterator.hasNext()) {
+ iterator.next(); // Skip the header
+ }
+
+ // Process each line
+ while (iterator.hasNext()) {
+ lineNumber++;
+ String line = iterator.next();
+ String[] columns = line.split(",");
+
+ if (columns.length > Math.max(timeIdx, valueIdx)) {
+ double time;
+ if (timeIdx < 0) {
+ time = lineNumber;
+ } else {
+ time = Double.parseDouble(columns[timeIdx]);
+ }
+ double value = Double.parseDouble(columns[valueIdx]);
+ res.add(new Point(time, value));
+ // System.out.println("Line " + lineNumber + " - Time: " + time + ",
+ // Value: " + value);
+ } else {
+ System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
}
- if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
- return new Object[]{true, new Point(x2, y2)};
+ if (N > 0 && lineNumber >= N) {
+ // N>0控制点数，否则N<=0表示读全部数据
+ break;
}
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return res;
+ }
+
+ // 计算三角形面积
+ public static double triArea(Point d0, Point d1, Point d2) {
+ double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
+ return (dArea > 0.0) ? dArea : -dArea; // abs
+ }
+
+ // 计算简单多边形（边不交叉）的面积（鞋带公式）
+ public static double calculatePolygonArea(List<Point> points) {
+ // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
+ int n = points.size();
+ double area = 0;
+ for (int i = 0; i < n; i++) {
+ int next = (i + 1) % n;
+ area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
+ }
+ return Math.abs(area) / 2.0;
+ }
+
+ // 计算两个向量的叉积
+ public static double crossProduct(double x1, double y1, double x2, double y2) {
+ // >0: (x2,y2)在(x1,y1)的逆时针方向
+ // <0: (x2,y2)在(x1,y1)的顺时针方向
+ // =0: 平行或共线
+ return x1 * y2 - y1 * x2;
+ }
+
+ // 判断两条线段是否相交并计算交点
+ // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
+ public static Object[] lineIntersection(Point[] L1, Point[] L2) {
+ double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
+ double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
+
+ // 判断是否相交（叉积）
+ double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
+ double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
+ double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
+ double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
+
+ // 如果叉积条件满足，表示有交点
+ // d1*d2<0意味着P3、P4分别在L12的两边
+ // d3*d4<0意味着P1、P2分别在L34的两边
+ // 两个同时满足说明有交点
+ if (d1 * d2 < 0 && d3 * d4 < 0) {
+ double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
+ double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
+ double x = x1 + t1 * (x2 - x1);
+ double y = y1 + t1 * (y2 - y1);
+ return new Object[] {true, new Point(x, y)};
+ }
- return new Object[]{false, null};
+ // 检查是否起点或终点重合
+ if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
+ return new Object[] {true, new Point(x1, y1)};
+ }
+ if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
+ return new Object[] {true, new Point(x2, y2)};
}
- // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
- public static double total_areal_displacement(
- List<Point> points, List<Point> points2, boolean debug) {
- double totalArea = 0;
- int i = 0, j = 0; // i for points, j for points2
- Point prevIntersection = null;
- int prevI = -1, prevJ = -1;
-
- // List<double[]> intersectionCoords = new ArrayList<>();
- List<Double> areaList = new ArrayList<>();
-
- while (i < points.size() - 1 && j < points2.size() - 1) {
- if (debug) {
- System.out.println("--------- " + i + " " + j + " ------------");
- }
-
- // 当前线段
- Point[] L1 = {points.get(i), points.get(i + 1)};
- Point[] L2 = {points2.get(j), points2.get(j + 1)};
-
- // 判断是否有交点
- Object[] result = lineIntersection(L1, L2);
- boolean isIntersect = (boolean) result[0];
- Point intersection = (Point) result[1];
-
- if (isIntersect) {
- // intersectionCoords.add(intersection);
-
- if (prevIntersection != null) {
- // 构造多边形点集
- List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
- polygon.add(prevIntersection);
- if (debug) {
- System.out.println("- start intersection: " + prevIntersection);
- }
- polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
- // polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i +
- // 1))); // 添加当前线段的点，左闭右开
- if (debug) {
- System.out.println("- one side: " + points.subList(prevI, i + 1));
- }
- polygon.add(intersection);
- if (debug) {
- System.out.println("- end intersection: " + intersection);
- }
- List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
- Collections.reverse(tempPoints2); // 添加另一序列的点
- polygon.addAll(tempPoints2);
- if (debug) {
- System.out.println("- another side: " + tempPoints2);
- }
-
- // double[][] polygonArray = new double[polygon.size()][2];
- // for (int k = 0; k < polygon.size(); k++) {
- // polygonArray[k] = polygon.get(k);
- // }
-
- // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
- double area = calculatePolygonArea(polygon);
- if (debug) {
- System.out.println("Area = " + area);
- }
- totalArea += area;
- areaList.add(area);
- }
-
- prevIntersection = intersection;
- prevI = i + 1;
- prevJ = j + 1;
- if (debug) {
- System.out.println(
- "This intersection = "
- + intersection
- + ", next polygon: side1 = "
- + prevI
- + ", side2 = "
- + prevJ);
- }
- }
-
- int currentI = i; // 临时存储i
- int currentJ = j; // 临时存储j
- if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
- i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
- }
- if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
- j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
- }
- } // end of while
+ return new Object[] {false, null};
+ }
+
+ // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
+ public static double total_areal_displacement(
+ List<Point> points, List<Point> points2, boolean debug) {
+ double totalArea = 0;
+ int i = 0, j = 0; // i for points, j for points2
+ Point prevIntersection = null;
+ int prevI = -1, prevJ = -1;
+
+ // List<double[]> intersectionCoords = new ArrayList<>();
+ List<Double> areaList = new ArrayList<>();
+
+ while (i < points.size() - 1 && j < points2.size() - 1) {
+ if (debug) {
+ System.out.println("--------- " + i + " " + j + " ------------");
+ }
+
+ // 当前线段
+ Point[] L1 = {points.get(i), points.get(i + 1)};
+ Point[] L2 = {points2.get(j), points2.get(j + 1)};
+
+ // 判断是否有交点
+ Object[] result = lineIntersection(L1, L2);
+ boolean isIntersect = (boolean) result[0];
+ Point intersection = (Point) result[1];
+
+ if (isIntersect) {
+ // intersectionCoords.add(intersection);
+
+ if (prevIntersection != null) {
+ // 构造多边形点集
+ List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
+ polygon.add(prevIntersection);
+ if (debug) {
+ System.out.println("- start intersection: " + prevIntersection);
+ }
+ polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
+ // polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i +
+ // 1))); // 添加当前线段的点，左闭右开
+ if (debug) {
+ System.out.println("- one side: " + points.subList(prevI, i + 1));
+ }
+ polygon.add(intersection);
+ if (debug) {
+ System.out.println("- end intersection: " + intersection);
+ }
+ List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
+ Collections.reverse(tempPoints2); // 添加另一序列的点
+ polygon.addAll(tempPoints2);
+ if (debug) {
+ System.out.println("- another side: " + tempPoints2);
+ }
+
+ // double[][] polygonArray = new double[polygon.size()][2];
+ // for (int k = 0; k < polygon.size(); k++) {
+ // polygonArray[k] = polygon.get(k);
+ // }
+
+ // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
+ double area = calculatePolygonArea(polygon);
+ if (debug) {
+ System.out.println("Area = " + area);
+ }
+ totalArea += area;
+ areaList.add(area);
+ }
+ prevIntersection = intersection;
+ prevI = i + 1;
+ prevJ = j + 1;
if (debug) {
- System.out.println(areaList);
+ System.out.println(
+ "This intersection = "
+ + intersection
+ + ", next polygon: side1 = "
+ + prevI
+ + ", side2 = "
+ + prevJ);
}
-
- return totalArea;
+ }
+
+ int currentI = i; // 临时存储i
+ int currentJ = j; // 临时存储j
+ if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
+ i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
+ j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ } // end of while
+
+ if (debug) {
+ System.out.println(areaList);
}
- // 测试方法
- public static void main(String[] args) {
- // 示例数据
- List<Point> points = new ArrayList<>();
- points.add(new Point(0, 0));
- points.add(new Point(1, 2));
- points.add(new Point(2, -20));
- points.add(new Point(3, 0));
- points.add(new Point(4, -1));
- points.add(new Point(5, -1.5));
- points.add(new Point(6, 0));
-
- List<Point> points2 = new ArrayList<>();
- // points2.add(points.get(0));
- // points2.add(points.get(3));
- // points2.add(points.get(6));
- points2.add(new Point(1, -10));
- points2.add(new Point(3, 0));
-
- double area = total_areal_displacement(points, points2, true);
- System.out.println("Total area: " + area);
-
- // points = new ArrayList<>();
- // points.add(new Point(0, 0));
- // points.add(new Point(1, 2));
- // points.add(new Point(1.5, -10));
- // double area = calculatePolygonArea(points);
- // System.out.println(area);
- //
- // Point[] L1 = new Point[2];
- // L1[0] = new Point(1, 2);
- // L1[1] = new Point(2, -2);
- // Point[] L2 = new Point[2];
- // L2[0] = new Point(0, 0);
- // L2[1] = new Point(3, 0);
- // Object[] res = lineIntersection(L1, L2);
- // System.out.println(res[1]);
- }
+ return totalArea;
+ }
+
+ // 测试方法
+ public static void main(String[] args) {
+ // 示例数据
+ List<Point> points = new ArrayList<>();
+ points.add(new Point(0, 0));
+ points.add(new Point(1, 2));
+ points.add(new Point(2, -20));
+ points.add(new Point(3, 0));
+ points.add(new Point(4, -1));
+ points.add(new Point(5, -1.5));
+ points.add(new Point(6, 0));
+
+ List<Point> points2 = new ArrayList<>();
+ // points2.add(points.get(0));
+ // points2.add(points.get(3));
+ // points2.add(points.get(6));
+ points2.add(new Point(1, -10));
+ points2.add(new Point(3, 0));
+
+ double area = total_areal_displacement(points, points2, true);
+ System.out.println("Total area: " + area);
+
+ // points = new ArrayList<>();
+ // points.add(new Point(0, 0));
+ // points.add(new Point(1, 2));
+ // points.add(new Point(1.5, -10));
+ // double area = calculatePolygonArea(points);
+ // System.out.println(area);
+ //
+ // Point[] L1 = new Point[2];
+ // L1[0] = new Point(1, 2);
+ // L1[1] = new Point(2, -2);
+ // Point[] L2 = new Point[2];
+ // L2[0] = new Point(0, 0);
+ // L2[1] = new Point(3, 0);
+ // Object[] res = lineIntersection(L1, L2);
+ // System.out.println(res[1]);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpL2.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpYdiff.java
similarity index 67%
rename from server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpL2.java
rename to server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpYdiff.java
index 62c4b5ffe30..ec96d1d45e8 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpL2.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpYdiff.java
@@ -7,11 +7,11 @@ import java.util.List;
import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
-public class sample_bottomUpL2 {
+public class sample_bottomUpYdiff {
public static void main(String[] args) throws IOException {
- if (args.length < 6) {
+ if (args.length < 7) {
System.out.println(
- "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,(outDir)");
+ "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,errorType,(outDir)");
}
String input = args[0];
boolean hasHeader = Boolean.parseBoolean(args[1]);
@@ -19,9 +19,20 @@ public class sample_bottomUpL2 {
int valueIdx = Integer.parseInt(args[3]);
int N = Integer.parseInt(args[4]); // N<=0表示读全部行，N>0表示读最多N行
int m = Integer.parseInt(args[5]);
+ String errorTypeStr = args[6];
+ DP.ERRORtype errorType;
+ if (errorTypeStr.equals("L1")) {
+ errorType = DP.ERRORtype.L1;
+ } else if (errorTypeStr.equals("L2")) {
+ errorType = DP.ERRORtype.L2;
+ } else if (errorTypeStr.equals("L_infy")) {
+ errorType = DP.ERRORtype.L_infy;
+ } else {
+ throw new IOException("please input errorType as L1/L2/L_infy");
+ }
String outDir;
- if (args.length > 6) {
- outDir = args[6]; // 表示输出文件保存到指定的文件夹
+ if (args.length > 7) {
+ outDir = args[7]; // 表示输出文件保存到指定的文件夹
} else {
outDir = null; // 表示输出文件保存到input文件所在的文件夹
}
@@ -33,18 +44,20 @@ public class sample_bottomUpL2 {
System.out.println("Value index: " + valueIdx);
System.out.println("N: " + N);
System.out.println("m: " + m);
+ System.out.println("errorType: " + errorType);
System.out.println("outDir: " + outDir);
// 读取原始序列
List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
String outputFile =
- generateOutputFileName(input, outDir, "-BUL2" + "-n" + points.size() + "-m" + m);
- System.out.println("Output file: " + outputFile);
+ generateOutputFileName(input, outDir, "-BU" + errorType + "-n" + points.size() + "-m" + m);
+ System.out.println("Output file: " + outputFile); // do not modify this hint string log
// 采样
long startTime = System.currentTimeMillis();
- List<Point> results = SWAB.seg_bottomUp_m_withTimestamps(points, m, null, false);
+ // List<Point> results = SWAB.seg_bottomUp_m_withTimestamps(points, m, null, false);
+ List<Point> results = BottomUpYdiff.reducePoints(points, m, errorType, false);
long endTime = System.currentTimeMillis();
System.out.println("result point number: " + results.size());
@@ -53,7 +66,7 @@ public class sample_bottomUpL2 {
+ points.size()
+ ", m="
+ m
- + ", Time taken to reduce points: "
+ + ", Time taken to reduce points: " // do not modify this hint string log
+ (endTime - startTime)
+ "ms");
(iotdb) 10/44: add ad
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6857dc2e13d65bbe6b074cc49593c85a1d057eb2
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jan 21 17:16:58 2025 +0800
add ad
---
.../java/org/apache/iotdb/db/query/eBUG/Point.java | 17 ++
.../org/apache/iotdb/db/query/eBUG/Polyline.java | 25 ++
.../java/org/apache/iotdb/db/query/eBUG/Tool.java | 198 +++++++++++++++
.../org/apache/iotdb/db/query/eBUG/Triangle.java | 49 ++++
.../query/{simpiece => eBUG}/Visval_standard.java | 174 ++++---------
.../Visval_standard.java => eBUG/eBUG.java} | 282 ++++++++++-----------
6 files changed, 480 insertions(+), 265 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
new file mode 100644
index 00000000000..77ebdb4cb39
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
@@ -0,0 +1,17 @@
+package org.apache.iotdb.db.query.eBUG;
+
+public class Point {
+
+ double x, y, z;
+
+ public Point(double x, double y) {
+ this.x = x;
+ this.y = y;
+ this.z = Double.POSITIVE_INFINITY; // effective area
+ }
+
+ @Override
+ public String toString() {
+ return "Point: (" + x + ", " + y + ", " + z + ")";
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
new file mode 100644
index 00000000000..b6e2f231c14
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
@@ -0,0 +1,25 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Polyline {
+
+ private List<Point> vertices = new ArrayList<>();
+
+ public void addVertex(Point point) {
+ vertices.add(point);
+ }
+
+ public List<Point> getVertices() {
+ return new ArrayList<>(vertices);
+ }
+
+ public int size() {
+ return vertices.size();
+ }
+
+ public Point get(int index) {
+ return vertices.get(index);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
new file mode 100644
index 00000000000..48c1544e76b
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
@@ -0,0 +1,198 @@
+package org.apache.iotdb.db.query.eBUG;
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class Tool {
+ // 计算三角形面积
+ public static double triArea(Point d0, Point d1, Point d2) {
+ double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
+ return (dArea > 0.0) ? dArea : -dArea; // abs
+ }
+
+ // 计算多边形面积（鞋带公式）
+ public static double calculatePolygonArea(List<Point> points) {
+ // points多边形顶点，按照逆时针或者顺时针的顺序枚举
+ int n = points.size();
+ double area = 0;
+ for (int i = 0; i < n; i++) {
+ int next = (i + 1) % n;
+ area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
+ }
+ return Math.abs(area) / 2.0;
+ }
+
+ // 计算两个向量的叉积
+ public static double crossProduct(double x1, double y1, double x2, double y2) {
+ // >0: (x2,y2)在(x1,y1)的逆时针方向
+ // <0: (x2,y2)在(x1,y1)的顺时针方向
+ // =0: 平行或共线
+ return x1 * y2 - y1 * x2;
+ }
+
+ // 判断两条线段是否相交并计算交点
+ // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
+ public static Object[] lineIntersection(Point[] L1, Point[] L2) {
+ double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
+ double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
+
+ // 判断是否相交（叉积）
+ double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
+ double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
+ double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
+ double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
+
+ // 如果叉积条件满足，表示有交点
+ // d1*d2<0意味着P3、P4分别在L12的两边
+ // d3*d4<0意味着P1、P2分别在L34的两边
+ // 两个同时满足说明有交点
+ if (d1 * d2 < 0 && d3 * d4 < 0) {
+ double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
+ double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
+ double x = x1 + t1 * (x2 - x1);
+ double y = y1 + t1 * (y2 - y1);
+ return new Object[]{true, new Point(x, y)};
+ }
+
+ // 检查是否起点或终点重合
+ if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
+ return new Object[]{true, new Point(x1, y1)};
+ }
+ if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
+ return new Object[]{true, new Point(x2, y2)};
+ }
+
+ return new Object[]{false, null};
+ }
+
+ // 计算总的多边形面积（通过时间序列扫描交点）
+ public static double total_areal_displacement(List<Point> points, List<Point> points2, boolean debug) {
+ double totalArea = 0;
+ int i = 0, j = 0; // i for points, j for points2
+ Point prevIntersection = null;
+ int prevI = -1, prevJ = -1;
+
+// List<double[]> intersectionCoords = new ArrayList<>();
+ List<Double> areaList = new ArrayList<>();
+
+ while (i < points.size() - 1 && j < points2.size() - 1) {
+ if (debug) {
+ System.out.println("--------- " + i + " " + j + " ------------");
+ }
+
+ // 当前线段
+ Point[] L1 = {points.get(i), points.get(i + 1)};
+ Point[] L2 = {points2.get(j), points2.get(j + 1)};
+
+ // 判断是否有交点
+ Object[] result = lineIntersection(L1, L2);
+ boolean isIntersect = (boolean) result[0];
+ Point intersection = (Point) result[1];
+
+ if (isIntersect) {
+// intersectionCoords.add(intersection);
+
+ if (prevIntersection != null) {
+ // 构造多边形点集
+ List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
+ polygon.add(prevIntersection);
+ if (debug) {
+ System.out.println("- start intersection: " + prevIntersection);
+ }
+ polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
+// polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i + 1))); // 添加当前线段的点，左闭右开
+ if (debug) {
+ System.out.println("- one side: " + points.subList(prevI, i + 1));
+ }
+ polygon.add(intersection);
+ if (debug) {
+ System.out.println("- end intersection: " + intersection);
+ }
+ List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
+ Collections.reverse(tempPoints2); // 添加另一序列的点
+ polygon.addAll(tempPoints2);
+ if (debug) {
+ System.out.println("- another side: " + tempPoints2);
+ }
+
+// double[][] polygonArray = new double[polygon.size()][2];
+// for (int k = 0; k < polygon.size(); k++) {
+// polygonArray[k] = polygon.get(k);
+// }
+
+ // 计算多边形面积
+ double area = calculatePolygonArea(polygon);
+ if (debug) {
+ System.out.println("Area = " + area);
+ }
+ totalArea += area;
+ areaList.add(area);
+ }
+
+ prevIntersection = intersection;
+ prevI = i + 1;
+ prevJ = j + 1;
+ if (debug) {
+ System.out.println("This intersection = " + intersection
+ + ", next polygon: side1 = " + prevI + ", side2 = " + prevJ);
+ }
+ }
+
+
+ int currentI = i; // 临时存储i
+ int currentJ = j; // 临时存储j
+ if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
+ i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
+ j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ } // end of while
+
+ if (debug) {
+ System.out.println(areaList);
+ }
+
+ return totalArea;
+ }
+
+ // 测试方法
+ public static void main(String[] args) {
+ // 示例数据
+ List<Point> points = new ArrayList<>();
+ points.add(new Point(0, 0));
+ points.add(new Point(1, 2));
+ points.add(new Point(2, -20));
+ points.add(new Point(3, 0));
+ points.add(new Point(4, -1));
+ points.add(new Point(5, -1.5));
+ points.add(new Point(6, 0));
+
+ List<Point> points2 = new ArrayList<>();
+ points2.add(points.get(0));
+ points2.add(points.get(3));
+ points2.add(points.get(6));
+// points2.add(points.get(5));
+
+ double area = total_areal_displacement(points, points2, true);
+ System.out.println("Total area: " + area);
+
+// points = new ArrayList<>();
+// points.add(new Point(0, 0));
+// points.add(new Point(1, 2));
+// points.add(new Point(1.5, -10));
+// double area = calculatePolygonArea(points);
+// System.out.println(area);
+//
+// Point[] L1 = new Point[2];
+// L1[0] = new Point(1, 2);
+// L1[1] = new Point(2, -2);
+// Point[] L2 = new Point[2];
+// L2[0] = new Point(0, 0);
+// L2[1] = new Point(3, 0);
+// Object[] res = lineIntersection(L1, L2);
+// System.out.println(res[1]);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Triangle.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Triangle.java
new file mode 100644
index 00000000000..79f7d440c99
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Triangle.java
@@ -0,0 +1,49 @@
+package org.apache.iotdb.db.query.eBUG;
+
+// adapted from the open source C++ code
+// https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
+public class Triangle {
+
+ int[] indices = new int[3];
+ double area;
+ Triangle prev;
+ Triangle next;
+ boolean isDeleted;
+
+ public Triangle(int index1, int index2, int index3, double area) {
+ this.indices[0] = index1;
+ this.indices[1] = index2;
+ this.indices[2] = index3;
+ this.area = area;
+ this.isDeleted = false; // flag for removal. Avoid using heap.remove(x) as it is O(n) complexity
+ }
+
+ public Triangle(Triangle oldTri) {
+ // deep copy and inherit connection
+
+ this.indices[0] = oldTri.indices[0];
+ this.indices[1] = oldTri.indices[1];
+ this.indices[2] = oldTri.indices[2];
+ this.area = oldTri.area;
+ this.prev = oldTri.prev;
+ this.next = oldTri.next;
+
+ // TODO important! inherit connection relationship to this new point
+ if (this.prev != null) { // previous point to this new point
+ this.prev.next = this;
+ }
+ if (this.next != null) { // next point to this new point
+ this.next.prev = this;
+ }
+
+ this.isDeleted = false; // this new triangle is not deleted
+ }
+
+ public boolean isValid() {
+ return !isDeleted;
+ }
+
+ public void markDeleted() {
+ this.isDeleted = true;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Visval_standard.java
similarity index 72%
copy from server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
copy to server/src/main/java/org/apache/iotdb/db/query/eBUG/Visval_standard.java
index 4edf126b531..a85332d3846 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Visval_standard.java
@@ -17,102 +17,18 @@
* under the License.
*/
-package org.apache.iotdb.db.query.simpiece;
+package org.apache.iotdb.db.query.eBUG;
import java.io.FileWriter;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
-// adapted from the open source C++ code
-// https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
-class Triangle {
-
- int[] indices = new int[3];
- double area;
- Triangle prev;
- Triangle next;
- boolean isDeleted;
-
- public Triangle(int index1, int index2, int index3, double area) {
- this.indices[0] = index1;
- this.indices[1] = index2;
- this.indices[2] = index3;
- this.area = area;
- this.isDeleted = false; // flag for removal. Avoid using heap.remove(x) as it is O(n) complexity
- }
-
- public Triangle(Triangle oldTri) {
- // deep copy and inherit connection
-
- this.indices[0] = oldTri.indices[0];
- this.indices[1] = oldTri.indices[1];
- this.indices[2] = oldTri.indices[2];
- this.area = oldTri.area;
- this.prev = oldTri.prev;
- this.next = oldTri.next;
-
- // TODO important! inherit connection relationship to this new point
- if (this.prev != null) { // previous point to this new point
- this.prev.next = this;
- }
- if (this.next != null) { // next point to this new point
- this.next.prev = this;
- }
-
- this.isDeleted = false; // this new triangle is not deleted
- }
-
- public boolean isValid() {
- return !isDeleted;
- }
-
- public void markDeleted() {
- this.isDeleted = true;
- }
-}
-
-class vPoint {
-
- double x, y, z;
-
- public vPoint(double x, double y) {
- this.x = x;
- this.y = y;
- this.z = Double.POSITIVE_INFINITY; // effective area
- }
-}
-
-class Polyline {
-
- private List<vPoint> vertices = new ArrayList<>();
-
- public void addVertex(vPoint point) {
- vertices.add(point);
- }
-
- public List<vPoint> getVertices() {
- return new ArrayList<>(vertices);
- }
-
- public int size() {
- return vertices.size();
- }
-
- public vPoint get(int index) {
- return vertices.get(index);
- }
-}
+import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
public class Visval_standard {
- // 计算三角形面积
- private static double triArea(vPoint d0, vPoint d1, vPoint d2) {
- double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
- return (double) ((dArea > 0.0) ? dArea : -dArea); // abs
- }
-
- public static void buildEffectiveArea(Polyline lineToSimplify, List<vPoint> results) {
+ public static void buildEffectiveArea(Polyline lineToSimplify, List<Point> results) {
results.clear();
results.addAll(lineToSimplify.getVertices()); // TODO debug
@@ -216,37 +132,37 @@ public class Visval_standard {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 100_0000;
+ int n = 10_0000;
- int p = 1000;
+ int p = 700;
for (int i = 0; i < n; i += p) {
Polyline polylineBatch = new Polyline();
for (int j = i; j < Math.min(i + p, n); j++) {
double v = rand.nextInt(1000000);
- polyline.addVertex(new vPoint(j, v));
+ polyline.addVertex(new Point(j, v));
- polylineBatch.addVertex(new vPoint(j, v));
+ polylineBatch.addVertex(new Point(j, v));
}
polylineList.add(polylineBatch);
}
-// try (FileWriter writer = new FileWriter("raw.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < polyline.size(); i++) {
-// vPoint point = polyline.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
System.out.println("---------------------------------");
- List<vPoint> results = new ArrayList<>();
+ List<Point> results = new ArrayList<>();
// 计算运行时间
long startTime = System.currentTimeMillis();
buildEffectiveArea(polyline, results);
@@ -258,32 +174,32 @@ public class Visval_standard {
if (results.size() <= 100) {
System.out.println("+++++++++++++++++++");
for (int i = 0; i < results.size(); i++) {
- vPoint point = results.get(i);
+ Point point = results.get(i);
System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
}
}
-// try (FileWriter writer = new FileWriter("fast.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < results.size(); i++) {
-// vPoint point = results.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
+ try (FileWriter writer = new FileWriter("fast.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
System.out.println("---------------------------------");
- List<List<vPoint>> resultsBatchList = new ArrayList<>();
+ List<List<Point>> resultsBatchList = new ArrayList<>();
// 计算运行时间
int cnt = 0;
startTime = System.currentTimeMillis();
for (Polyline polylineBatch : polylineList) {
- List<vPoint> resultsBatch = new ArrayList<>();
+ List<Point> resultsBatch = new ArrayList<>();
buildEffectiveArea(polylineBatch, resultsBatch);
cnt += resultsBatch.size();
resultsBatchList.add(resultsBatch);
@@ -295,7 +211,7 @@ public class Visval_standard {
System.out.println("---------------------------------");
// 使用 Stream API 合并所有列表
- List<vPoint> mergedList =
+ List<Point> mergedList =
resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
int sameCnt = 0;
for (int i = 0; i < mergedList.size(); i++) {
@@ -304,6 +220,20 @@ public class Visval_standard {
}
}
System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
+
+ try (FileWriter writer = new FileWriter("batch.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < mergedList.size(); i++) {
+ Point point = mergedList.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
}
// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
similarity index 57%
rename from server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
rename to server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 4edf126b531..cc512cbcc9c 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -17,105 +17,50 @@
* under the License.
*/
-package org.apache.iotdb.db.query.simpiece;
+package org.apache.iotdb.db.query.eBUG;
import java.io.FileWriter;
import java.io.IOException;
import java.util.*;
-import java.util.stream.Collectors;
-
-// adapted from the open source C++ code
-// https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
-class Triangle {
-
- int[] indices = new int[3];
- double area;
- Triangle prev;
- Triangle next;
- boolean isDeleted;
-
- public Triangle(int index1, int index2, int index3, double area) {
- this.indices[0] = index1;
- this.indices[1] = index2;
- this.indices[2] = index3;
- this.area = area;
- this.isDeleted = false; // flag for removal. Avoid using heap.remove(x) as it is O(n) complexity
- }
-
- public Triangle(Triangle oldTri) {
- // deep copy and inherit connection
-
- this.indices[0] = oldTri.indices[0];
- this.indices[1] = oldTri.indices[1];
- this.indices[2] = oldTri.indices[2];
- this.area = oldTri.area;
- this.prev = oldTri.prev;
- this.next = oldTri.next;
- // TODO important! inherit connection relationship to this new point
- if (this.prev != null) { // previous point to this new point
- this.prev.next = this;
- }
- if (this.next != null) { // next point to this new point
- this.next.prev = this;
+import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
+
+
+public class eBUG {
+ public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
+ Point pa, Point pi, Point pb) {
+ // TODO 寻找最近淘汰的e个点，并且是位于pa~pb之间的
+ // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
+ // pi: point need significance updating
+ // pa: left adjacent non-eliminated point
+ // pb: right adjacent non-eliminated point
+ // return a list of points, containing pa,p,pb and points between pa&pa from the e most recently eliminated points,
+ // order by time in ascending order
+ List<Point> res = new ArrayList<>();
+ for (int idx : recentEliminated) { // e points
+ if (idx == 0) { // init, not filled by real eliminated points yet
+ break;
+ }
+ Point p = lineToSimplify.get(idx);
+ if (p.x > pa.x && p.x < pb.x) {
+ res.add(p);
+ }
}
-
- this.isDeleted = false; // this new triangle is not deleted
+ res.add(pa);
+ res.add(pi);
+ res.add(pb);
+ res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 TODO 这样的话时间复杂度要变成e*log(e)了吧？
+ return res;
}
- public boolean isValid() {
- return !isDeleted;
- }
-
- public void markDeleted() {
- this.isDeleted = true;
- }
-}
-
-class vPoint {
-
- double x, y, z;
-
- public vPoint(double x, double y) {
- this.x = x;
- this.y = y;
- this.z = Double.POSITIVE_INFINITY; // effective area
- }
-}
-
-class Polyline {
-
- private List<vPoint> vertices = new ArrayList<>();
-
- public void addVertex(vPoint point) {
- vertices.add(point);
- }
-
- public List<vPoint> getVertices() {
- return new ArrayList<>(vertices);
- }
-
- public int size() {
- return vertices.size();
- }
-
- public vPoint get(int index) {
- return vertices.get(index);
- }
-}
-
-public class Visval_standard {
-
- // 计算三角形面积
- private static double triArea(vPoint d0, vPoint d1, vPoint d2) {
- double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
- return (double) ((dArea > 0.0) ? dArea : -dArea); // abs
- }
-
- public static void buildEffectiveArea(Polyline lineToSimplify, List<vPoint> results) {
+ public static void buildEffectiveArea(Polyline lineToSimplify, List<Point> results, int e) {
results.clear();
results.addAll(lineToSimplify.getVertices()); // TODO debug
+ // TODO 需要一个东西来记录最近淘汰的点依次是谁
+ int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
+ int recentEliminatedIdx = 0; // index in the array, circulate
+
int total = lineToSimplify.size();
if (total < 3) {
return; // 不足 3 个点无法形成三角形
@@ -153,6 +98,16 @@ public class Visval_standard {
continue;
}
+ // 真正的淘汰点
+ //TODO 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+// popOrder.add(tri.indices[1]);
+ System.out.println("eliminate " + lineToSimplify.get(tri.indices[1]));
+ if (e > 0) {
+ recentEliminated[recentEliminatedIdx] = tri.indices[1];
+ recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1
+ }
+ System.out.println(Arrays.toString(recentEliminated));
+
// 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
if (tri.area > previousEA) {
previousEA = tri.area;
@@ -171,13 +126,26 @@ public class Visval_standard {
tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
-// tri.prev = newPre; // replace TODO can omit
+// tri.prev = newPre; // replace TODO can omit, because already done by new Triangle(tri.prev)
// 2. 处理pi被淘汰引起tri.prev被更新的事情
// 前一个三角形连到后一个三角形
tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
tri.prev.indices[2] = tri.indices[2];
- tri.prev.area = triArea(lineToSimplify.get(tri.prev.indices[0]), lineToSimplify.get(tri.prev.indices[1]), lineToSimplify.get(tri.prev.indices[2]));
+
+ // TODO e parameter
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
+ lineToSimplify.get(tri.prev.indices[0]),
+ lineToSimplify.get(tri.prev.indices[1]),
+ lineToSimplify.get(tri.prev.indices[2]));
+ System.out.println("for updating Point on the left " + lineToSimplify.get(tri.prev.indices[1]));
+ for (Point point : pointsForSig) {
+ System.out.println(point);
+ }
+
+ tri.prev.area = triArea(lineToSimplify.get(tri.prev.indices[0]),
+ lineToSimplify.get(tri.prev.indices[1]),
+ lineToSimplify.get(tri.prev.indices[2]));
// 重新加入堆
// 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
@@ -193,7 +161,7 @@ public class Visval_standard {
tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
-// tri.next = newNext; // replace TODO can omit
+// tri.next = newNext; // replace TODO can omit, because already done by new Triangle(tri.prev)
if (tri.prev != null) {
tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
@@ -202,13 +170,27 @@ public class Visval_standard {
// 2. 处理pi被淘汰引起tri.next被更新的事情
tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
tri.next.indices[0] = tri.indices[0];
- tri.next.area = triArea(lineToSimplify.get(tri.next.indices[0]), lineToSimplify.get(tri.next.indices[1]), lineToSimplify.get(tri.next.indices[2]));
+
+ // TODO e parameter
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
+ lineToSimplify.get(tri.next.indices[0]),
+ lineToSimplify.get(tri.next.indices[1]),
+ lineToSimplify.get(tri.next.indices[2]));
+ System.out.println("for updating Point on the right " + lineToSimplify.get(tri.next.indices[1]));
+ for (Point point : pointsForSig) {
+ System.out.println(point);
+ }
+
+ tri.next.area = triArea(lineToSimplify.get(tri.next.indices[0]),
+ lineToSimplify.get(tri.next.indices[1]),
+ lineToSimplify.get(tri.next.indices[2]));
// 重新加入堆
// 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
// 所以必须通过add 来显式重新插入修改后的元素
triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
}
+ System.out.println(">>>");
}
}
@@ -216,40 +198,41 @@ public class Visval_standard {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 100_0000;
+ int n = 10;
- int p = 1000;
+ int p = 10;
for (int i = 0; i < n; i += p) {
Polyline polylineBatch = new Polyline();
for (int j = i; j < Math.min(i + p, n); j++) {
double v = rand.nextInt(1000000);
- polyline.addVertex(new vPoint(j, v));
+ polyline.addVertex(new Point(j, v));
- polylineBatch.addVertex(new vPoint(j, v));
+ polylineBatch.addVertex(new Point(j, v));
}
polylineList.add(polylineBatch);
}
-// try (FileWriter writer = new FileWriter("raw.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < polyline.size(); i++) {
-// vPoint point = polyline.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
System.out.println("---------------------------------");
- List<vPoint> results = new ArrayList<>();
+ List<Point> results = new ArrayList<>();
// 计算运行时间
+ int eParam = 3;
long startTime = System.currentTimeMillis();
- buildEffectiveArea(polyline, results);
+ buildEffectiveArea(polyline, results, eParam);
// 输出结果
long endTime = System.currentTimeMillis();
System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
@@ -258,52 +241,65 @@ public class Visval_standard {
if (results.size() <= 100) {
System.out.println("+++++++++++++++++++");
for (int i = 0; i < results.size(); i++) {
- vPoint point = results.get(i);
+ Point point = results.get(i);
System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
}
}
-// try (FileWriter writer = new FileWriter("fast.csv")) {
+ try (FileWriter writer = new FileWriter("fast.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+// System.out.println("---------------------------------");
+// List<List<Point>> resultsBatchList = new ArrayList<>();
+// // 计算运行时间
+// int cnt = 0;
+// startTime = System.currentTimeMillis();
+// for (Polyline polylineBatch : polylineList) {
+// List<Point> resultsBatch = new ArrayList<>();
+// buildEffectiveArea(polylineBatch, resultsBatch);
+// cnt += resultsBatch.size();
+// resultsBatchList.add(resultsBatch);
+// }
+// // 输出结果
+// endTime = System.currentTimeMillis();
+// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+// System.out.println(cnt);
+//
+// System.out.println("---------------------------------");
+// // 使用 Stream API 合并所有列表
+// List<Point> mergedList = resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
+// int sameCnt = 0;
+// for (int i = 0; i < mergedList.size(); i++) {
+// if (mergedList.get(i).z == results.get(i).z) {
+// sameCnt += 1;
+// }
+// }
+// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
+//
+// try (FileWriter writer = new FileWriter("batch.csv")) {
// // 写入CSV头部
// writer.append("x,y,z\n");
//
// // 写入每个点的数据
-// for (int i = 0; i < results.size(); i++) {
-// vPoint point = results.get(i);
+// for (int i = 0; i < mergedList.size(); i++) {
+// Point point = mergedList.get(i);
// writer.append(point.x + "," + point.y + "," + point.z + "\n");
// }
// System.out.println("Data has been written");
// } catch (IOException e) {
// System.out.println("Error writing to CSV file: " + e.getMessage());
// }
-
- System.out.println("---------------------------------");
- List<List<vPoint>> resultsBatchList = new ArrayList<>();
- // 计算运行时间
- int cnt = 0;
- startTime = System.currentTimeMillis();
- for (Polyline polylineBatch : polylineList) {
- List<vPoint> resultsBatch = new ArrayList<>();
- buildEffectiveArea(polylineBatch, resultsBatch);
- cnt += resultsBatch.size();
- resultsBatchList.add(resultsBatch);
- }
- // 输出结果
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(cnt);
-
- System.out.println("---------------------------------");
- // 使用 Stream API 合并所有列表
- List<vPoint> mergedList =
- resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
- int sameCnt = 0;
- for (int i = 0; i < mergedList.size(); i++) {
- if (mergedList.get(i).z == results.get(i).z) {
- sameCnt += 1;
- }
- }
- System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
}
// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
(iotdb) 34/44: before
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2270c6c291a1134a8f807581660f10c3942193b9
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jan 28 15:50:15 2025 +0800
before
---
server/pom.xml | 6 +-
server/sample_swab-jar-with-dependencies.jar | Bin 0 -> 40768969 bytes
.../java/org/apache/iotdb/db/query/eBUG/Rdp.java | 520 +++++++++++----------
.../eBUG/{sampled_Rdp.java => sample_Rdp.java} | 2 +-
4 files changed, 268 insertions(+), 260 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index 998fcc8f41f..f21af9a30cd 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -287,14 +287,16 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
- <finalName>sample_rdp</finalName>
+ <finalName>sample_swab</finalName>
+ <!-- <finalName>sample_rdp</finalName>-->
<!-- <finalName>sample_minmax</finalName>-->
<!-- <finalName>sample_fsw</finalName>-->
<!-- <finalName>sample_eBUG</finalName>-->
<!-- <finalName>sample_BUYdiff</finalName>-->
<archive>
<manifest>
- <mainClass>org.apache.iotdb.db.query.eBUG.sampled_Rdp</mainClass>
+ <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>-->
diff --git a/server/sample_swab-jar-with-dependencies.jar b/server/sample_swab-jar-with-dependencies.jar
new file mode 100644
index 00000000000..30bcf6213f7
Binary files /dev/null and b/server/sample_swab-jar-with-dependencies.jar differ
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
index 64cfc12e889..77b6df5aa3c 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
@@ -7,280 +7,286 @@ import java.util.Random;
import java.util.Stack;
public class Rdp {
- // Ramer–Douglas–Peucker algorithm (RDP)
- // top-down, for min perpendicular distance
- // worst case complexity n2
- // best case complexity nlogn
-
- public static double point_line_distance(Point point, Point lineStart, Point lineEnd) {
- // 计算点到直线的垂直正交距离
- // (x0,y0)到直线（经过(x1,y1)、(x2,y2)点的直线）的正交距离
- double x0 = point.x;
- double y0 = point.y;
- double x1 = lineStart.x;
- double y1 = lineStart.y;
- double x2 = lineEnd.x;
- double y2 = lineEnd.y;
-
- double numerator = Math.abs((x2 - x1) * (y1 - y0) - (x1 - x0) * (y2 - y1));
- double denominator = Math.sqrt(Math.pow(x2 - x1, 2) + Math.pow(y2 - y1, 2));
- return numerator / denominator;
- }
-
- // public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
- // if (points == null || points.size() < 3) {
- // return points;
- // }
- //
- // // 找到距离最远的点
- // double maxDistance = 0;
- // int index = 0;
- // Point start = points.get(0);
- // Point end = points.get(points.size() - 1);
- //
- // for (int i = 1; i < points.size() - 1; i++) {
- // double distance = point_line_distance(points.get(i), start, end);
- // if (distance > maxDistance) {
- // maxDistance = distance;
- // index = i;
- // }
- // }
- //
- // List<Point> result = new ArrayList<>();
- // if (maxDistance > epsilon) {
- // List<Point> firstPart = reducePoints(points.subList(0, index + 1), epsilon);
- // List<Point> secondPart = reducePoints(points.subList(index, points.size()), epsilon);
- //
- // // 合并结果
- // result.addAll(firstPart.subList(0, firstPart.size() - 1)); // 避免重复添加中间点
- // result.addAll(secondPart);
- // } else {
- // // 如果最远距离小于阈值，则只保留起点和终点
- // result.add(start);
- // result.add(end);
- // }
- //
- // return result;
- // }
-
- /**
- * 使用迭代实现 RDP 算法。避免递归深度过大。
- *
- * @param points 点集
- * @param epsilon 简化阈值
- * @return 简化后的点集
- */
- public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
- if (points.size() <= 2) {
- return points;
+ // Ramer–Douglas–Peucker algorithm (RDP)
+ // top-down, for min perpendicular distance
+ // worst case complexity n2
+ // best case complexity nlogn
+
+ public static double point_line_distance(Point point, Point lineStart, Point lineEnd) {
+ // 计算点到直线的垂直正交距离
+ // (x0,y0)到直线（经过(x1,y1)、(x2,y2)点的直线）的正交距离
+ double x0 = point.x;
+ double y0 = point.y;
+ double x1 = lineStart.x;
+ double y1 = lineStart.y;
+ double x2 = lineEnd.x;
+ double y2 = lineEnd.y;
+
+ double numerator = Math.abs((x2 - x1) * (y1 - y0) - (x1 - x0) * (y2 - y1));
+ double denominator = Math.sqrt(Math.pow(x2 - x1, 2) + Math.pow(y2 - y1, 2));
+ return numerator / denominator;
}
- // 使用栈模拟递归
- Stack<int[]> stack = new Stack<>();
- stack.push(new int[] {0, points.size() - 1});
-
- // 用于标记需要保留的点
- boolean[] keep = new boolean[points.size()];
- keep[0] = true;
- keep[points.size() - 1] = true;
-
- while (!stack.isEmpty()) {
- int[] range = stack.pop();
- int start = range[0];
- int end = range[1];
-
- // 找到距离最远的点
- double maxDistance = 0;
- int index = start;
- Point startPoint = points.get(start);
- Point endPoint = points.get(end);
-
- for (int i = start + 1; i < end; i++) {
- double distance = point_line_distance(points.get(i), startPoint, endPoint);
- if (distance > maxDistance) {
- maxDistance = distance;
- index = i;
+ // public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
+ // if (points == null || points.size() < 3) {
+ // return points;
+ // }
+ //
+ // // 找到距离最远的点
+ // double maxDistance = 0;
+ // int index = 0;
+ // Point start = points.get(0);
+ // Point end = points.get(points.size() - 1);
+ //
+ // for (int i = 1; i < points.size() - 1; i++) {
+ // double distance = point_line_distance(points.get(i), start, end);
+ // if (distance > maxDistance) {
+ // maxDistance = distance;
+ // index = i;
+ // }
+ // }
+ //
+ // List<Point> result = new ArrayList<>();
+ // if (maxDistance > epsilon) {
+ // List<Point> firstPart = reducePoints(points.subList(0, index + 1), epsilon);
+ // List<Point> secondPart = reducePoints(points.subList(index, points.size()), epsilon);
+ //
+ // // 合并结果
+ // result.addAll(firstPart.subList(0, firstPart.size() - 1)); // 避免重复添加中间点
+ // result.addAll(secondPart);
+ // } else {
+ // // 如果最远距离小于阈值，则只保留起点和终点
+ // result.add(start);
+ // result.add(end);
+ // }
+ //
+ // return result;
+ // }
+
+ /**
+ * 使用迭代实现 RDP 算法。避免递归深度过大。
+ *
+ * @param points 点集
+ * @param epsilon 简化阈值
+ * @return 简化后的点集
+ */
+ public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
+ if (points.size() <= 2) {
+ return points;
+ }
+
+ // 使用栈模拟递归
+ Stack<int[]> stack = new Stack<>();
+ stack.push(new int[]{0, points.size() - 1});
+
+ // 用于标记需要保留的点
+ boolean[] keep = new boolean[points.size()];
+ keep[0] = true;
+ keep[points.size() - 1] = true;
+
+ int count = 0; // TODO debug
+
+ while (!stack.isEmpty()) {
+ int[] range = stack.pop();
+ int start = range[0];
+ int end = range[1];
+
+ // 找到距离最远的点
+ double maxDistance = 0;
+ int index = start;
+ Point startPoint = points.get(start);
+ Point endPoint = points.get(end);
+
+ for (int i = start + 1; i < end; i++) {
+ count++;
+ double distance = point_line_distance(points.get(i), startPoint, endPoint);
+ if (distance > maxDistance) {
+ maxDistance = distance;
+ index = i;
+ }
+ }
+
+ // 如果最远距离大于阈值，则继续分割
+ if (maxDistance > epsilon) {
+ keep[index] = true; // 保留当前点
+ stack.push(new int[]{start, index});
+ stack.push(new int[]{index, end});
+ }
+ }
+
+ // 构建简化后的点集
+ List<Point> result = new ArrayList<>();
+ for (int i = 0; i < points.size(); i++) {
+ if (keep[i]) {
+ result.add(points.get(i));
+ }
}
- }
-
- // 如果最远距离大于阈值，则继续分割
- if (maxDistance > epsilon) {
- keep[index] = true; // 保留当前点
- stack.push(new int[] {start, index});
- stack.push(new int[] {index, end});
- }
- }
- // 构建简化后的点集
- List<Point> result = new ArrayList<>();
- for (int i = 0; i < points.size(); i++) {
- if (keep[i]) {
- result.add(points.get(i));
- }
+ System.out.println("count=" + count);
+ return result;
}
- return result;
- }
+ public static Object[] normalizePoints(List<Point> points) {
+ // 提取 x 和 y 值
+ double[] xValues = points.stream().mapToDouble(p -> p.x).toArray();
+ double[] yValues = points.stream().mapToDouble(p -> p.y).toArray();
- public static Object[] normalizePoints(List<Point> points) {
- // 提取 x 和 y 值
- double[] xValues = points.stream().mapToDouble(p -> p.x).toArray();
- double[] yValues = points.stream().mapToDouble(p -> p.y).toArray();
+ // 计算 x 和 y 的最小值和最大值
+ double xMin = Double.MAX_VALUE, xMax = Double.MIN_VALUE;
+ double yMin = Double.MAX_VALUE, yMax = Double.MIN_VALUE;
- // 计算 x 和 y 的最小值和最大值
- double xMin = Double.MAX_VALUE, xMax = Double.MIN_VALUE;
- double yMin = Double.MAX_VALUE, yMax = Double.MIN_VALUE;
+ for (double x : xValues) {
+ if (x < xMin) xMin = x;
+ if (x > xMax) xMax = x;
+ }
+ for (double y : yValues) {
+ if (y < yMin) yMin = y;
+ if (y > yMax) yMax = y;
+ }
- for (double x : xValues) {
- if (x < xMin) xMin = x;
- if (x > xMax) xMax = x;
- }
- for (double y : yValues) {
- if (y < yMin) yMin = y;
- if (y > yMax) yMax = y;
- }
+ // 标准化
+ List<Point> normalizedPoints = new ArrayList<>();
+ for (Point p : points) {
+ double xNormalized = (xMax != xMin) ? (p.x - xMin) / (xMax - xMin) : 0;
+ double yNormalized = (yMax != yMin) ? (p.y - yMin) / (yMax - yMin) : 0;
+ normalizedPoints.add(new Point(xNormalized, yNormalized));
+ }
- // 标准化
- List<Point> normalizedPoints = new ArrayList<>();
- for (Point p : points) {
- double xNormalized = (xMax != xMin) ? (p.x - xMin) / (xMax - xMin) : 0;
- double yNormalized = (yMax != yMin) ? (p.y - yMin) / (yMax - yMin) : 0;
- normalizedPoints.add(new Point(xNormalized, yNormalized));
+ return new Object[]{normalizedPoints, xMin, xMax, yMin, yMax};
}
- return new Object[] {normalizedPoints, xMin, xMax, yMin, yMax};
- }
-
- public static List<Point> denormalizePoints(List<Point> points, double[] originalScale) {
- double xMin = originalScale[0];
- double xMax = originalScale[1];
- double yMin = originalScale[2];
- double yMax = originalScale[3];
+ public static List<Point> denormalizePoints(List<Point> points, double[] originalScale) {
+ double xMin = originalScale[0];
+ double xMax = originalScale[1];
+ double yMin = originalScale[2];
+ double yMax = originalScale[3];
+
+ List<Point> denormalizedPoints = new ArrayList<>();
+ for (Point p : points) {
+ double xOriginal = p.x * (xMax - xMin) + xMin;
+ double yOriginal = p.y * (yMax - yMin) + yMin;
+ denormalizedPoints.add(new Point(xOriginal, yOriginal));
+ }
- List<Point> denormalizedPoints = new ArrayList<>();
- for (Point p : points) {
- double xOriginal = p.x * (xMax - xMin) + xMin;
- double yOriginal = p.y * (yMax - yMin) + yMin;
- denormalizedPoints.add(new Point(xOriginal, yOriginal));
+ return denormalizedPoints;
}
- return denormalizedPoints;
- }
-
- public static void main(String[] args) throws IOException {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = true;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 1000_0000;
- List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- // Polyline polyline = new Polyline();
- // for (int i = 0; i < N; i += 1) {
- // double v = rand.nextInt(1000);
- // polyline.addVertex(new Point(i, v));
- // }
- // List<Point> points = polyline.getVertices();
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (Point point : points) {
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(points.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
+ public static void main(String[] args) throws IOException {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 10;
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ // Polyline polyline = new Polyline();
+ // for (int i = 0; i < N; i += 1) {
+ // double v = rand.nextInt(1000);
+ // polyline.addVertex(new Point(i, v));
+ // }
+ // List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ // 标准化
+ Object[] normalizedData = normalizePoints(points);
+ List<Point> normalizedPoints = (List<Point>) normalizedData[0];
+ double xMin = (double) normalizedData[1];
+ double xMax = (double) normalizedData[2];
+ double yMin = (double) normalizedData[3];
+ double yMax = (double) normalizedData[4];
+
+ int m = 7;
+ double tolerantPts = 0;
+ double epsilon = Tool.getParam(normalizedPoints, m, Rdp::reducePoints, tolerantPts);
+// double epsilon = 0.001;
+// double epsilon = 0.004547272052729534;
+ System.out.println("epsilon=" + epsilon);
+
+ long startTime = System.currentTimeMillis();
+ List<Point> sampled_tmp = reducePoints(normalizedPoints, epsilon);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ // 还原到原始尺度
+ double[] originalScale = {xMin, xMax, yMin, yMax};
+ List<Point> sampled = denormalizePoints(sampled_tmp, originalScale);
+
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ System.out.println(sampled.size());
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
- // 标准化
- Object[] normalizedData = normalizePoints(points);
- List<Point> normalizedPoints = (List<Point>) normalizedData[0];
- double xMin = (double) normalizedData[1];
- double xMax = (double) normalizedData[2];
- double yMin = (double) normalizedData[3];
- double yMax = (double) normalizedData[4];
-
- int m = 10;
- double tolerantPts = 0;
- double epsilon = Tool.getParam(normalizedPoints, m, Rdp::reducePoints, tolerantPts);
- // double epsilon = 0.001;
-
- long startTime = System.currentTimeMillis();
- List<Point> sampled_tmp = reducePoints(normalizedPoints, epsilon);
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime - startTime) + "ms");
-
- // 还原到原始尺度
- double[] originalScale = {xMin, xMax, yMin, yMax};
- List<Point> sampled = denormalizePoints(sampled_tmp, originalScale);
-
- // for (Point p : sampled) {
- // System.out.println(p);
+ // public static void reducePoints(List<Point> points, double epsilon, int s, int e,
+ // List<Point> resultList) {
+ // double dmax = 0;
+ // int index = 0;
+ //
+ // final int start = s;
+ // final int end = e - 1;
+ // for (int i = start + 1; i < end; i++) {
+ // // Point
+ // final double px = list.get(i)[0];
+ // final double py = list.get(i)[1];
+ // // Start
+ // final double vx = list.get(start)[0];
+ // final double vy = list.get(start)[1];
+ // // End
+ // final double wx = list.get(end)[0];
+ // final double wy = list.get(end)[1];
+ // final double d = perpendicularDistance(px, py, vx, vy, wx, wy);
+ // if (d > dmax) {
+ // index = i;
+ // dmax = d;
+ // }
// }
- System.out.println(sampled.size());
-
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < sampled.size(); i++) {
- writer.println(sampled.get(i).x + "," + sampled.get(i).y);
- }
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- }
-
- // public static void reducePoints(List<Point> points, double epsilon, int s, int e,
- // List<Point> resultList) {
- // double dmax = 0;
- // int index = 0;
- //
- // final int start = s;
- // final int end = e - 1;
- // for (int i = start + 1; i < end; i++) {
- // // Point
- // final double px = list.get(i)[0];
- // final double py = list.get(i)[1];
- // // Start
- // final double vx = list.get(start)[0];
- // final double vy = list.get(start)[1];
- // // End
- // final double wx = list.get(end)[0];
- // final double wy = list.get(end)[1];
- // final double d = perpendicularDistance(px, py, vx, vy, wx, wy);
- // if (d > dmax) {
- // index = i;
- // dmax = d;
- // }
- // }
- // // If max distance is greater than epsilon, recursively simplify
- // if (dmax > epsilon) {
- // // Recursive call
- // douglasPeucker(list, s, index, epsilon, resultList);
- // douglasPeucker(list, index, e, epsilon, resultList);
- // } else {
- // if ((end - start) > 0) {
- // resultList.add(list.get(start));
- // resultList.add(list.get(end));
- // } else {
- // resultList.add(list.get(start));
- // }
- // }
- // }
-
- // /**
- // * Given a curve composed of line segments find a similar curve with fewer points.
- // *
- // * @param list List of Double[] points (x,y)
- // * @param epsilon Distance dimension
- // * @return Similar curve with fewer points
- // */
- // public static final List<Double[]> douglasPeucker(List<Double[]> list, double epsilon) {
- // final List<Double[]> resultList = new ArrayList<Double[]>();
- // douglasPeucker(list, 0, list.size(), epsilon, resultList);
- // return resultList;
- // }
+ // // If max distance is greater than epsilon, recursively simplify
+ // if (dmax > epsilon) {
+ // // Recursive call
+ // douglasPeucker(list, s, index, epsilon, resultList);
+ // douglasPeucker(list, index, e, epsilon, resultList);
+ // } else {
+ // if ((end - start) > 0) {
+ // resultList.add(list.get(start));
+ // resultList.add(list.get(end));
+ // } else {
+ // resultList.add(list.get(start));
+ // }
+ // }
+ // }
+
+ // /**
+ // * Given a curve composed of line segments find a similar curve with fewer points.
+ // *
+ // * @param list List of Double[] points (x,y)
+ // * @param epsilon Distance dimension
+ // * @return Similar curve with fewer points
+ // */
+ // public static final List<Double[]> douglasPeucker(List<Double[]> list, double epsilon) {
+ // final List<Double[]> resultList = new ArrayList<Double[]>();
+ // douglasPeucker(list, 0, list.size(), epsilon, resultList);
+ // return resultList;
+ // }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sampled_Rdp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_Rdp.java
similarity index 99%
rename from server/src/main/java/org/apache/iotdb/db/query/eBUG/sampled_Rdp.java
rename to server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_Rdp.java
index 834f84d09b7..245e398d4c9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sampled_Rdp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_Rdp.java
@@ -10,7 +10,7 @@ import static org.apache.iotdb.db.query.eBUG.Rdp.normalizePoints;
import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
import static org.apache.iotdb.db.query.eBUG.Tool.getParam;
-public class sampled_Rdp {
+public class sample_Rdp {
public static void main(String[] args) throws IOException {
if (args.length < 7) {
System.out.println(
(iotdb) 14/44: de
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b263ae6b18173016b9b90f399225c61c105bfc52
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jan 22 00:45:36 2025 +0800
de
---
.../db/query/eBUG/TimestampPriorityQueue.java | 42 +++----
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 121 +++++++++++----------
.../db/query/eBUG/{eBUG.java => eBUG_old_tmp.java} | 50 +++++----
3 files changed, 107 insertions(+), 106 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java
index 222790aed42..15c2dbc051c 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java
@@ -1,10 +1,5 @@
package org.apache.iotdb.db.query.eBUG;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.PriorityQueue;
-
import java.util.*;
public class TimestampPriorityQueue {
@@ -16,43 +11,40 @@ public class TimestampPriorityQueue {
public TimestampPriorityQueue(int e) {
this.e = e;
this.insertionOrder = new LinkedList<>();
- this.queue = new PriorityQueue<>(Comparator.comparingDouble(p -> p.timestamp));
+ this.queue = new PriorityQueue<>(Comparator.comparingDouble(p -> p.x));
+ }
+
+ public int size() {
+ return queue.size();
}
// 插入新的元素
public void insert(Point newPoint) {
// 如果队列已满，移除最早插入的点
if (queue.size() == e) {
- Point removedPoint = insertionOrder.removeFirst(); // 移除最早加入的点
- queue.remove(removedPoint); // 从优先队列中移除该点
+ Point removedPoint = insertionOrder.removeFirst(); // 移除最早加入的点 复杂度1
+ queue.remove(removedPoint); // 从优先队列中移除该点 TODO 复杂度O(e)!
}
// 将新点添加到插入顺序队列中
- insertionOrder.addLast(newPoint);
+ insertionOrder.addLast(newPoint); // 复杂度1
// 插入到优先队列，保持时间戳顺序
- queue.offer(newPoint);
+ queue.offer(newPoint); // 复杂度log(e)
}
// 获取队列中的所有元素（按时间戳排序）
public List<Point> getQueue() {
- return new ArrayList<>(queue);
+ return new ArrayList<>(queue); // 浅复制
}
- // Point 类，代表一个带时间戳的点
- public static class Point {
- int value;
- long timestamp;
-
- public Point(int value, long timestamp) {
- this.value = value;
- this.timestamp = timestamp;
- }
-
- @Override
- public String toString() {
- return "Value: " + value + ", Timestamp: " + timestamp;
+ public String toString() {
+ StringBuffer stringBuffer = new StringBuffer();
+ for (Point point : queue) {
+ stringBuffer.append(point.toString());
+ stringBuffer.append(",");
}
+ return stringBuffer.toString();
}
public static void main(String[] args) {
@@ -61,7 +53,7 @@ public class TimestampPriorityQueue {
tq.insert(new Point(10, 1000));
tq.insert(new Point(20, 2000));
tq.insert(new Point(15, 1500));
- tq.insert(new Point(25, 2500));
+ tq.insert(new Point(25, 500));
// 打印队列内容
for (Point point : tq.getQueue()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 62e67a0f1f9..61721a9e328 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.db.query.eBUG;
+import java.io.FileWriter;
+import java.io.IOException;
import java.util.*;
import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
@@ -26,12 +28,9 @@ import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
public class eBUG {
- public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
+ public static List<Point> findEliminated(Polyline lineToSimplify, TimestampPriorityQueue recentEliminatedOrdered,
int pa_idx, int pi_idx, int pb_idx) {
// TODO 复杂度分析
- // 从最近淘汰的e个点里寻找位于pa~pb之间的：e
- // 把上一步找到的点和pa pi pb一起按照时间戳递增排序：记c=b-a。如果e<c，那么eloge，否则c
- // 后面鞋带公式计算面积：min(e+3,c)
// 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
// pi: the point whose significance needs updating
// pa: left adjacent non-eliminated point of pi
@@ -53,7 +52,7 @@ public class eBUG {
// return res; // pa pi pb已经按照时间戳递增
// }
- if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
+ if (recentEliminatedOrdered.size() >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
return res; // 已经按照时间戳递增
@@ -62,23 +61,26 @@ public class eBUG {
// 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
// System.out.println("k>=[c=(b-a-2)]>e, search among e");
res.add(pa);
- res.add(pi);
- res.add(pb);
// 包括了c>e=0的情况
- for (int idx : recentEliminated) { // e points
- if (idx == 0) { // init, not filled by real eliminated points yet
+ boolean addPi = false;
+ for (Point p : recentEliminatedOrdered.getQueue()) { // 已经按时间戳递增排序，为了后面计算total areal displacement需要
+ if (p.x <= pa.x) {
+ continue;
+ }
+ if (p.x >= pb.x) {
break;
}
- Point p = lineToSimplify.get(idx);
- if (p.x > pa.x && p.x < pb.x) {
- res.add(p);
+ if (p.x > pi.x && !addPi) {
+ res.add(pi);
+ addPi = true;
}
+ res.add(p);
}
-
- // 按照时间戳递增排序，这是为了后面计算total areal displacement需要
- res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 TODO 这样的话时间复杂度要变成e*log(e)了吧？
-
+ if (!addPi) { // may happen when no eliminated points found above
+ res.add(pi);
+ }
+ res.add(pb);
return res;
}
@@ -91,8 +93,9 @@ public class eBUG {
// results.addAll(lineToSimplify.getVertices()); // TODO debug
// TODO 需要一个东西来记录最近淘汰的点依次是谁
- int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
- int recentEliminatedIdx = 0; // index in the array, circulate
+// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
+// int recentEliminatedIdx = 0; // index in the array, circulate
+ TimestampPriorityQueue recentEliminatedOrdered = new TimestampPriorityQueue(e);
int total = lineToSimplify.size();
if (total < 3) {
@@ -140,11 +143,12 @@ public class eBUG {
System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
}
if (e > 0) { // otherwise e=0
- recentEliminated[recentEliminatedIdx] = tri.indices[1];
- recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
+// recentEliminated[recentEliminatedIdx] = tri.indices[1];
+// recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
+ recentEliminatedOrdered.insert(lineToSimplify.get(tri.indices[1]));
}
if (debug) {
- System.out.println("the e most recently eliminated points:" + Arrays.toString(recentEliminated));
+ System.out.println("the e most recently eliminated points:" + recentEliminatedOrdered);
}
// 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
@@ -173,7 +177,7 @@ public class eBUG {
tri.prev.indices[2] = tri.indices[2];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
tri.prev.indices[0],
tri.prev.indices[1],
tri.prev.indices[2]); // TODO complexity ?
@@ -222,7 +226,7 @@ public class eBUG {
tri.next.indices[0] = tri.indices[0];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
tri.next.indices[0],
tri.next.indices[1],
tri.next.indices[2]); // TODO complexity
@@ -262,7 +266,8 @@ public class eBUG {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 1000_0000;
+// int n = 1000_0000;
+ int n = 20;
int p = 10;
for (int i = 0; i < n; i += p) {
@@ -277,52 +282,52 @@ public class eBUG {
polylineList.add(polylineBatch);
}
-// try (FileWriter writer = new FileWriter("raw.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < polyline.size(); i++) {
-// Point point = polyline.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
System.out.println("---------------------------------");
// List<Point> results = new ArrayList<>();
// 计算运行时间
int eParam = 10;
long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ List<Point> results = buildEffectiveArea(polyline, eParam, true);
// 输出结果
long endTime = System.currentTimeMillis();
System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
System.out.println(results.size());
-// if (results.size() <= 100) {
-// System.out.println("+++++++++++++++++++");
-// for (int i = 0; i < results.size(); i++) {
-// Point point = results.get(i);
-// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
-// }
-// }
+ if (results.size() <= 100) {
+ System.out.println("+++++++++++++++++++");
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ }
+ }
-// try (FileWriter writer = new FileWriter("fast.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < results.size(); i++) {
-// Point point = results.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
+ try (FileWriter writer = new FileWriter("fast.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
// System.out.println("---------------------------------");
// List<List<Point>> resultsBatchList = new ArrayList<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old_tmp.java
similarity index 93%
copy from server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
copy to server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old_tmp.java
index 62e67a0f1f9..2270b8253c2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old_tmp.java
@@ -19,13 +19,15 @@
package org.apache.iotdb.db.query.eBUG;
+import java.io.FileWriter;
+import java.io.IOException;
import java.util.*;
import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-public class eBUG {
+public class eBUG_old_tmp {
public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
int pa_idx, int pi_idx, int pb_idx) {
// TODO 复杂度分析
@@ -262,7 +264,7 @@ public class eBUG {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 1000_0000;
+ int n = 20;
int p = 10;
for (int i = 0; i < n; i += p) {
@@ -294,35 +296,37 @@ public class eBUG {
System.out.println("---------------------------------");
// List<Point> results = new ArrayList<>();
// 计算运行时间
+// int eParam = 10;
+// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
int eParam = 10;
long startTime = System.currentTimeMillis();
List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
long endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
System.out.println(results.size());
-// if (results.size() <= 100) {
-// System.out.println("+++++++++++++++++++");
-// for (int i = 0; i < results.size(); i++) {
-// Point point = results.get(i);
-// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
-// }
-// }
+ if (results.size() <= 100) {
+ System.out.println("+++++++++++++++++++");
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ }
+ }
-// try (FileWriter writer = new FileWriter("fast.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < results.size(); i++) {
-// Point point = results.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
+ try (FileWriter writer = new FileWriter("fast2.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
// System.out.println("---------------------------------");
// List<List<Point>> resultsBatchList = new ArrayList<>();
(iotdb) 25/44: swab
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4f8e6f3264b995b27646cd86c495ed03a32d6b38
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Jan 26 22:51:47 2025 +0800
swab
---
.../java/org/apache/iotdb/db/query/eBUG/Point.java | 60 +-
.../org/apache/iotdb/db/query/eBUG/Polyline.java | 56 +-
.../java/org/apache/iotdb/db/query/eBUG/SWAB.java | 536 ++++++++++++++++
.../java/org/apache/iotdb/db/query/eBUG/Test1.java | 2 +-
.../java/org/apache/iotdb/db/query/eBUG/Test4.java | 79 +++
.../java/org/apache/iotdb/db/query/eBUG/Test5.java | 64 ++
.../java/org/apache/iotdb/db/query/eBUG/Test6.java | 89 +++
.../java/org/apache/iotdb/db/query/eBUG/Tool.java | 418 +++++++------
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 696 ++++++++++-----------
.../org/apache/iotdb/db/query/eBUG/eBUG_Build.java | 236 +++----
10 files changed, 1517 insertions(+), 719 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
index e9bcf241f0f..3d47e88f422 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
@@ -2,33 +2,35 @@ package org.apache.iotdb.db.query.eBUG;
public class Point {
- double x, y, z;
-
- public Point prev; // 指向T'_max{0,k-e}里当前点的前一个点 for eBUG usage
- public Point next; // 指向T'_max{0,k-e}里当前点的后一个点 for eBUG usage
- // 注意Triangle里的prev&next用于最新状态下存在三角形之间的连接关系，
- // 而这里Point的prev&next用于滞后e个淘汰点（也就是最近的e个淘汰点先不施加）状态下的未淘汰点之间的连接关系
-
- public Point(double x, double y) {
- this.x = x;
- this.y = y;
- this.z = Double.POSITIVE_INFINITY; // effective area
-
- // for eBUG usage
- // this.eliminated = false;
- this.prev = null;
- this.next = null;
- }
-
- public void markEliminated() {
- // to avoid traversing each point between pa to pb,
- // instead only traversing at most e most recently eliminated points lagged
- prev.next = next;
- next.prev = prev;
- }
-
- @Override
- public String toString() {
- return "Point: (" + x + ", " + y + ", " + z + ")";
- }
+ double x, y, z;
+
+ public Point prev; // 指向T'_max{0,k-e}里当前点的前一个点 for eBUG usage
+ public Point next; // 指向T'_max{0,k-e}里当前点的后一个点 for eBUG usage
+ // 注意Triangle里的prev&next用于最新状态下存在三角形之间的连接关系，
+ // 而这里Point的prev&next用于滞后e个淘汰点（也就是最近的e个淘汰点先不施加）状态下的未淘汰点之间的连接关系
+
+ public int index; // for swab usage
+
+ public Point(double x, double y) {
+ this.x = x;
+ this.y = y;
+ this.z = Double.POSITIVE_INFINITY; // effective area
+
+ // for eBUG usage
+ // this.eliminated = false;
+ this.prev = null;
+ this.next = null;
+ }
+
+ public void markEliminated() {
+ // to avoid traversing each point between pa to pb,
+ // instead only traversing at most e most recently eliminated points lagged
+ prev.next = next;
+ next.prev = prev;
+ }
+
+ @Override
+ public String toString() {
+ return "Point: (" + x + ", " + y + ", " + z + ")";
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
index 534c7a6f67e..5b1f5354871 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
@@ -5,30 +5,34 @@ import java.util.List;
public class Polyline {
- private List<Point> vertices = new ArrayList<>();
-
- public void addVertex(Point point) {
- // if (!vertices.isEmpty()) { //before adding this point
- // vertices.get(vertices.size() - 1).next = point;
- // point.prev = vertices.get(vertices.size() - 1);
- // }
- vertices.add(point);
- }
-
- public List<Point> getVertices() {
- // return new ArrayList<>(vertices);
- return vertices;
- }
-
- public int size() {
- return vertices.size();
- }
-
- public Point get(int index) {
- return vertices.get(index);
- }
-
- public void clear() {
- vertices.clear();
- }
+ private List<Point> vertices = new ArrayList<>();
+
+ public void addVertex(Point point) {
+ // if (!vertices.isEmpty()) { //before adding this point
+ // vertices.get(vertices.size() - 1).next = point;
+ // point.prev = vertices.get(vertices.size() - 1);
+ // }
+ vertices.add(point);
+ }
+
+ public List<Point> getVertices() {
+ // return new ArrayList<>(vertices);
+ return vertices;
+ }
+
+ public void setVertices(List<Point> points) {
+ this.vertices = points;
+ }
+
+ public int size() {
+ return vertices.size();
+ }
+
+ public Point get(int index) {
+ return vertices.get(index);
+ }
+
+ public void clear() {
+ vertices.clear();
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
new file mode 100644
index 00000000000..574f59e3bf4
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
@@ -0,0 +1,536 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.*;
+
+
+public class SWAB {
+ public static Object[] prefixSum(List<Point> points) {
+ int n = points.size();
+
+ // 预计算prefix sum从而可以加速L2误差计算
+ double[] prefixSumX = new double[n];
+ double[] prefixSumY = new double[n];
+ double[] prefixSumX2 = new double[n];
+ double[] prefixSumY2 = new double[n];
+ double[] prefixSumXY = new double[n];
+
+ Point pend = points.get(points.size() - 1); // 最后一个点，按照时间戳递增排序所以应该baseX是最大时间戳，从而归一化
+ double baseX = pend.x; // 归一化避免数值溢出，主要就是这个时间戳太大
+ double baseY = pend.y; // 归一化避免数值溢出
+ if (baseX == 0) {
+ baseX = 1;
+ }
+ if (baseY == 0) {
+ baseY = 1;
+ }
+
+ Point p0 = points.get(0);
+ prefixSumX[0] = p0.x / baseX;
+ prefixSumY[0] = p0.y / baseY;
+ prefixSumX2[0] = (p0.x * p0.x) / (baseX * baseX);
+ prefixSumY2[0] = (p0.y * p0.y) / (baseY * baseY);
+ prefixSumXY[0] = (p0.x * p0.y) / (baseX * baseY);
+
+ for (int i = 1; i < n; i++) {
+ Point p = points.get(i);
+ prefixSumX[i] = prefixSumX[i - 1] + (p.x / baseX);
+ prefixSumY[i] = prefixSumY[i - 1] + (p.y / baseY);
+ prefixSumX2[i] = prefixSumX2[i - 1] + (p.x * p.x) / (baseX * baseX);
+ prefixSumY2[i] = prefixSumY2[i - 1] + (p.y * p.y) / (baseY * baseY);
+ prefixSumXY[i] = prefixSumXY[i - 1] + (p.x * p.y) / (baseX * baseY);
+ }
+
+ return new Object[]{prefixSumX, prefixSumY, prefixSumX2, prefixSumY2, prefixSumXY, baseX, baseY};
+ }
+
+ public static double myLA_withTimestamps(List<Point> points, Object[] prefixSum, int lx, int rx) {
+ // 默认joint=true, residual=true，即使用linear interpolation近似分段，且使用L2误差
+ // lx~rx 左闭右闭
+ // 使用prefix sum加速，但是注意防止溢出处理
+ double x1 = points.get(lx).x;
+ double y1 = points.get(lx).y;
+ double x2 = points.get(rx).x;
+ double y2 = points.get(rx).y;
+
+ double k = (y2 - y1) / (x2 - x1);
+ double b = (y1 * x2 - y2 * x1) / (x2 - x1);
+
+ // 计算L2误差，使用prefix sum加速计算
+ double[] prefixSumX = (double[]) prefixSum[0];
+ double[] prefixSumY = (double[]) prefixSum[1];
+ double[] prefixSumX2 = (double[]) prefixSum[2];
+ double[] prefixSumY2 = (double[]) prefixSum[3];
+ double[] prefixSumXY = (double[]) prefixSum[4];
+ double baseX = (double) prefixSum[5];
+ double baseY = (double) prefixSum[6];
+
+ double sumX, sumY, sumX2, sumY2, sumXY;
+
+ if (lx > 0) {
+ sumX = prefixSumX[rx] - prefixSumX[lx - 1];
+ sumY = prefixSumY[rx] - prefixSumY[lx - 1];
+ sumX2 = prefixSumX2[rx] - prefixSumX2[lx - 1];
+ sumY2 = prefixSumY2[rx] - prefixSumY2[lx - 1];
+ sumXY = prefixSumXY[rx] - prefixSumXY[lx - 1];
+ } else { // lx=0
+ sumX = prefixSumX[rx];
+ sumY = prefixSumY[rx];
+ sumX2 = prefixSumX2[rx];
+ sumY2 = prefixSumY2[rx];
+ sumXY = prefixSumXY[rx];
+ }
+
+ // 计算L2误差，注意归一化还原
+ return sumY2 * baseY * baseY
+ + k * k * sumX2 * baseX * baseX
+ + 2 * k * b * sumX * baseX
+ - 2 * b * sumY * baseY
+ - 2 * k * sumXY * baseX * baseY
+ + b * b * (rx - lx + 1);
+ }
+
+ public static double myLA_withTimestamps(List<Point> points, int lx, int rx) {
+ // 默认joint=true, residual=true，即使用linear interpolation近似分段，且使用L2误差
+ // lx~rx 左闭右闭
+ // 没有使用prefix sum加速
+ double x1 = points.get(lx).x;
+ double y1 = points.get(lx).y;
+ double x2 = points.get(rx).x;
+ double y2 = points.get(rx).y;
+
+ double k = (y2 - y1) / (x2 - x1);
+ double b = (y1 * x2 - y2 * x1) / (x2 - x1);
+
+ double tmp = 0;
+ for (int i = lx; i <= rx; i++) {
+ double e = (k * points.get(i).x + b - points.get(i).y) * (k * points.get(i).x + b - points.get(i).y);
+ tmp += e;
+ }
+ return tmp;
+ }
+
+ public static List<Point> seg_bottomUp_maxerror_withTimestamps(
+ List<Point> points, double maxError, Object[] prefixSum, boolean debug) {
+ int total = points.size();
+ if (total < 3) {
+ return points; // 不足 3 个点无法形成三角形
+ }
+
+ // 每个triangle对应两个相邻的待合并的分段
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ points.get(0).prev = null;
+ points.get(0).next = points.get(1);
+// points.get(0).index = 0;
+ points.get(total - 1).next = null;
+ points.get(total - 1).prev = points.get(total - 2);
+// points.get(total - 1).index = points.size() - 1;
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+ double mc;
+ if (prefixSum == null) {
+ mc = myLA_withTimestamps(points, index1, index3);
+ } else {
+ mc = myLA_withTimestamps(points, prefixSum, index1, index3);
+ }
+ triangles[i - 1] = new Triangle(index1, index2, index3, mc);
+
+ // 初始化点的状态 用于最后在线采样结果顺序输出未淘汰点
+ points.get(i).prev = points.get(i - 1);
+ points.get(i).next = points.get(i + 1);
+// points.get(i).index = i; // for swab usage
+ }
+
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ }
+
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+
+ int fakeCnt = 0;
+ while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll(); // O(logn)
+
+ // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ if (tri.isDeleted) {
+ fakeCnt--; // 取出了一个被标记删除点
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ continue;
+ }
+
+ // 真正的淘汰点
+ // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ if (debug) {
+ System.out.println("(1) eliminate " + points.get(tri.indices[1]));
+ }
+ points.get(tri.indices[1]).markEliminated();
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
+
+ Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+
+ // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // 前一个三角形连到后一个三角形
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ tri.prev.indices[2] = tri.indices[2];
+
+ double mc;
+ if (prefixSum == null) {
+ mc = myLA_withTimestamps(points,
+ tri.prev.indices[0], tri.prev.indices[2]);
+ } else {
+ mc = myLA_withTimestamps(points,
+ prefixSum, tri.prev.indices[0], tri.prev.indices[2]);
+ }
+ tri.prev.area = mc;
+ if (debug) {
+ System.out.println("(2) updating point on the left "
+ + points.get(tri.prev.indices[1])
+ + ", ranging [" + tri.prev.indices[0] + "," + tri.prev.indices[2] + "]");
+ }
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add来显式重新插入修改后的元素
+ triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+
+ if (tri.next != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+ tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+
+ Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
+
+ if (tri.prev != null) {
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
+ }
+
+ // 2. 处理pi被淘汰引起tri.next被更新的事情
+ tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ tri.next.indices[0] = tri.indices[0];
+
+ double mc;
+ if (prefixSum == null) {
+ mc = myLA_withTimestamps(points,
+ tri.next.indices[0], tri.next.indices[2]);
+ } else {
+ mc = myLA_withTimestamps(points,
+ prefixSum, tri.next.indices[0], tri.next.indices[2]);
+ }
+ tri.next.area = mc;
+ if (debug) {
+ System.out.println("(3) updating point on the right "
+ + points.get(tri.next.indices[1])
+ + ", ranging [" + tri.next.indices[0] + "," + tri.next.indices[2] + "]");
+ }
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ }
+
+ List<Point> onlineSampled = new ArrayList<>();
+ Point start = points.get(0);
+ Point end = points.get(points.size() - 1);
+ while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ }
+ onlineSampled.add(end);
+ return onlineSampled;
+ }
+
+ public static int nextSlidingWindowWithTimestamps(List<Point> points, double maxError, Object[] prefixSum) {
+ if (points.size() <= 1) {
+ return points.size() - 1;
+ }
+ int i = 2; // 从第二个点开始
+ if (prefixSum == null) {
+ while (i < points.size() && myLA_withTimestamps(points, 0, i) < maxError) {
+ i++; // 窗口扩大
+ }
+ } else {
+ while (i < points.size() && myLA_withTimestamps(points, prefixSum, 0, i) < maxError) {
+ i++; // 窗口扩大
+ }
+ }
+ return i - 1; // 从0开始，并且是从0到i-1的左闭右闭
+ }
+
+
+ public static List<Point> swab_framework(List<Point> points, double maxError, int m, boolean debug) {
+ // 在这里给Point添加全局排序idx, 用于全局定位
+ for (int i = 0; i < points.size(); i++) {
+ points.get(i).index = i;
+ }
+
+ if (debug) {
+ System.out.println("data length=" + points.size() + ", max_error=" + maxError + ", m=" + m);
+ }
+ // m在这里的含义是分段数，虽然SWAB不能直接控制分段数，但是也需要这个参数来确定buffer大小
+ int bs = (int) Math.floor((double) points.size() / m * 6); // buffer size
+ int lb = bs / 2, ub = bs * 2;
+ if (debug) {
+ System.out.println("bs=" + bs + ", lb=" + lb + ", ub=" + ub);
+ }
+
+ int buffer_startIdx = 0; // 左闭
+ int buffer_endIdx = bs; // 右开。
+ // 注意虽然joint，但是buffer和remaining points是断开的
+ // buffer: [0:bs] 左闭右开
+ // remaining: [bs:] 左闭
+
+// List<Point> w = points.subList(0, bs); // 浅复制 左闭右开
+// List<Point> remaining_points = points.subList(bs, points.size()); // TODO
+
+ List<Point> segTs = new ArrayList<>(); // TODO
+ segTs.add(points.get(0)); //全局首点
+
+// int base = 0; // global base
+// if (debug) {
+// System.out.println("base=" + base);
+// }
+ boolean needBottomUp = true; // false when only output, no input
+ int reuseIdx = 0; // TODO
+ List<Point> sampled = Collections.emptyList(); // TODO
+ while (true) {
+ if (debug) {
+ System.out.println("#####################################################");
+ System.out.println("buffer data=" + (buffer_endIdx - buffer_startIdx)
+ + ", remaining data=" + (points.size() - buffer_endIdx));
+ System.out.println("reBottomUp:" + needBottomUp);
+ }
+
+ if (needBottomUp) {
+
+ if (debug) {
+ System.out.println(">>>>bottom up on the buffer");
+ }
+ reuseIdx = 0;
+ sampled = seg_bottomUp_maxerror_withTimestamps(
+ points.subList(buffer_startIdx, buffer_endIdx),
+ maxError, null, false);
+ // TODO prefix sum acc
+
+
+ // TODO eBUG also
+// sampled = eBUG.buildEffectiveArea(points.subList(buffer_startIdx, buffer_endIdx), 0, false, m);
+ }
+
+ if (sampled.size() - reuseIdx >= 2) { // 左边输出一个segment
+ if (debug) {
+ System.out.println(">>>>left output a segment:["
+ + sampled.get(0 + reuseIdx).index + "," + sampled.get(1 + reuseIdx) + "]");
+ }
+// Map<String, Integer> firstSegment = segment.get(0);
+// firstSegment.put("lx", firstSegment.get("lx") + base);
+// firstSegment.put("rx", firstSegment.get("rx") + base);
+// sampled.get(1).index += base; // # note that no need to add base later for segment(1)
+ segTs.add(sampled.get(1 + reuseIdx));
+
+ buffer_startIdx = sampled.get(1 + reuseIdx).index;
+
+// if (sampled.size() == 2) {
+// // 更新buffer w
+// buffer_startIdx = buffer_endIdx - 1; // 注意左开右闭
+//// w = w.subList(w.size() - 1, w.size()); // last point in buffer for joint
+//
+// // 更新base
+// // NOTE python start from 0 not 1
+// // NOTE segment(0) rx already add base! i.e., segment(0) rx is already global position
+//// base = sampled.get(1).index;
+// } else {
+// // 更新buffer w
+// int out = sampled.get(1).index - sampled.get(0).index + 1;
+//
+// // 更新base
+// w = w.subList(out - 1, w.size()); // last point in buffer for joint
+// buffer_startIdx = sampled.get(1).index;
+//// base = base + sampled.get(1).index; // 对于joint来说，这个既是第一段的结尾又是第二段的开始位置
+// }
+ }
+
+ if (debug) {
+ System.out.println("buffer data: [" + buffer_startIdx + "," + buffer_endIdx + "]");
+ System.out.println("---------------------------------");
+ }
+ if (buffer_endIdx >= points.size()) { // TODO no more point added into buffer w
+ if (debug) {
+ System.out.println(">>>>no more data remaining");
+ }
+ // Add remaining segments and break if no more input data
+ if (sampled.size() - reuseIdx > 2) {
+ // 注意此时segment并没有把上面“左边输出一个segment”的第一个segment pop出去
+ // 更新base
+ // 这次不需要re-bottomUp了，所以回退到本次分段起点，这样后面的相对位置加上base才是绝对位置
+ // NOTE segments does NOT have their index recalculated by bottom-up to start from 0,
+ // therefore minus the previously added segment[1]["lx"]
+// base = base - sampled.get(1).index;
+ for (int k = 2 + reuseIdx; k < sampled.size(); k++) {
+// Map<String, Integer> seg = segment.get(k);
+// seg.put("lx", seg.get("lx") + base);
+// seg.put("rx", seg.get("rx") + base);
+ // 注意此时segment并没有把上面“左边输出一个segment”的第一个segment pop出去！
+ // 所以要从第二个segment开始继续输出，同时注意python从0开始计数
+// sampled.get(k).index += base;
+ segTs.add(sampled.get(k));
+ }
+ }
+ break;
+ }
+
+ if ((buffer_endIdx - buffer_startIdx) >= lb) {
+ // buffer w上面已经更新了，已经是去除了左边输出的第一个segment之后的剩余的点
+ if (debug) {
+ System.out.println(">>>>no need adding the sliding segment from right because len(w) >= lb");
+ }
+ needBottomUp = false;
+
+ // 更新base
+ // 这次不需要re-bottomUp了，所以回退到本次分段起点，这样后面的相对位置加上base才是绝对位置
+ // NOTE segments does NOT have their index recalculated by bottom-up to start from 0,
+ // therefore minus the previously added segment[1]["lx"]
+// base = base - sampled.get(1).index;
+
+ // 注意此步骤不要在更新base之前执行！！！因为上面更新base时假设“左边输出一个segment”的第一个segment还没有pop
+ // 注意这里真的把上面“左边输出一个segment”的第一个segment pop出去了，
+ // 因为下一轮迭代要直接用剩余的segment
+// segment.remove(0); // Pop the first segment
+// sampled.remove(0); // TODO 这个复杂度？
+ reuseIdx++; // TODO
+ if (debug) {
+ System.out.println("next iteration uses the remaining segment without re-bottomUp, reuseIdx=" + reuseIdx);
+ }
+
+ // # assume lb>1, at least two segments
+ } else {
+ if (debug) {
+ System.out.println(">>>>adding the sliding segment from right because len(w) < lb");
+ }
+ needBottomUp = true; // 加了新的数据进来意味着下一轮里就要re-bottomUp
+
+ while ((buffer_endIdx - buffer_startIdx) < lb && (points.size() - buffer_endIdx) > 0) {
+ // 从0开始，并且rx是从0开始的右闭 [0,rx]一共rx+1个点
+ List<Point> remaining_points = points.subList(buffer_endIdx, points.size());
+ int rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, null); // TODO prefixsum加速
+ if ((buffer_endIdx - buffer_startIdx) + (rx + 1) > ub) {
+ // # avoid more than ub
+ rx = ub - (buffer_endIdx - buffer_startIdx) - 1;
+ }
+ buffer_endIdx += (rx + 1); // TODO
+// w.addAll(remaining_points.subList(0, rx + 1)); // TODO
+// remaining_points = remaining_points.subList(rx + 1, remaining_points.size());
+ if (debug) {
+ System.out.println("input len=" + (rx + 1));
+ }
+ }
+ }
+
+ if (debug) {
+ System.out.println("buffer data: [" + buffer_startIdx + "," + buffer_endIdx + "), "
+ + "remaining data: [" + buffer_endIdx + "," + points.size() + ")");
+ if ((buffer_endIdx - buffer_startIdx) < lb) {
+ System.out.println("warn less");
+ }
+ if ((buffer_endIdx - buffer_startIdx) > ub) {
+ System.out.println("warn more");
+ }
+ }
+ }
+ return segTs;
+ }
+
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 100;
+// Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ long startTime = System.currentTimeMillis();
+ Object[] prefixSum = prefixSum(polyline.getVertices());
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
+
+ double maxError = 1000000000;
+ int m = 10;
+
+ // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
+ startTime = System.currentTimeMillis();
+ List<Point> sampled = swab_framework(polyline.getVertices(), maxError, m, true);
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ for (Point p : sampled) {
+ System.out.println(p);
+ }
+ System.out.println(sampled.size());
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
index 385362b9e6b..5823ca86c3d 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
@@ -15,7 +15,7 @@ public class Test1 {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(10);
- int n = 10000;
+ int n = 20000;
int eParam = 1;
int p = 10;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test4.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test4.java
new file mode 100644
index 00000000000..ada4b47b14e
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test4.java
@@ -0,0 +1,79 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.SWAB.myLA_withTimestamps;
+import static org.apache.iotdb.db.query.eBUG.SWAB.prefixSum;
+
+public class Test4 {
+ // 用于测试使用prefix sum加速L2误差计算的加速效果
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 2000_0000;
+// Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+// try (FileWriter writer = new FileWriter("raw.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
+//
+// // 写入每个点的数据
+// for (int i = 0; i < polyline.size(); i++) {
+// Point point = polyline.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// }
+// System.out.println(polyline.size() + " Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
+// }
+
+ long startTime = System.currentTimeMillis();
+ Object[] prefixSum = prefixSum(polyline.getVertices());
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
+
+ double error;
+
+ List<Double> error1 = new ArrayList<>();
+ startTime = System.currentTimeMillis();
+ for (int lx = 0; lx < polyline.size() / 2; lx += 10000) {
+ error = myLA_withTimestamps(polyline.getVertices(), prefixSum, lx, polyline.size() - 1);
+ System.out.println(lx + "," + error);
+ error1.add(error);
+ }
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to compute L2 error with prefix sum: " + (endTime - startTime) + "ms");
+
+ List<Double> error2 = new ArrayList<>();
+ startTime = System.currentTimeMillis();
+ for (int lx = 0; lx < polyline.size() / 2; lx += 10000) {
+ error = myLA_withTimestamps(polyline.getVertices(), lx, polyline.size() - 1);
+ System.out.println(lx + "," + error);
+ error2.add(error);
+ }
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to compute L2 error without prefix sum: " + (endTime - startTime) + "ms");
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < error1.size(); i++) {
+ writer.println(error1.get(i) + "," + error2.get(i) + "," + (error2.get(i) - error1.get(i)));
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test5.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test5.java
new file mode 100644
index 00000000000..c9ac846a0b2
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test5.java
@@ -0,0 +1,64 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.SWAB.seg_bottomUp_maxerror_withTimestamps;
+
+public class Test5 {
+ // 用于验证java bottomUpMaxError实现正确性
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 2000;
+// Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ long startTime = System.currentTimeMillis();
+ Object[] prefixSum = SWAB.prefixSum(polyline.getVertices());
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
+
+ startTime = System.currentTimeMillis();
+ double maxError = 5000;
+ List<Point> sampled = seg_bottomUp_maxerror_withTimestamps(polyline.getVertices(), maxError, prefixSum, false);
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to : " + (endTime - startTime) + "ms");
+
+ for (Point p : sampled) {
+ System.out.println(p);
+ }
+ System.out.println(sampled.size());
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test6.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test6.java
new file mode 100644
index 00000000000..75f288a061c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test6.java
@@ -0,0 +1,89 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.SWAB.prefixSum;
+import static org.apache.iotdb.db.query.eBUG.SWAB.seg_bottomUp_maxerror_withTimestamps;
+
+public class Test6 {
+ // 用于实测bottomUp-L2-with/without prefix sum acceleration和eBUG算法的耗时比较
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 1000_0000;
+// Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ long startTime = System.currentTimeMillis();
+ Object[] prefixSum = prefixSum(polyline.getVertices());
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
+
+ double maxError = 500000;
+
+ // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
+ startTime = System.currentTimeMillis();
+ List<Point> sampled = seg_bottomUp_maxerror_withTimestamps(polyline.getVertices(), maxError, null, false);
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to bottomUp-L2 without prefix sum acceleration: " + (endTime - startTime) + "ms");
+ System.out.println(sampled.size());
+
+ // 30s worst case O(nlogn)主要就是heap的操作耗时了
+ startTime = System.currentTimeMillis();
+ sampled = seg_bottomUp_maxerror_withTimestamps(polyline.getVertices(), maxError, prefixSum, false);
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to bottomUp-L2 with prefix sum acceleration:" + (endTime - startTime) + "ms");
+ System.out.println(sampled.size());
+
+ // 39s worst-case O(n2)但是实际达不到worst-case所以实际运行不会那么大耗时。
+ // 不过还是会比L2误差的耗时要大，毕竟AD计算比L2更复杂一些虽然都是线性，但是常数大一些
+ startTime = System.currentTimeMillis();
+ sampled = eBUG.buildEffectiveArea(polyline, 1000_0000, false, 3591486);
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to eBUG with e>n-3: " + (endTime - startTime) + "ms");
+ System.out.println(sampled.size());
+
+ // 33s worst case O(nlogn)主要就是heap的操作耗时了
+ startTime = System.currentTimeMillis();
+ sampled = eBUG.buildEffectiveArea(polyline, 0, false, 3591486);
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to eBUG with e=0: " + (endTime - startTime) + "ms");
+ System.out.println(sampled.size());
+
+// for (Point p : sampled) {
+// System.out.println(p);
+// }
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
index 86658e24385..2f76097918e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
@@ -1,204 +1,254 @@
package org.apache.iotdb.db.query.eBUG;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
+import java.util.stream.Stream;
public class Tool {
- // 计算三角形面积
- public static double triArea(Point d0, Point d1, Point d2) {
- double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
- return (dArea > 0.0) ? dArea : -dArea; // abs
- }
-
- // 计算简单多边形（边不交叉）的面积（鞋带公式）
- public static double calculatePolygonArea(List<Point> points) {
- // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
- int n = points.size();
- double area = 0;
- for (int i = 0; i < n; i++) {
- int next = (i + 1) % n;
- area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
+
+ public static Polyline readFromFile(String input, boolean hasHeader, int timeIdx, int valueIdx, int N) {
+ Polyline polyline = new Polyline();
+
+ // Using Files.lines() for memory-efficient line-by-line reading
+ try (Stream<String> lines = Files.lines(Paths.get(input))) {
+ Iterator<String> iterator = lines.iterator();
+ int lineNumber = 0;
+
+ // Skip the header line if necessary
+ if (hasHeader && iterator.hasNext()) {
+ iterator.next(); // Skip the header
+ }
+
+ // Process each line
+ while (iterator.hasNext()) {
+ lineNumber++;
+ String line = iterator.next();
+ String[] columns = line.split(",");
+
+ if (columns.length > Math.max(timeIdx, valueIdx)) {
+ double time;
+ if (timeIdx < 0) {
+ time = lineNumber;
+ } else {
+ time = Double.parseDouble(columns[timeIdx]);
+ }
+ double value = Double.parseDouble(columns[valueIdx]);
+ polyline.addVertex(new Point(time, value));
+ // System.out.println("Line " + lineNumber + " - Time: " + time + ",
+ // Value: " + value);
+ } else {
+ System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
+ }
+ if (N > 0 && lineNumber >= N) {
+ // N>0控制点数，否则N<=0表示读全部数据
+ break;
+ }
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return polyline;
}
- return Math.abs(area) / 2.0;
- }
-
- // 计算两个向量的叉积
- public static double crossProduct(double x1, double y1, double x2, double y2) {
- // >0: (x2,y2)在(x1,y1)的逆时针方向
- // <0: (x2,y2)在(x1,y1)的顺时针方向
- // =0: 平行或共线
- return x1 * y2 - y1 * x2;
- }
-
- // 判断两条线段是否相交并计算交点
- // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
- public static Object[] lineIntersection(Point[] L1, Point[] L2) {
- double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
- double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
-
- // 判断是否相交（叉积）
- double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
- double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
- double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
- double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
-
- // 如果叉积条件满足，表示有交点
- // d1*d2<0意味着P3、P4分别在L12的两边
- // d3*d4<0意味着P1、P2分别在L34的两边
- // 两个同时满足说明有交点
- if (d1 * d2 < 0 && d3 * d4 < 0) {
- double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
- double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
- double x = x1 + t1 * (x2 - x1);
- double y = y1 + t1 * (y2 - y1);
- return new Object[] {true, new Point(x, y)};
+
+ // 计算三角形面积
+ public static double triArea(Point d0, Point d1, Point d2) {
+ double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
+ return (dArea > 0.0) ? dArea : -dArea; // abs
}
- // 检查是否起点或终点重合
- if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
- return new Object[] {true, new Point(x1, y1)};
+ // 计算简单多边形（边不交叉）的面积（鞋带公式）
+ public static double calculatePolygonArea(List<Point> points) {
+ // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
+ int n = points.size();
+ double area = 0;
+ for (int i = 0; i < n; i++) {
+ int next = (i + 1) % n;
+ area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
+ }
+ return Math.abs(area) / 2.0;
}
- if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
- return new Object[] {true, new Point(x2, y2)};
+
+ // 计算两个向量的叉积
+ public static double crossProduct(double x1, double y1, double x2, double y2) {
+ // >0: (x2,y2)在(x1,y1)的逆时针方向
+ // <0: (x2,y2)在(x1,y1)的顺时针方向
+ // =0: 平行或共线
+ return x1 * y2 - y1 * x2;
}
- return new Object[] {false, null};
- }
-
- // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
- public static double total_areal_displacement(
- List<Point> points, List<Point> points2, boolean debug) {
- double totalArea = 0;
- int i = 0, j = 0; // i for points, j for points2
- Point prevIntersection = null;
- int prevI = -1, prevJ = -1;
-
- // List<double[]> intersectionCoords = new ArrayList<>();
- List<Double> areaList = new ArrayList<>();
-
- while (i < points.size() - 1 && j < points2.size() - 1) {
- if (debug) {
- System.out.println("--------- " + i + " " + j + " ------------");
- }
-
- // 当前线段
- Point[] L1 = {points.get(i), points.get(i + 1)};
- Point[] L2 = {points2.get(j), points2.get(j + 1)};
-
- // 判断是否有交点
- Object[] result = lineIntersection(L1, L2);
- boolean isIntersect = (boolean) result[0];
- Point intersection = (Point) result[1];
-
- if (isIntersect) {
- // intersectionCoords.add(intersection);
-
- if (prevIntersection != null) {
- // 构造多边形点集
- List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
- polygon.add(prevIntersection);
- if (debug) {
- System.out.println("- start intersection: " + prevIntersection);
- }
- polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
- // polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i +
- // 1))); // 添加当前线段的点，左闭右开
- if (debug) {
- System.out.println("- one side: " + points.subList(prevI, i + 1));
- }
- polygon.add(intersection);
- if (debug) {
- System.out.println("- end intersection: " + intersection);
- }
- List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
- Collections.reverse(tempPoints2); // 添加另一序列的点
- polygon.addAll(tempPoints2);
- if (debug) {
- System.out.println("- another side: " + tempPoints2);
- }
-
- // double[][] polygonArray = new double[polygon.size()][2];
- // for (int k = 0; k < polygon.size(); k++) {
- // polygonArray[k] = polygon.get(k);
- // }
-
- // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
- double area = calculatePolygonArea(polygon);
- if (debug) {
- System.out.println("Area = " + area);
- }
- totalArea += area;
- areaList.add(area);
+ // 判断两条线段是否相交并计算交点
+ // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
+ public static Object[] lineIntersection(Point[] L1, Point[] L2) {
+ double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
+ double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
+
+ // 判断是否相交（叉积）
+ double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
+ double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
+ double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
+ double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
+
+ // 如果叉积条件满足，表示有交点
+ // d1*d2<0意味着P3、P4分别在L12的两边
+ // d3*d4<0意味着P1、P2分别在L34的两边
+ // 两个同时满足说明有交点
+ if (d1 * d2 < 0 && d3 * d4 < 0) {
+ double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
+ double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
+ double x = x1 + t1 * (x2 - x1);
+ double y = y1 + t1 * (y2 - y1);
+ return new Object[]{true, new Point(x, y)};
}
- prevIntersection = intersection;
- prevI = i + 1;
- prevJ = j + 1;
+ // 检查是否起点或终点重合
+ if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
+ return new Object[]{true, new Point(x1, y1)};
+ }
+ if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
+ return new Object[]{true, new Point(x2, y2)};
+ }
+
+ return new Object[]{false, null};
+ }
+
+ // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
+ public static double total_areal_displacement(
+ List<Point> points, List<Point> points2, boolean debug) {
+ double totalArea = 0;
+ int i = 0, j = 0; // i for points, j for points2
+ Point prevIntersection = null;
+ int prevI = -1, prevJ = -1;
+
+ // List<double[]> intersectionCoords = new ArrayList<>();
+ List<Double> areaList = new ArrayList<>();
+
+ while (i < points.size() - 1 && j < points2.size() - 1) {
+ if (debug) {
+ System.out.println("--------- " + i + " " + j + " ------------");
+ }
+
+ // 当前线段
+ Point[] L1 = {points.get(i), points.get(i + 1)};
+ Point[] L2 = {points2.get(j), points2.get(j + 1)};
+
+ // 判断是否有交点
+ Object[] result = lineIntersection(L1, L2);
+ boolean isIntersect = (boolean) result[0];
+ Point intersection = (Point) result[1];
+
+ if (isIntersect) {
+ // intersectionCoords.add(intersection);
+
+ if (prevIntersection != null) {
+ // 构造多边形点集
+ List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
+ polygon.add(prevIntersection);
+ if (debug) {
+ System.out.println("- start intersection: " + prevIntersection);
+ }
+ polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
+ // polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i +
+ // 1))); // 添加当前线段的点，左闭右开
+ if (debug) {
+ System.out.println("- one side: " + points.subList(prevI, i + 1));
+ }
+ polygon.add(intersection);
+ if (debug) {
+ System.out.println("- end intersection: " + intersection);
+ }
+ List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
+ Collections.reverse(tempPoints2); // 添加另一序列的点
+ polygon.addAll(tempPoints2);
+ if (debug) {
+ System.out.println("- another side: " + tempPoints2);
+ }
+
+ // double[][] polygonArray = new double[polygon.size()][2];
+ // for (int k = 0; k < polygon.size(); k++) {
+ // polygonArray[k] = polygon.get(k);
+ // }
+
+ // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
+ double area = calculatePolygonArea(polygon);
+ if (debug) {
+ System.out.println("Area = " + area);
+ }
+ totalArea += area;
+ areaList.add(area);
+ }
+
+ prevIntersection = intersection;
+ prevI = i + 1;
+ prevJ = j + 1;
+ if (debug) {
+ System.out.println(
+ "This intersection = "
+ + intersection
+ + ", next polygon: side1 = "
+ + prevI
+ + ", side2 = "
+ + prevJ);
+ }
+ }
+
+ int currentI = i; // 临时存储i
+ int currentJ = j; // 临时存储j
+ if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
+ i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
+ j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ } // end of while
+
if (debug) {
- System.out.println(
- "This intersection = "
- + intersection
- + ", next polygon: side1 = "
- + prevI
- + ", side2 = "
- + prevJ);
+ System.out.println(areaList);
}
- }
-
- int currentI = i; // 临时存储i
- int currentJ = j; // 临时存储j
- if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
- i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
- }
- if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
- j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
- }
- } // end of while
-
- if (debug) {
- System.out.println(areaList);
+
+ return totalArea;
}
- return totalArea;
- }
-
- // 测试方法
- public static void main(String[] args) {
- // 示例数据
- List<Point> points = new ArrayList<>();
- points.add(new Point(0, 0));
- points.add(new Point(1, 2));
- points.add(new Point(2, -20));
- points.add(new Point(3, 0));
- points.add(new Point(4, -1));
- points.add(new Point(5, -1.5));
- points.add(new Point(6, 0));
-
- List<Point> points2 = new ArrayList<>();
- // points2.add(points.get(0));
- // points2.add(points.get(3));
- // points2.add(points.get(6));
- points2.add(new Point(1, -10));
- points2.add(new Point(3, 0));
-
- double area = total_areal_displacement(points, points2, true);
- System.out.println("Total area: " + area);
-
- // points = new ArrayList<>();
- // points.add(new Point(0, 0));
- // points.add(new Point(1, 2));
- // points.add(new Point(1.5, -10));
- // double area = calculatePolygonArea(points);
- // System.out.println(area);
- //
- // Point[] L1 = new Point[2];
- // L1[0] = new Point(1, 2);
- // L1[1] = new Point(2, -2);
- // Point[] L2 = new Point[2];
- // L2[0] = new Point(0, 0);
- // L2[1] = new Point(3, 0);
- // Object[] res = lineIntersection(L1, L2);
- // System.out.println(res[1]);
- }
+ // 测试方法
+ public static void main(String[] args) {
+ // 示例数据
+ List<Point> points = new ArrayList<>();
+ points.add(new Point(0, 0));
+ points.add(new Point(1, 2));
+ points.add(new Point(2, -20));
+ points.add(new Point(3, 0));
+ points.add(new Point(4, -1));
+ points.add(new Point(5, -1.5));
+ points.add(new Point(6, 0));
+
+ List<Point> points2 = new ArrayList<>();
+ // points2.add(points.get(0));
+ // points2.add(points.get(3));
+ // points2.add(points.get(6));
+ points2.add(new Point(1, -10));
+ points2.add(new Point(3, 0));
+
+ double area = total_areal_displacement(points, points2, true);
+ System.out.println("Total area: " + area);
+
+ // points = new ArrayList<>();
+ // points.add(new Point(0, 0));
+ // points.add(new Point(1, 2));
+ // points.add(new Point(1.5, -10));
+ // double area = calculatePolygonArea(points);
+ // System.out.println(area);
+ //
+ // Point[] L1 = new Point[2];
+ // L1[0] = new Point(1, 2);
+ // L1[1] = new Point(2, -2);
+ // Point[] L2 = new Point[2];
+ // L2[0] = new Point(0, 0);
+ // L2[1] = new Point(3, 0);
+ // Object[] res = lineIntersection(L1, L2);
+ // System.out.println(res[1]);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 201f99d2313..cc86a3fe8f9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -28,389 +28,363 @@ import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
public class eBUG {
- public static List<Point> findEliminated(Polyline lineToSimplify, int pa_idx, int pb_idx) {
- // pa: left adjacent non-eliminated point of pi
- // pb: right adjacent non-eliminated point of pi
- // return a list of points, T'_max{0,k-e}[a:b] order by time in ascending order
-
- // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！至于其他全局更早淘汰的点则不一定位于a:b之间
- // pa,xxx,pi,xxx,pb
- // when e=0，遍历点数就是3
- // when e>0，遍历点数大于等于4、小于等于e+3
-
- List<Point> res = new ArrayList<>();
- Point start = lineToSimplify.get(pa_idx);
- Point end = lineToSimplify.get(pb_idx);
- // int cnt = 0;
- while (start
- != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- res.add(start);
- start = start.next; // when e=0, only traversing three points pa pi pb
- // cnt += 1;
- }
- res.add(end);
- // cnt += 1;
- // System.out.println(cnt); // 3<=cnt<=e+3
-
- return res;
- }
-
- public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
- // precomputation mode
- return buildEffectiveArea(lineToSimplify, e, debug, 0);
- }
-
- public static List<Point> buildEffectiveArea(
- Polyline lineToSimplify, int e, boolean debug, int m) {
- if (e < 0) {
- throw new IllegalArgumentException("Parameter e must be non-negative.");
- }
+ public static List<Point> findEliminated(Polyline lineToSimplify, int pa_idx, int pb_idx) {
+ // pa: left adjacent non-eliminated point of pi
+ // pb: right adjacent non-eliminated point of pi
+ // return a list of points, T'_max{0,k-e}[a:b] order by time in ascending order
+
+ // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！至于其他全局更早淘汰的点则不一定位于a:b之间
+ // pa,xxx,pi,xxx,pb
+ // when e=0，遍历点数就是3
+ // when e>0，遍历点数大于等于4、小于等于e+3
+
+ List<Point> res = new ArrayList<>();
+ Point start = lineToSimplify.get(pa_idx);
+ Point end = lineToSimplify.get(pb_idx);
+ // int cnt = 0;
+ while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ res.add(start);
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ // cnt += 1;
+ }
+ res.add(end);
+ // cnt += 1;
+ // System.out.println(cnt); // 3<=cnt<=e+3
- if (m > 2) {
- System.out.println(
- "online sampling mode, "
- + "returning "
- + m
- + " sampled points sorted by time in ascending order");
- } else {
- System.out.println(
- "offline precomputation mode, "
- + "returning each point sorted by dominated significance (DS) in ascending order");
+ return res;
}
- // List<Point> results = lineToSimplify.getVertices(); // 浅复制
- // TODO 预计算结果改成按照bottom-up逐点淘汰顺序（DS递增）排列而不是按照时间戳，这样省去在线时对DS排序的过程
- // add的是Point引用，所以没有多用一倍的空间
- List<Point> resultsBottomUpEliminated = new ArrayList<>();
-
- // 存储的是点的引用，这样可以修改原来序列里点的淘汰状态
- // 存储的是距离当前最新状态的滞后的尚未施加、待施加的e个淘汰点
- LinkedList<Point> laggedEliminatedPoints = new LinkedList<>();
-
- int total = lineToSimplify.size();
- if (total < 3) {
- return lineToSimplify.getVertices(); // 不足 3 个点无法形成三角形
+ public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
+ // precomputation mode
+ return buildEffectiveArea(lineToSimplify, e, debug, 0);
}
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- lineToSimplify.get(0).prev = null;
- lineToSimplify.get(0).next = lineToSimplify.get(1);
- lineToSimplify.get(total - 1).next = null;
- lineToSimplify.get(total - 1).prev = lineToSimplify.get(total - 2);
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- double area =
- triArea(
- lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
- triangles[i - 1] = new Triangle(index1, index2, index3, area);
-
- // 初始化点的状态 for eBUG usage
- lineToSimplify.get(i).prev = lineToSimplify.get(i - 1);
- lineToSimplify.get(i).next = lineToSimplify.get(i + 1);
+ public static List<Point> buildEffectiveArea(List<Point> points, int e, boolean debug) {
+ // precomputation mode
+ Polyline polyline = new Polyline();
+ polyline.setVertices(points);
+ return buildEffectiveArea(polyline, e, debug, 0);
}
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ public static List<Point> buildEffectiveArea(List<Point> points, int e, boolean debug, int m) {
+ // precomputation mode
+ Polyline polyline = new Polyline();
+ polyline.setVertices(points);
+ return buildEffectiveArea(polyline, e, debug, m);
}
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap =
- new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
-
- double previousEA = -1;
- // while (!triangleHeap.isEmpty()) { // TODO 注意triangleHeap里装的是non-terminal point对应的三角形
- // TODO 在线采样m个点，也就是说最后留下m-2个non-terminal point
- // 注意：triangleHeap里装的包括了标记删除的点！所以triangleHeap.size()不是真正的留下的未被淘汰点数！
- // 因此需要用fakeCnt来统计heap里的非真实点数，这样triangleHeap.size()-fakeCnt就是真正的留下的未被淘汰点数
- int remainNonTerminalPoint = Math.max(0, m - 2);
- int fakeCnt = 0;
- while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll(); // O(logn)
-
- // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
- // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
- if (tri.isDeleted) {
- fakeCnt--; // 取出了一个被标记删除点
- if (debug) {
- System.out.println(
- ">>>bottom-up, remaining "
- + triangleHeap.size()
- + " triangles (including those marked for removal)");
- }
- continue;
- }
-
- // 真正的淘汰点
- // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
- if (debug) {
- System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
- }
- if (e > 0) {
- if (laggedEliminatedPoints.size() == e) {
- // 已经有e个滞后淘汰点了，为了把最近淘汰点加入，得先把最老的淘汰点施加上去
- Point removedPoint = laggedEliminatedPoints.removeFirst(); // 取出最早加入的淘汰点 复杂度1
- removedPoint.markEliminated(); // 注意是引用，所以改的内容后面可以看到
- }
- laggedEliminatedPoints.addLast(lineToSimplify.get(tri.indices[1])); // 加入最新的淘汰点，滞后在这里先不施加
- } else { // e=0 没有滞后 立即标记删除 T'_k
- lineToSimplify.get(tri.indices[1]).markEliminated();
- }
- if (debug) {
- System.out.println(
- "the e most recently eliminated points (lagged):" + laggedEliminatedPoints);
- }
-
- // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
- if (tri.area > previousEA) {
- previousEA = tri.area;
- }
- // results.get(tri.indices[1]).z = previousEA; // dominated significance
- lineToSimplify.get(tri.indices[1]).z = previousEA; // dominated significance
- resultsBottomUpEliminated.add(
- lineToSimplify.get(tri.indices[1])); // TODO add的是Point引用，所以没有多用一倍的空间
- if (debug) {
- System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
- }
-
- // 更新相邻三角形
- if (tri.prev != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-
- Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
- // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
-
- // 2. 处理pi被淘汰引起tri.prev被更新的事情
- // 前一个三角形连到后一个三角形
- tri.prev.next =
- tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
- tri.prev.indices[2] = tri.indices[2];
-
- // e parameter
- List<Point> pointsForSig =
- findEliminated(lineToSimplify, tri.prev.indices[0], tri.prev.indices[2]);
- if (debug) {
- System.out.println(
- "(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
- System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- double sig = total_areal_displacement(pointsForSig, baseLine, false);
- tri.prev.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri =
- triArea(
- lineToSimplify.get(tri.prev.indices[0]),
- lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2]));
- System.out.println(
- "\t"
- + "tri="
- + tmpTri
- + ", "
- + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
+ public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug, int m) {
+ if (e < 0) {
+ throw new IllegalArgumentException("Parameter e must be non-negative.");
}
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add来显式重新插入修改后的元素
- triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
-
- if (tri.next != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+ if (m > 2) {
+ System.out.println("online sampling mode, " + "returning " + m + " sampled points sorted by time in ascending order");
+ } else {
+ System.out.println("offline precomputation mode, " + "returning each point sorted by dominated significance (DS) in ascending order");
+ }
- // 1. 处理旧的tri.next被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
- tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+ // List<Point> results = lineToSimplify.getVertices(); // 浅复制
+ // TODO 预计算结果改成按照bottom-up逐点淘汰顺序（DS递增）排列而不是按照时间戳，这样省去在线时对DS排序的过程
+ // add的是Point引用，所以没有多用一倍的空间
+ List<Point> resultsBottomUpEliminated = new ArrayList<>();
- Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
- // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
+ // 存储的是点的引用，这样可以修改原来序列里点的淘汰状态
+ // 存储的是距离当前最新状态的滞后的尚未施加、待施加的e个淘汰点
+ LinkedList<Point> laggedEliminatedPoints = new LinkedList<>();
- if (tri.prev != null) {
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
+ int total = lineToSimplify.size();
+ if (total < 3) {
+ return lineToSimplify.getVertices(); // 不足 3 个点无法形成三角形
}
- // 2. 处理pi被淘汰引起tri.next被更新的事情
- tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
- tri.next.indices[0] = tri.indices[0];
-
- // e parameter
- List<Point> pointsForSig =
- findEliminated(lineToSimplify, tri.next.indices[0], tri.next.indices[2]);
- if (debug) {
- System.out.println(
- "(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
- System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ lineToSimplify.get(0).prev = null;
+ lineToSimplify.get(0).next = lineToSimplify.get(1);
+ lineToSimplify.get(total - 1).next = null;
+ lineToSimplify.get(total - 1).prev = lineToSimplify.get(total - 2);
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+ double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
+ triangles[i - 1] = new Triangle(index1, index2, index3, area);
+
+ // 初始化点的状态 for eBUG usage 用于快速按照按照时间顺序排列的滞后k个淘汰点状态下位于pa~pb的点；也用于最后在线采样结果顺序输出未淘汰点
+ lineToSimplify.get(i).prev = lineToSimplify.get(i - 1);
+ lineToSimplify.get(i).next = lineToSimplify.get(i + 1);
}
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- double sig = total_areal_displacement(pointsForSig, baseLine, false);
- tri.next.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri =
- triArea(
- lineToSimplify.get(tri.next.indices[0]),
- lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2]));
- System.out.println(
- "\t"
- + "tri="
- + tmpTri
- + ", "
- + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
+
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
}
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
- triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
- if (debug) {
- System.out.println(
- ">>>bottom-up, remaining "
- + triangleHeap.size()
- + " triangles (including those marked for removal)");
- }
- }
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+
+ double previousEA = -1;
+ // while (!triangleHeap.isEmpty()) { // TODO 注意triangleHeap里装的是non-terminal point对应的三角形
+ // TODO 在线采样m个点，也就是说最后留下m-2个non-terminal point
+ // 注意：triangleHeap里装的包括了标记删除的点！所以triangleHeap.size()不是真正的留下的未被淘汰点数！
+ // 因此需要用fakeCnt来统计heap里的非真实点数，这样triangleHeap.size()-fakeCnt就是真正的留下的未被淘汰点数
+ int remainNonTerminalPoint = Math.max(0, m - 2);
+ int fakeCnt = 0;
+ while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll(); // O(logn)
+
+ // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ if (tri.isDeleted) {
+ fakeCnt--; // 取出了一个被标记删除点
+ if (debug) {
+ System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
+ }
+ continue;
+ }
+
+ // 真正的淘汰点
+ // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ if (debug) {
+ System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
+ }
+ if (e > 0) {
+ if (laggedEliminatedPoints.size() == e) {
+ // 已经有e个滞后淘汰点了，为了把最近淘汰点加入，得先把最老的淘汰点施加上去
+ Point removedPoint = laggedEliminatedPoints.removeFirst(); // 取出最早加入的淘汰点 复杂度1
+ removedPoint.markEliminated(); // 注意是引用，所以改的内容后面可以看到
+ }
+ laggedEliminatedPoints.addLast(lineToSimplify.get(tri.indices[1])); // 加入最新的淘汰点，滞后在这里先不施加
+ } else { // e=0 没有滞后 立即标记删除 T'_k
+ lineToSimplify.get(tri.indices[1]).markEliminated();
+ }
+ if (debug) {
+ System.out.println("the e most recently eliminated points (lagged):" + laggedEliminatedPoints);
+ }
+
+ // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
+ if (tri.area > previousEA) {
+ previousEA = tri.area;
+ }
+ // results.get(tri.indices[1]).z = previousEA; // dominated significance
+ lineToSimplify.get(tri.indices[1]).z = previousEA; // dominated significance
+ resultsBottomUpEliminated.add(lineToSimplify.get(tri.indices[1])); // TODO add的是Point引用，所以没有多用一倍的空间
+ if (debug) {
+ System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
+ }
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
+
+ Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+
+ // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // 前一个三角形连到后一个三角形
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ tri.prev.indices[2] = tri.indices[2];
+
+ // e parameter
+ double sig;
+ if (e > 0) {
+ List<Point> pointsForSig = findEliminated(lineToSimplify, tri.prev.indices[0], tri.prev.indices[2]);
+ if (debug) {
+ System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
+ System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
+ for (Point point : pointsForSig) {
+ System.out.println("\t" + point);
+ }
+ }
+ List<Point> baseLine = new ArrayList<>();
+ baseLine.add(pointsForSig.get(0));
+ baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
+ sig = total_areal_displacement(pointsForSig, baseLine, false);
+ } else { // 直接用三角形面积
+ sig = triArea(lineToSimplify.get(tri.prev.indices[0]), lineToSimplify.get(tri.prev.indices[1]), lineToSimplify.get(tri.prev.indices[2]));
+ }
+ tri.prev.area = sig;
+
+ if (debug) {
+ System.out.println("sig=" + sig);
+ double tmpTri = triArea(lineToSimplify.get(tri.prev.indices[0]), lineToSimplify.get(tri.prev.indices[1]), lineToSimplify.get(tri.prev.indices[2]));
+ System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
+ }
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add来显式重新插入修改后的元素
+ triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+
+ if (tri.next != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+ tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+
+ Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
+
+ if (tri.prev != null) {
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
+ }
+
+ // 2. 处理pi被淘汰引起tri.next被更新的事情
+ tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ tri.next.indices[0] = tri.indices[0];
+
+ // e parameter
+ double sig;
+ if (e > 0) {
+ List<Point> pointsForSig = findEliminated(lineToSimplify, tri.next.indices[0], tri.next.indices[2]);
+ if (debug) {
+ System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
+ System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
+ for (Point point : pointsForSig) {
+ System.out.println("\t" + point);
+ }
+ }
+ List<Point> baseLine = new ArrayList<>();
+ baseLine.add(pointsForSig.get(0));
+ baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
+ sig = total_areal_displacement(pointsForSig, baseLine, false);
+ } else { // 直接用三角形面积
+ sig = triArea(lineToSimplify.get(tri.next.indices[0]), lineToSimplify.get(tri.next.indices[1]), lineToSimplify.get(tri.next.indices[2]));
+ }
+ tri.next.area = sig;
+
+ if (debug) {
+ System.out.println("sig=" + sig);
+ double tmpTri = triArea(lineToSimplify.get(tri.next.indices[0]), lineToSimplify.get(tri.next.indices[1]), lineToSimplify.get(tri.next.indices[2]));
+ System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
+ }
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+ if (debug) {
+ System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
+ }
+ }
- if (m > 2) { // online sampling mode
- // 把滞后的淘汰点施加上去，然后返回在线采样结果（也就是返回剩余未被淘汰的点）
- for (Point p : laggedEliminatedPoints) {
- p.markEliminated();
- if (debug) {
- System.out.println("apply lagged elimination of " + p);
+ if (m > 2) { // online sampling mode
+ // 把滞后的淘汰点施加上去，然后返回在线采样结果（也就是返回剩余未被淘汰的点）
+ for (Point p : laggedEliminatedPoints) {
+ p.markEliminated();
+ if (debug) {
+ System.out.println("apply lagged elimination of " + p);
+ }
+ }
+ List<Point> onlineSampled = new ArrayList<>();
+ Point start = lineToSimplify.get(0);
+ Point end = lineToSimplify.get(lineToSimplify.size() - 1);
+ while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ }
+ onlineSampled.add(end);
+ return onlineSampled;
+ } else { // offline precomputation mode, for precomputing the dominated significance of each
+ // point
+ // return results; // 注意这就是lineToSimplify.getVertices()
+ // TODO
+ resultsBottomUpEliminated.add(lineToSimplify.get(0)); // 全局首点
+ resultsBottomUpEliminated.add(lineToSimplify.get(lineToSimplify.size() - 1)); // 全局尾点
+ return resultsBottomUpEliminated;
}
- }
- List<Point> onlineSampled = new ArrayList<>();
- Point start = lineToSimplify.get(0);
- Point end = lineToSimplify.get(lineToSimplify.size() - 1);
- while (start
- != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
- start = start.next; // when e=0, only traversing three points pa pi pb
- }
- onlineSampled.add(end);
- return onlineSampled;
- } else { // offline precomputation mode, for precomputing the dominated significance of each
- // point
- // return results; // 注意这就是lineToSimplify.getVertices()
- // TODO
- resultsBottomUpEliminated.add(lineToSimplify.get(0)); // 全局首点
- resultsBottomUpEliminated.add(lineToSimplify.get(lineToSimplify.size() - 1)); // 全局尾点
- return resultsBottomUpEliminated;
}
- }
-
- public static void main(String[] args) {
- // 实验：预计算耗时关于两个参数e、n的变化规律，是否符合复杂度理论建模
-
- // int eParam = 0;
- // int[] eParamList = {0, 0, 1, 2, 3, 4, 5, 10, 14, 15, 16, 20, 30, 40, 50, 100, 200,
- // 500, 1000, 2000, 5000,
- // 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000, 300_0000,
- // 500_0000, 800_0000, 1000_0000,
- // 1500_0000, 2000_0000, 2500_0000, 3000_0000};
- int[] eParamList = {100_0000};
- for (int eParam : eParamList) {
- try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
- for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
- // TODO 注意 要有一个点是n=300w
-
- Polyline polyline = new Polyline();
- int seed = 1;
- Random rand = new Random(seed); // TODO 注意seed在每轮里重设
- for (int i = 0; i < n; i++) {
- double v = rand.nextInt(1000000);
- polyline.addVertex(new Point(i, v));
- }
-
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false, 0);
- long endTime = System.currentTimeMillis();
-
- System.out.println(
- "n="
- + n
- + ", e="
- + eParam
- + ", Time taken to reduce points: "
- + (endTime - startTime)
- + "ms");
- writer.println(n + "," + eParam + "," + (endTime - startTime));
- System.out.println(results.size());
-
- // // clear
- // polyline.clear();
- // results.clear();
- // polyline = null;
- // results = null;
- // System.gc();
+
+ public static void main(String[] args) {
+ // 实验：预计算耗时关于两个参数e、n的变化规律，是否符合复杂度理论建模
+
+ // int eParam = 0;
+ // int[] eParamList = {0, 0, 1, 2, 3, 4, 5, 10, 14, 15, 16, 20, 30, 40, 50, 100, 200,
+ // 500, 1000, 2000, 5000,
+ // 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000, 300_0000,
+ // 500_0000, 800_0000, 1000_0000,
+ // 1500_0000, 2000_0000, 2500_0000, 3000_0000};
+ int[] eParamList = {100_0000};
+ for (int eParam : eParamList) {
+ try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
+ for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
+ // TODO 注意 要有一个点是n=300w
+
+ Polyline polyline = new Polyline();
+ int seed = 1;
+ Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+ for (int i = 0; i < n; i++) {
+ double v = rand.nextInt(1000000);
+ polyline.addVertex(new Point(i, v));
+ }
+
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false, 0);
+ long endTime = System.currentTimeMillis();
+
+ System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+ writer.println(n + "," + eParam + "," + (endTime - startTime));
+ System.out.println(results.size());
+
+ // // clear
+ // polyline.clear();
+ // results.clear();
+ // polyline = null;
+ // results = null;
+ // System.gc();
+ }
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
}
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
- }
- }
- // int n = 300_0000;
- //
- // int seed = 1;
- // Random rand = new Random(seed); // TODO 注意seed在每轮里重设
- // Polyline polyline = new Polyline(); // 注意：如果是固定n变化e实验，这个数据要一次性生成，不然每次随机不一样
- // for (int i = 0; i < n; i++) {
- // double v = rand.nextInt(1000000);
- // polyline.addVertex(new Point(i, v)); //
- // point的状态会在buildEffectiveArea里重置的，所以在下面的e遍历里可以复用这一份数据
- // }
- //
- // try (PrintWriter writer = new PrintWriter(new File("exp_varyE_n" + n + ".csv"))) {
- // int[] eParamList = {0, 1, 2, 10, 1000, 10000, 10_0000, 30_0000, 50_0000, 60_0000,
- // 80_0000, 100_0000, 150_0000, 200_0000};
- // for (int eParam : eParamList) {
- // eParam *= 3; // TODO 注意 因为n=300w而不是100w
- //
- // long startTime = System.currentTimeMillis();
- // List<Point> results = buildEffectiveArea(polyline, eParam, false);
- // long endTime = System.currentTimeMillis();
- //
- // System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce
- // points: " + (endTime - startTime) + "ms");
- // writer.println(n + "," + eParam + "," + (endTime - startTime));
- // System.out.println(results.size());
- //
- // // clear 注意不能把原始数据清除，还要接着用
- //// System.gc();
- // }
- // } catch (FileNotFoundException e) {
- // throw new RuntimeException(e);
- // }
-
- System.out.println("finish");
- }
+ // int n = 300_0000;
+ //
+ // int seed = 1;
+ // Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+ // Polyline polyline = new Polyline(); // 注意：如果是固定n变化e实验，这个数据要一次性生成，不然每次随机不一样
+ // for (int i = 0; i < n; i++) {
+ // double v = rand.nextInt(1000000);
+ // polyline.addVertex(new Point(i, v)); //
+ // point的状态会在buildEffectiveArea里重置的，所以在下面的e遍历里可以复用这一份数据
+ // }
+ //
+ // try (PrintWriter writer = new PrintWriter(new File("exp_varyE_n" + n + ".csv"))) {
+ // int[] eParamList = {0, 1, 2, 10, 1000, 10000, 10_0000, 30_0000, 50_0000, 60_0000,
+ // 80_0000, 100_0000, 150_0000, 200_0000};
+ // for (int eParam : eParamList) {
+ // eParam *= 3; // TODO 注意 因为n=300w而不是100w
+ //
+ // long startTime = System.currentTimeMillis();
+ // List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ // long endTime = System.currentTimeMillis();
+ //
+ // System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce
+ // points: " + (endTime - startTime) + "ms");
+ // writer.println(n + "," + eParam + "," + (endTime - startTime));
+ // System.out.println(results.size());
+ //
+ // // clear 注意不能把原始数据清除，还要接着用
+ //// System.gc();
+ // }
+ // } catch (FileNotFoundException e) {
+ // throw new RuntimeException(e);
+ // }
+
+ System.out.println("finish");
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_Build.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_Build.java
index 94713f71ec8..fd8c6399c8f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_Build.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_Build.java
@@ -3,137 +3,137 @@ package org.apache.iotdb.db.query.eBUG;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.Iterator;
import java.util.List;
-import java.util.stream.Stream;
import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
public class eBUG_Build {
- // 输入一条时间序列 t,v
- // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
- // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
- public static void main(String[] args) {
- if (args.length < 5) {
- System.out.println(
- "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,eParam,(outDir)");
- }
- String input = args[0];
- boolean hasHeader = Boolean.parseBoolean(args[1]);
- int timeIdx = Integer.parseInt(args[2]);
- int valueIdx = Integer.parseInt(args[3]);
- int eParam = Integer.parseInt(args[4]);
- String outDir;
- if (args.length > 5) {
- outDir = args[5]; // 表示输出文件保存到指定的文件夹
- } else {
- outDir = null; // 表示输出文件保存到input文件所在的文件夹
- }
- String outputFile = generateOutputFileName(input, eParam, outDir);
-
- // 打印信息供用户确认
- System.out.println("Input file: " + input);
- System.out.println("Has header: " + hasHeader);
- System.out.println("Time index: " + timeIdx);
- System.out.println("Value index: " + valueIdx);
- System.out.println("eParam: " + eParam);
- System.out.println("outDir: " + outDir);
- System.out.println("Output file: " + outputFile);
-
- // 开始运算
- Polyline polyline = new Polyline();
-
- // Using Files.lines() for memory-efficient line-by-line reading
- try (Stream<String> lines = Files.lines(Paths.get(input))) {
- Iterator<String> iterator = lines.iterator();
- int lineNumber = 0;
-
- // Skip the header line if necessary
- if (hasHeader && iterator.hasNext()) {
- iterator.next(); // Skip the header
- }
-
- // Process each line
- while (iterator.hasNext()) {
- lineNumber++;
- String line = iterator.next();
- String[] columns = line.split(",");
-
- if (columns.length > Math.max(timeIdx, valueIdx)) {
- double time;
- if (timeIdx < 0) {
- time = lineNumber;
- } else {
- time = Double.parseDouble(columns[timeIdx]);
- }
- double value = Double.parseDouble(columns[valueIdx]);
- polyline.addVertex(new Point(time, value));
- // System.out.println("Line " + lineNumber + " - Time: " + time + ",
- // Value: " + value);
+ // 输入一条时间序列 t,v
+ // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
+ // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
+ public static void main(String[] args) {
+ if (args.length < 6) {
+ System.out.println(
+ "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,eParam,(outDir)");
+ }
+ String input = args[0];
+ boolean hasHeader = Boolean.parseBoolean(args[1]);
+ int timeIdx = Integer.parseInt(args[2]);
+ int valueIdx = Integer.parseInt(args[3]);
+ int N = Integer.parseInt(args[4]); // N<=0表示读全部行，N>0表示读最多N行
+ int eParam = Integer.parseInt(args[5]);
+ String outDir;
+ if (args.length > 6) {
+ outDir = args[6]; // 表示输出文件保存到指定的文件夹
} else {
- System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
+ outDir = null; // 表示输出文件保存到input文件所在的文件夹
+ }
+ String outputFile = generateOutputFileName(input, eParam, outDir);
+
+ // 打印信息供用户确认
+ System.out.println("Input file: " + input);
+ System.out.println("Has header: " + hasHeader);
+ System.out.println("Time index: " + timeIdx);
+ System.out.println("Value index: " + valueIdx);
+ System.out.println("N: " + N);
+ System.out.println("eParam: " + eParam);
+ System.out.println("outDir: " + outDir);
+ System.out.println("Output file: " + outputFile);
+
+ // 开始运算
+ Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+// Polyline polyline = new Polyline();
+//
+// // Using Files.lines() for memory-efficient line-by-line reading
+// try (Stream<String> lines = Files.lines(Paths.get(input))) {
+// Iterator<String> iterator = lines.iterator();
+// int lineNumber = 0;
+//
+// // Skip the header line if necessary
+// if (hasHeader && iterator.hasNext()) {
+// iterator.next(); // Skip the header
+// }
+//
+// // Process each line
+// while (iterator.hasNext()) {
+// lineNumber++;
+// String line = iterator.next();
+// String[] columns = line.split(",");
+//
+// if (columns.length > Math.max(timeIdx, valueIdx)) {
+// double time;
+// if (timeIdx < 0) {
+// time = lineNumber;
+// } else {
+// time = Double.parseDouble(columns[timeIdx]);
+// }
+// double value = Double.parseDouble(columns[valueIdx]);
+// polyline.addVertex(new Point(time, value));
+// // System.out.println("Line " + lineNumber + " - Time: " + time + ",
+// // Value: " + value);
+// } else {
+// System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
+// }
+// }
+// } catch (IOException e) {
+// e.printStackTrace();
+// }
+
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false); // precomputation mode
+ long endTime = System.currentTimeMillis();
+
+ // System.out.println("result point number: " + results.size()); // precomputation
+ // mode所以自然是等于n个点
+ System.out.println(
+ "n="
+ + polyline.getVertices().size()
+ + ", e="
+ + eParam
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+
+ // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
+ // 写入表头
+ writer.write("z,x,y");
+ writer.newLine();
+
+ // 写入数据行，按顺序 z, x, y
+ for (Point point : results) {
+ writer.write(point.z + "," + point.x + "," + point.y);
+ writer.newLine();
+ }
+
+ System.out.println("CSV file written successfully to " + outputFile);
+ } catch (IOException e) {
+ e.printStackTrace();
}
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false); // precomputation mode
- long endTime = System.currentTimeMillis();
-
- // System.out.println("result point number: " + results.size()); // precomputation
- // mode所以自然是等于n个点
- System.out.println(
- "n="
- + polyline.getVertices().size()
- + ", e="
- + eParam
- + ", Time taken to reduce points: "
- + (endTime - startTime)
- + "ms");
-
- // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
- // 写入表头
- writer.write("z,x,y");
- writer.newLine();
-
- // 写入数据行，按顺序 z, x, y
- for (Point point : results) {
- writer.write(point.z + "," + point.x + "," + point.y);
- writer.newLine();
- }
-
- System.out.println("CSV file written successfully to " + outputFile);
- } catch (IOException e) {
- e.printStackTrace();
}
- }
- // Method to generate the output file name based on input path
- private static String generateOutputFileName(String inputFilePath, int e, String outDir) {
- // Get the file name without extension
- Path path = Paths.get(inputFilePath);
- String fileNameWithoutExtension = path.getFileName().toString();
- String name = fileNameWithoutExtension.substring(0, fileNameWithoutExtension.lastIndexOf('.'));
+ // Method to generate the output file name based on input path
+ private static String generateOutputFileName(String inputFilePath, int e, String outDir) {
+ // Get the file name without extension
+ Path path = Paths.get(inputFilePath);
+ String fileNameWithoutExtension = path.getFileName().toString();
+ String name = fileNameWithoutExtension.substring(0, fileNameWithoutExtension.lastIndexOf('.'));
- // Get the file extension
- String extension =
- path.getFileName().toString().substring(fileNameWithoutExtension.lastIndexOf('.'));
+ // Get the file extension
+ String extension =
+ path.getFileName().toString().substring(fileNameWithoutExtension.lastIndexOf('.'));
- // Create output file path by appending '-ds' before the extension
- String outputFile = name + "-ds-e" + e + extension;
+ // Create output file path by appending '-ds' before the extension
+ String outputFile = name + "-ds-e" + e + extension;
- if (outDir == null) { // 表示使用input文件所在的文件夹
- // Handle path compatibility for different operating systems (Linux/Windows)
- return path.getParent().resolve(outputFile).toString();
- } else {
- Path out = Paths.get(outDir);
- return out.resolve(outputFile).toString();
+ if (outDir == null) { // 表示使用input文件所在的文件夹
+ // Handle path compatibility for different operating systems (Linux/Windows)
+ return path.getParent().resolve(outputFile).toString();
+ } else {
+ Path out = Paths.get(outDir);
+ return out.resolve(outputFile).toString();
+ }
}
- }
}
(iotdb) 17/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cb9ca0b06f855fe056004f111bc578668e48841b
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jan 22 15:14:17 2025 +0800
add
---
.../java/org/apache/iotdb/db/query/eBUG/Test1.java | 80 ++++
.../db/query/eBUG/TimestampPriorityQueue.java | 121 ------
.../iotdb/db/query/eBUG/Visval_standard.java | 263 -------------
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 24 +-
.../eBUG/eBUG_sortedEliminatedDeprecated.java | 406 ---------------------
.../eBUG/eBUG_sortedEliminatedDeprecated_tmp.java | 398 --------------------
.../org/apache/iotdb/db/query/eBUG/eBUG_tmp.java | 397 --------------------
7 files changed, 92 insertions(+), 1597 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
new file mode 100644
index 00000000000..56e73cc17e0
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
@@ -0,0 +1,80 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
+
+public class Test1 {
+ public static void main(String[] args) {
+ Polyline polyline = new Polyline();
+ List<Polyline> polylineList = new ArrayList<>();
+ Random rand = new Random(1);
+ int n = 10000;
+
+ int p = 10;
+ for (int i = 0; i < n; i += p) {
+ Polyline polylineBatch = new Polyline();
+ for (int j = i; j < Math.min(i + p, n); j++) {
+ double v = rand.nextInt(1000000);
+
+ polyline.addVertex(new Point(j, v));
+
+ polylineBatch.addVertex(new Point(j, v));
+ }
+ polylineList.add(polylineBatch);
+ }
+
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ System.out.println("---------------------------------");
+// List<Point> results = new ArrayList<>();
+ // 计算运行时间
+// int eParam = 10;
+// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
+ int eParam = 2;
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ // 输出结果
+ long endTime = System.currentTimeMillis();
+ System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println(results.size());
+
+ if (results.size() <= 100) {
+ System.out.println("+++++++++++++++++++");
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ }
+ }
+
+ try (FileWriter writer = new FileWriter("fast.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java
deleted file mode 100644
index 8e8fb18ebfa..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java
+++ /dev/null
@@ -1,121 +0,0 @@
-package org.apache.iotdb.db.query.eBUG;
-
-import java.util.*;
-
-public class TimestampPriorityQueue {
- private final int e; // 队列的最大大小
- private final LinkedList<Point> insertionOrder; // 保持插入顺序
- // public final PriorityQueue<Point> queue; // 按时间戳排序的优先队列
- private final List<Point> sortedList; // 维护排序后的列表
-
- // 构造函数：初始化队列大小、插入顺序队列和优先队列
- public TimestampPriorityQueue(int e) {
- this.e = e;
- this.insertionOrder = new LinkedList<>();
-// this.queue = new PriorityQueue<>(Comparator.comparingDouble(p -> p.x));
- this.sortedList = new ArrayList<>();
- }
-
- public int size() {
-// return queue.size();
- return sortedList.size();
- }
-
- // 插入新的元素
- public void insert(Point newPoint) {
- // 如果队列已满，移除最早插入的点
- if (sortedList.size() == e) {
- Point removedPoint = insertionOrder.removeFirst(); // 移除最早加入的点 复杂度1
-// queue.remove(removedPoint); // 从优先队列中移除该点 TODO 复杂度O(e)!
- sortedList.remove(removedPoint); // 从排序后的列表中移除该点 复杂度O(e)
- }
-
- // 将新点添加到插入顺序队列中
- insertionOrder.addLast(newPoint); // 复杂度1
-
- // 插入到优先队列，保持时间戳顺序
-// queue.add(newPoint); // 复杂度log(e)
-
- // 插入到排序后的列表中，保持有序
- int insertIndex = Collections.binarySearch(sortedList, newPoint, Comparator.comparingDouble(p -> p.x)); // 复杂度log(e)
- if (insertIndex < 0) {
- insertIndex = -(insertIndex + 1); // 如果未找到，binarySearch返回插入点的负值
- }
- sortedList.add(insertIndex, newPoint); // 复杂度O(e)
- }
-
- // 获取队列中的所有元素（按时间戳排序）
- public List<Point> getQueue() {
-// return new ArrayList<>(queue); // 这个不是排序的！
- return sortedList;
- }
-
- public String toString() {
- StringBuffer stringBuffer = new StringBuffer();
- for (Point point : sortedList) {
- stringBuffer.append(point.toString());
- stringBuffer.append(",");
- }
- return stringBuffer.toString();
- }
-
- public static void main(String[] args) {
- TimestampPriorityQueue tq = new TimestampPriorityQueue(10);
-
- tq.insert(new Point(0, 1000));
- tq.insert(new Point(3, 2000));
- tq.insert(new Point(11, 1500));
- tq.insert(new Point(5, 500));
-
- // 打印队列内容
- for (Point point : tq.getQueue()) {
- System.out.println(point);
- }
- }
-}
-
-
-//public class TimestampPriorityQueue { // for managing the e most recently eliminated points, sorted by time in ascending order
-// private final int e;
-// public final PriorityQueue<Point> queue;
-//
-// // 构造函数：初始化优先队列的最大大小
-// public TimestampPriorityQueue(int e) {
-// this.e = e;
-// this.queue = new PriorityQueue<>(Comparator.comparingDouble(p -> p.x));
-// }
-//
-// // 插入新的元素
-// public void insert(Point newPoint) {
-// // 如果队列已满，移除最小（最早）的元素
-// if (queue.size() == e) {
-// queue.poll();
-// }
-//
-// // 插入元素，队列会根据时间戳自动排序
-// queue.offer(newPoint);
-// }
-//
-// // 获取队列中的所有元素
-// public List<Point> getQueue() {
-// return new ArrayList<>(queue);
-// }
-//
-// public static void main(String[] args) {
-// TimestampPriorityQueue tq = new TimestampPriorityQueue(3);
-//
-// tq.insert(new Point(10, 1000));
-// tq.insert(new Point(20, 2000));
-// tq.insert(new Point(15, 1500));
-// tq.insert(new Point(25, 2500));
-//
-// // 打印队列内容
-// long tmp = 2000;
-// for (Point point : tq.getQueue()) {
-// point.x = tmp--;
-// System.out.println("Value: " + point.x + ", Timestamp: " + point.y);
-// }
-//
-// System.out.println(tq.queue.poll());
-// }
-//}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Visval_standard.java
deleted file mode 100644
index a85332d3846..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Visval_standard.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.query.eBUG;
-
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-
-public class Visval_standard {
-
- public static void buildEffectiveArea(Polyline lineToSimplify, List<Point> results) {
- results.clear();
- results.addAll(lineToSimplify.getVertices()); // TODO debug
-
- int total = lineToSimplify.size();
- if (total < 3) {
- return; // 不足 3 个点无法形成三角形
- }
-
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
- triangles[i - 1] = new Triangle(index1, index2, index3, area);
- }
-
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
- }
-
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO
-
- double previousEA = -1;
- while (!triangleHeap.isEmpty()) {
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll(); // O(logn)
-
- // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
- // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
- if (tri.isDeleted) {
- continue;
- }
-
- // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
- if (tri.area > previousEA) {
- previousEA = tri.area;
- }
- results.get(tri.indices[1]).z = previousEA;
- // System.out.println(tri.indices[1] + "," + previousEA);
-
-// System.out.println(Arrays.toString(tri.indices) + "," + previousEA);
-
- // 更新相邻三角形
- if (tri.prev != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-
- Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
-// tri.prev = newPre; // replace TODO can omit
-
- // 2. 处理pi被淘汰引起tri.prev被更新的事情
- // 前一个三角形连到后一个三角形
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
- tri.prev.indices[2] = tri.indices[2];
- tri.prev.area = triArea(lineToSimplify.get(tri.prev.indices[0]), lineToSimplify.get(tri.prev.indices[1]), lineToSimplify.get(tri.prev.indices[2]));
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add来显式重新插入修改后的元素
- triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
- }
-
- if (tri.next != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.next被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
- tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
-
- Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
-// tri.next = newNext; // replace TODO can omit
-
- if (tri.prev != null) {
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
- }
-
- // 2. 处理pi被淘汰引起tri.next被更新的事情
- tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
- tri.next.indices[0] = tri.indices[0];
- tri.next.area = triArea(lineToSimplify.get(tri.next.indices[0]), lineToSimplify.get(tri.next.indices[1]), lineToSimplify.get(tri.next.indices[2]));
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
- triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
- }
- }
- }
-
- public static void main(String[] args) {
- Polyline polyline = new Polyline();
- List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random(1);
- int n = 10_0000;
-
- int p = 700;
- for (int i = 0; i < n; i += p) {
- Polyline polylineBatch = new Polyline();
- for (int j = i; j < Math.min(i + p, n); j++) {
- double v = rand.nextInt(1000000);
-
- polyline.addVertex(new Point(j, v));
-
- polylineBatch.addVertex(new Point(j, v));
- }
- polylineList.add(polylineBatch);
- }
-
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
-
- System.out.println("---------------------------------");
- List<Point> results = new ArrayList<>();
- // 计算运行时间
- long startTime = System.currentTimeMillis();
- buildEffectiveArea(polyline, results);
- // 输出结果
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(results.size());
-
- if (results.size() <= 100) {
- System.out.println("+++++++++++++++++++");
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
- }
- }
-
- try (FileWriter writer = new FileWriter("fast.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
-
- System.out.println("---------------------------------");
- List<List<Point>> resultsBatchList = new ArrayList<>();
- // 计算运行时间
- int cnt = 0;
- startTime = System.currentTimeMillis();
- for (Polyline polylineBatch : polylineList) {
- List<Point> resultsBatch = new ArrayList<>();
- buildEffectiveArea(polylineBatch, resultsBatch);
- cnt += resultsBatch.size();
- resultsBatchList.add(resultsBatch);
- }
- // 输出结果
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(cnt);
-
- System.out.println("---------------------------------");
- // 使用 Stream API 合并所有列表
- List<Point> mergedList =
- resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
- int sameCnt = 0;
- for (int i = 0; i < mergedList.size(); i++) {
- if (mergedList.get(i).z == results.get(i).z) {
- sameCnt += 1;
- }
- }
- System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
-
- try (FileWriter writer = new FileWriter("batch.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < mergedList.size(); i++) {
- Point point = mergedList.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
- }
-
- // float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
- // 23089};
- // for (int i = 0; i < vlist.length; i++) {
- // polyline.addVertex(new vPoint(i, vlist[i]));
- // }
-
- // ArrayList<Double> v = new ArrayList<>();
- // String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
- // try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
- // String line;
- // int count = 0;
- // while ((line = br.readLine()) != null && count < 1000) {
- // String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
- // v.add(Double.parseDouble(columns[0])); // 读取第一列数据
- // count++;
- // }
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // // 打印数据长度
- // System.out.println("Data length: " + v.size());
- // for (int i = 0; i < v.size(); i++) {
- // polyline.addVertex(new vPoint(i, v.get(i)));
- // }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index d4a8ebfc36f..8c4d41f37f0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -31,11 +31,12 @@ import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
public class eBUG {
public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
int pa_idx, int pi_idx, int pb_idx) {
- // TODO 复杂度分析
- // 从最近淘汰的e个点里寻找位于pa~pb之间的：e
- // 把上一步找到的点和pa pi pb一起按照时间戳递增排序：记c=b-a。如果e<c，那么eloge，否则c
- // 后面鞋带公式计算面积：min(e+3,c)
+ // 复杂度分析 记c=b-a。
+ // 如果e<c，那么从最近淘汰的e个点里寻找位于pa~pb之间的，找到的点和pa pi pb一起按照时间戳递增排序，后面鞋带公式计算面积：O(e+eloge)
+ // 否则e>=c，直接取T[a:b]，已经排序号了，后面鞋带公式计算面积：O(c)
+
// 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
+
// pi: the point whose significance needs updating
// pa: left adjacent non-eliminated point of pi
// pb: right adjacent non-eliminated point of pi
@@ -80,7 +81,7 @@ public class eBUG {
}
// 按照时间戳递增排序，这是为了后面计算total areal displacement需要
- res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 TODO 这样的话时间复杂度要变成e*log(e)了吧？
+ res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 这样的话时间复杂度变成e*log(e)
return res;
}
@@ -91,9 +92,8 @@ public class eBUG {
}
List<Point> results = lineToSimplify.getVertices(); // 浅复制
-// results.addAll(lineToSimplify.getVertices()); // TODO debug
- // TODO 需要一个东西来记录最近淘汰的点依次是谁
+ // 用循环数组来记录最近淘汰的点
int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
int recentEliminatedIdx = 0; // index in the array, circulate
@@ -265,7 +265,7 @@ public class eBUG {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 100_0000;
+ int n = 1000_0000;
int p = 10;
for (int i = 0; i < n; i += p) {
@@ -299,10 +299,10 @@ public class eBUG {
// 计算运行时间
// int eParam = 10;
try (PrintWriter writer = new PrintWriter(new File("exp.csv"))) {
- int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 30_0000, 50_0000, 60_0000,
- 70_0000, 80_0000, 90_0000, 100_0000, 150_0000, 200_0000, 250_0000, 300_0000};
-// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
- for (int eParam : eParamList) {
+// int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 30_0000, 50_0000, 60_0000,
+// 70_0000, 80_0000, 90_0000, 100_0000, 150_0000, 200_0000, 250_0000, 300_0000};
+ for (int eParam = 0; eParam < n*1.5; eParam += 2_0000) {
+// for (int eParam : eParamList) {
long startTime = System.currentTimeMillis();
List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated.java
deleted file mode 100644
index 61c508added..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.query.eBUG;
-
-import java.io.*;
-import java.util.*;
-
-import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
-import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-
-
-public class eBUG_sortedEliminatedDeprecated {
- public static List<Point> findEliminated(Polyline lineToSimplify, TimestampPriorityQueue recentEliminatedOrdered,
- int pa_idx, int pi_idx, int pb_idx) {
- // TODO 复杂度分析
- // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
- // pi: the point whose significance needs updating
- // pa: left adjacent non-eliminated point of pi
- // pb: right adjacent non-eliminated point of pi
- // return a list of points, containing pa,p,pb and points between pa&pa from the e most recently eliminated points,
- // order by time in ascending order
-
- Point pa = lineToSimplify.get(pa_idx);
- Point pi = lineToSimplify.get(pi_idx);
- Point pb = lineToSimplify.get(pb_idx);
-
- List<Point> res = new ArrayList<>();
-
-// if (recentEliminated.length == 0) { // e=0
-// System.out.println("e=0");
-// res.add(pa);
-// res.add(pi);
-// res.add(pb);
-// return res; // pa pi pb已经按照时间戳递增
-// }
-
- if (recentEliminatedOrdered.size() >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
-// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
- res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
- return res; // 已经按照时间戳递增
- }
-
- // 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
-// System.out.println("k>=[c=(b-a-2)]>e, search among e");
- res.add(pa);
-
- // 包括了c>e=0的情况
- boolean addPi = false;
- for (Point p : recentEliminatedOrdered.getQueue()) { // 已经按时间戳递增排序，为了后面计算total areal displacement需要
- if (p.x <= pa.x) {
- continue;
- }
- if (p.x >= pb.x) {
- break;
- }
- if (p.x > pi.x && !addPi) {
- res.add(pi);
- addPi = true;
- }
- res.add(p);
- }
- if (!addPi) { // may happen when no eliminated points found above
- res.add(pi);
- }
- res.add(pb);
- return res;
- }
-
- public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
- if (e < 0) {
- throw new IllegalArgumentException("Parameter e must be non-negative.");
- }
-
- List<Point> results = lineToSimplify.getVertices(); // 浅复制
-// results.addAll(lineToSimplify.getVertices()); // TODO debug
-
- // TODO 需要一个东西来记录最近淘汰的点依次是谁
-// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
-// int recentEliminatedIdx = 0; // index in the array, circulate
- TimestampPriorityQueue recentEliminatedOrdered = new TimestampPriorityQueue(e);
-
- int total = lineToSimplify.size();
- if (total < 3) {
- return results; // 不足 3 个点无法形成三角形
- }
-
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
- triangles[i - 1] = new Triangle(index1, index2, index3, area);
- }
-
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
- }
-
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
-
- double previousEA = -1;
- while (!triangleHeap.isEmpty()) {
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll(); // O(logn)
-
- // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
- // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
- if (tri.isDeleted) {
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
- continue;
- }
-
- // 真正的淘汰点
- // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
- if (debug) {
- System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
- }
- if (e > 0) { // otherwise e=0
-// recentEliminated[recentEliminatedIdx] = tri.indices[1];
-// recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
- recentEliminatedOrdered.insert(lineToSimplify.get(tri.indices[1]));
- }
- if (debug) {
- System.out.println("the e most recently eliminated points:" + recentEliminatedOrdered);
- }
-
- // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
- if (tri.area > previousEA) {
- previousEA = tri.area;
- }
- results.get(tri.indices[1]).z = previousEA; // dominated significance
- if (debug) {
- System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
- }
-
- // 更新相邻三角形
- if (tri.prev != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-
- Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
-// tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
-
- // 2. 处理pi被淘汰引起tri.prev被更新的事情
- // 前一个三角形连到后一个三角形
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
- tri.prev.indices[2] = tri.indices[2];
-
- // e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
- tri.prev.indices[0],
- tri.prev.indices[1],
- tri.prev.indices[2]); // TODO complexity ?
- if (debug) {
- System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- double sig = total_areal_displacement(pointsForSig, baseLine, false);
- tri.prev.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri = triArea(lineToSimplify.get(tri.prev.indices[0]),
- lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2]));
- System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add来显式重新插入修改后的元素
- triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
- }
-
- if (tri.next != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.next被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
- tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
-
- Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
-// tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
-
- if (tri.prev != null) {
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
- }
-
- // 2. 处理pi被淘汰引起tri.next被更新的事情
- tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
- tri.next.indices[0] = tri.indices[0];
-
- // e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
- tri.next.indices[0],
- tri.next.indices[1],
- tri.next.indices[2]); // TODO complexity
- if (debug) {
- System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- double sig = total_areal_displacement(pointsForSig, baseLine, false);
- tri.next.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri = triArea(lineToSimplify.get(tri.next.indices[0]),
- lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2]));
- System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
- triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
- }
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
- }
- return results; // 注意这就是lineToSimplify.getVertices()
- }
-
- public static void main(String[] args) {
- Polyline polyline = new Polyline();
- List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random(1);
- int n = 100_0000;
-
- int p = 10;
- for (int i = 0; i < n; i += p) {
- Polyline polylineBatch = new Polyline();
- for (int j = i; j < Math.min(i + p, n); j++) {
- double v = rand.nextInt(1000000);
-
- polyline.addVertex(new Point(j, v));
-
- polylineBatch.addVertex(new Point(j, v));
- }
- polylineList.add(polylineBatch);
- }
-
-// try (FileWriter writer = new FileWriter("raw.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < polyline.size(); i++) {
-// Point point = polyline.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
-
- System.out.println("---------------------------------");
-// List<Point> results = new ArrayList<>();
- // 计算运行时间
-// int eParam = 10;
- try (PrintWriter writer = new PrintWriter(new File("exp2.csv"))) {
- int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 50_0000, 60_0000, 70_0000,
- 80_0000, 90_0000, 100_0000, 150_0000, 200_0000};
-// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
- for (int eParam : eParamList) {
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false);
- // 输出结果
- long endTime = System.currentTimeMillis();
- System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(results.size());
- writer.println(n + "," + eParam + "," + (endTime - startTime));
- }
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
- }
-
-// if (results.size() <= 100) {
-// System.out.println("+++++++++++++++++++");
-// for (int i = 0; i < results.size(); i++) {
-// Point point = results.get(i);
-// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
-// }
-// }
-
-// try (FileWriter writer = new FileWriter("fast.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < results.size(); i++) {
-// Point point = results.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
-
-// System.out.println("---------------------------------");
-// List<List<Point>> resultsBatchList = new ArrayList<>();
-// // 计算运行时间
-// int cnt = 0;
-// startTime = System.currentTimeMillis();
-// for (Polyline polylineBatch : polylineList) {
-// List<Point> resultsBatch = new ArrayList<>();
-// buildEffectiveArea(polylineBatch, resultsBatch);
-// cnt += resultsBatch.size();
-// resultsBatchList.add(resultsBatch);
-// }
-// // 输出结果
-// endTime = System.currentTimeMillis();
-// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
-// System.out.println(cnt);
-//
-// System.out.println("---------------------------------");
-// // 使用 Stream API 合并所有列表
-// List<Point> mergedList = resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
-// int sameCnt = 0;
-// for (int i = 0; i < mergedList.size(); i++) {
-// if (mergedList.get(i).z == results.get(i).z) {
-// sameCnt += 1;
-// }
-// }
-// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
-//
-// try (FileWriter writer = new FileWriter("batch.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < mergedList.size(); i++) {
-// Point point = mergedList.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
- }
-
- // float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
- // 23089};
- // for (int i = 0; i < vlist.length; i++) {
- // polyline.addVertex(new vPoint(i, vlist[i]));
- // }
-
- // ArrayList<Double> v = new ArrayList<>();
- // String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
- // try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
- // String line;
- // int count = 0;
- // while ((line = br.readLine()) != null && count < 1000) {
- // String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
- // v.add(Double.parseDouble(columns[0])); // 读取第一列数据
- // count++;
- // }
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // // 打印数据长度
- // System.out.println("Data length: " + v.size());
- // for (int i = 0; i < v.size(); i++) {
- // polyline.addVertex(new vPoint(i, v.get(i)));
- // }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated_tmp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated_tmp.java
deleted file mode 100644
index 4d933cfcd2e..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated_tmp.java
+++ /dev/null
@@ -1,398 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.query.eBUG;
-
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.*;
-
-import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
-import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-
-
-public class eBUG_sortedEliminatedDeprecated_tmp {
- public static List<Point> findEliminated(Polyline lineToSimplify, TimestampPriorityQueue recentEliminatedOrdered,
- int pa_idx, int pi_idx, int pb_idx) {
- // TODO 复杂度分析
- // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
- // pi: the point whose significance needs updating
- // pa: left adjacent non-eliminated point of pi
- // pb: right adjacent non-eliminated point of pi
- // return a list of points, containing pa,p,pb and points between pa&pa from the e most recently eliminated points,
- // order by time in ascending order
-
- Point pa = lineToSimplify.get(pa_idx);
- Point pi = lineToSimplify.get(pi_idx);
- Point pb = lineToSimplify.get(pb_idx);
-
- List<Point> res = new ArrayList<>();
-
-// if (recentEliminated.length == 0) { // e=0
-// System.out.println("e=0");
-// res.add(pa);
-// res.add(pi);
-// res.add(pb);
-// return res; // pa pi pb已经按照时间戳递增
-// }
-
- if (recentEliminatedOrdered.size() >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
-// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
- res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
- return res; // 已经按照时间戳递增
- }
-
- // 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
-// System.out.println("k>=[c=(b-a-2)]>e, search among e");
- res.add(pa);
-
- // 包括了c>e=0的情况
- boolean addPi = false;
- for (Point p : recentEliminatedOrdered.getQueue()) { // 已经按时间戳递增排序，为了后面计算total areal displacement需要
- if (p.x <= pa.x) {
- continue;
- }
- if (p.x >= pb.x) {
- break;
- }
- if (p.x > pi.x && !addPi) {
- res.add(pi);
- addPi = true;
- }
- res.add(p);
- }
- if (!addPi) { // may happen when no eliminated points found above
- res.add(pi);
- }
- res.add(pb);
- return res;
- }
-
- public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
- if (e < 0) {
- throw new IllegalArgumentException("Parameter e must be non-negative.");
- }
-
- List<Point> results = lineToSimplify.getVertices(); // 浅复制
-// results.addAll(lineToSimplify.getVertices()); // TODO debug
-
- // TODO 需要一个东西来记录最近淘汰的点依次是谁
-// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
-// int recentEliminatedIdx = 0; // index in the array, circulate
- TimestampPriorityQueue recentEliminatedOrdered = new TimestampPriorityQueue(e);
-
- int total = lineToSimplify.size();
- if (total < 3) {
- return results; // 不足 3 个点无法形成三角形
- }
-
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
- triangles[i - 1] = new Triangle(index1, index2, index3, area);
- }
-
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
- }
-
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
-
- double previousEA = -1;
- while (!triangleHeap.isEmpty()) {
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll(); // O(logn)
-
- // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
- // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
- if (tri.isDeleted) {
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
- continue;
- }
-
- // 真正的淘汰点
- // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
- if (debug) {
- System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
- }
- if (e > 0) { // otherwise e=0
-// recentEliminated[recentEliminatedIdx] = tri.indices[1];
-// recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
- recentEliminatedOrdered.insert(lineToSimplify.get(tri.indices[1]));
- }
- if (debug) {
- System.out.println("the e most recently eliminated points:" + recentEliminatedOrdered);
- }
-
- // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
- if (tri.area > previousEA) {
- previousEA = tri.area;
- }
- results.get(tri.indices[1]).z = previousEA; // dominated significance
- if (debug) {
- System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
- }
-
- // 更新相邻三角形
- if (tri.prev != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-
- Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
-// tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
-
- // 2. 处理pi被淘汰引起tri.prev被更新的事情
- // 前一个三角形连到后一个三角形
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
- tri.prev.indices[2] = tri.indices[2];
-
- // e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
- tri.prev.indices[0],
- tri.prev.indices[1],
- tri.prev.indices[2]); // TODO complexity ?
- if (debug) {
- System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- double sig = total_areal_displacement(pointsForSig, baseLine, false);
- tri.prev.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri = triArea(lineToSimplify.get(tri.prev.indices[0]),
- lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2]));
- System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add来显式重新插入修改后的元素
- triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
- }
-
- if (tri.next != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.next被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
- tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
-
- Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
-// tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
-
- if (tri.prev != null) {
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
- }
-
- // 2. 处理pi被淘汰引起tri.next被更新的事情
- tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
- tri.next.indices[0] = tri.indices[0];
-
- // e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
- tri.next.indices[0],
- tri.next.indices[1],
- tri.next.indices[2]); // TODO complexity
- if (debug) {
- System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- double sig = total_areal_displacement(pointsForSig, baseLine, false);
- tri.next.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri = triArea(lineToSimplify.get(tri.next.indices[0]),
- lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2]));
- System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
- triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
- }
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
- }
- return results; // 注意这就是lineToSimplify.getVertices()
- }
-
- public static void main(String[] args) {
- Polyline polyline = new Polyline();
- List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random(1);
-// int n = 1000_0000;
- int n = 10000;
-
- int p = 10;
- for (int i = 0; i < n; i += p) {
- Polyline polylineBatch = new Polyline();
- for (int j = i; j < Math.min(i + p, n); j++) {
- double v = rand.nextInt(1000000);
-
- polyline.addVertex(new Point(j, v));
-
- polylineBatch.addVertex(new Point(j, v));
- }
- polylineList.add(polylineBatch);
- }
-
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
-
- System.out.println("---------------------------------");
-// List<Point> results = new ArrayList<>();
- // 计算运行时间
- int eParam = 0;
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false);
- // 输出结果
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(results.size());
-
- if (results.size() <= 100) {
- System.out.println("+++++++++++++++++++");
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
- }
- }
-
- try (FileWriter writer = new FileWriter("fast.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
-
-// System.out.println("---------------------------------");
-// List<List<Point>> resultsBatchList = new ArrayList<>();
-// // 计算运行时间
-// int cnt = 0;
-// startTime = System.currentTimeMillis();
-// for (Polyline polylineBatch : polylineList) {
-// List<Point> resultsBatch = new ArrayList<>();
-// buildEffectiveArea(polylineBatch, resultsBatch);
-// cnt += resultsBatch.size();
-// resultsBatchList.add(resultsBatch);
-// }
-// // 输出结果
-// endTime = System.currentTimeMillis();
-// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
-// System.out.println(cnt);
-//
-// System.out.println("---------------------------------");
-// // 使用 Stream API 合并所有列表
-// List<Point> mergedList = resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
-// int sameCnt = 0;
-// for (int i = 0; i < mergedList.size(); i++) {
-// if (mergedList.get(i).z == results.get(i).z) {
-// sameCnt += 1;
-// }
-// }
-// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
-//
-// try (FileWriter writer = new FileWriter("batch.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < mergedList.size(); i++) {
-// Point point = mergedList.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
- }
-
- // float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
- // 23089};
- // for (int i = 0; i < vlist.length; i++) {
- // polyline.addVertex(new vPoint(i, vlist[i]));
- // }
-
- // ArrayList<Double> v = new ArrayList<>();
- // String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
- // try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
- // String line;
- // int count = 0;
- // while ((line = br.readLine()) != null && count < 1000) {
- // String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
- // v.add(Double.parseDouble(columns[0])); // 读取第一列数据
- // count++;
- // }
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // // 打印数据长度
- // System.out.println("Data length: " + v.size());
- // for (int i = 0; i < v.size(); i++) {
- // polyline.addVertex(new vPoint(i, v.get(i)));
- // }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_tmp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_tmp.java
deleted file mode 100644
index 93267df65a7..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_tmp.java
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.query.eBUG;
-
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.*;
-
-import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
-import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-
-
-public class eBUG_tmp {
- public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
- int pa_idx, int pi_idx, int pb_idx) {
- // TODO 复杂度分析
- // 从最近淘汰的e个点里寻找位于pa~pb之间的：e
- // 把上一步找到的点和pa pi pb一起按照时间戳递增排序：记c=b-a。如果e<c，那么eloge，否则c
- // 后面鞋带公式计算面积：min(e+3,c)
- // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
- // pi: the point whose significance needs updating
- // pa: left adjacent non-eliminated point of pi
- // pb: right adjacent non-eliminated point of pi
- // return a list of points, containing pa,p,pb and points between pa&pa from the e most recently eliminated points,
- // order by time in ascending order
-
- Point pa = lineToSimplify.get(pa_idx);
- Point pi = lineToSimplify.get(pi_idx);
- Point pb = lineToSimplify.get(pb_idx);
-
- List<Point> res = new ArrayList<>();
-
-// if (recentEliminated.length == 0) { // e=0
-// System.out.println("e=0");
-// res.add(pa);
-// res.add(pi);
-// res.add(pb);
-// return res; // pa pi pb已经按照时间戳递增
-// }
-
- if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
-// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
- res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
- return res; // 已经按照时间戳递增
- }
-
- // 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
-// System.out.println("k>=[c=(b-a-2)]>e, search among e");
- res.add(pa);
- res.add(pi);
- res.add(pb);
-
- // 包括了c>e=0的情况
- for (int idx : recentEliminated) { // e points
- if (idx == 0) { // init, not filled by real eliminated points yet
- break;
- }
- Point p = lineToSimplify.get(idx);
- if (p.x > pa.x && p.x < pb.x) {
- res.add(p);
- }
- }
-
- // 按照时间戳递增排序，这是为了后面计算total areal displacement需要
- res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 TODO 这样的话时间复杂度要变成e*log(e)了吧？
-
- return res;
- }
-
- public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
- if (e < 0) {
- throw new IllegalArgumentException("Parameter e must be non-negative.");
- }
-
- List<Point> results = lineToSimplify.getVertices(); // 浅复制
-// results.addAll(lineToSimplify.getVertices()); // TODO debug
-
- // TODO 需要一个东西来记录最近淘汰的点依次是谁
- int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
- int recentEliminatedIdx = 0; // index in the array, circulate
-
- int total = lineToSimplify.size();
- if (total < 3) {
- return results; // 不足 3 个点无法形成三角形
- }
-
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
- triangles[i - 1] = new Triangle(index1, index2, index3, area);
- }
-
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
- }
-
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
-
- double previousEA = -1;
- while (!triangleHeap.isEmpty()) {
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll(); // O(logn)
-
- // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
- // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
- if (tri.isDeleted) {
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
- continue;
- }
-
- // 真正的淘汰点
- // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
- if (debug) {
- System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
- }
- if (e > 0) { // otherwise e=0
- recentEliminated[recentEliminatedIdx] = tri.indices[1];
- recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
- }
- if (debug) {
- System.out.println("the e most recently eliminated points:" + Arrays.toString(recentEliminated));
- }
-
- // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
- if (tri.area > previousEA) {
- previousEA = tri.area;
- }
- results.get(tri.indices[1]).z = previousEA; // dominated significance
- if (debug) {
- System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
- }
-
- // 更新相邻三角形
- if (tri.prev != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-
- Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
-// tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
-
- // 2. 处理pi被淘汰引起tri.prev被更新的事情
- // 前一个三角形连到后一个三角形
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
- tri.prev.indices[2] = tri.indices[2];
-
- // e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
- tri.prev.indices[0],
- tri.prev.indices[1],
- tri.prev.indices[2]); // TODO complexity ?
- if (debug) {
- System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- double sig = total_areal_displacement(pointsForSig, baseLine, false);
- tri.prev.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri = triArea(lineToSimplify.get(tri.prev.indices[0]),
- lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2]));
- System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add来显式重新插入修改后的元素
- triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
- }
-
- if (tri.next != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.next被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
- tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
-
- Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
-// tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
-
- if (tri.prev != null) {
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
- }
-
- // 2. 处理pi被淘汰引起tri.next被更新的事情
- tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
- tri.next.indices[0] = tri.indices[0];
-
- // e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
- tri.next.indices[0],
- tri.next.indices[1],
- tri.next.indices[2]); // TODO complexity
- if (debug) {
- System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- double sig = total_areal_displacement(pointsForSig, baseLine, false);
- tri.next.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri = triArea(lineToSimplify.get(tri.next.indices[0]),
- lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2]));
- System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
- triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
- }
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
- }
- return results; // 注意这就是lineToSimplify.getVertices()
- }
-
- public static void main(String[] args) {
- Polyline polyline = new Polyline();
- List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random(1);
- int n = 10000;
-
- int p = 10;
- for (int i = 0; i < n; i += p) {
- Polyline polylineBatch = new Polyline();
- for (int j = i; j < Math.min(i + p, n); j++) {
- double v = rand.nextInt(1000000);
-
- polyline.addVertex(new Point(j, v));
-
- polylineBatch.addVertex(new Point(j, v));
- }
- polylineList.add(polylineBatch);
- }
-
-// try (FileWriter writer = new FileWriter("raw.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < polyline.size(); i++) {
-// Point point = polyline.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
-
- System.out.println("---------------------------------");
-// List<Point> results = new ArrayList<>();
- // 计算运行时间
-// int eParam = 10;
-// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
- int eParam = 22;
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false);
- // 输出结果
- long endTime = System.currentTimeMillis();
- System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(results.size());
-
- if (results.size() <= 100) {
- System.out.println("+++++++++++++++++++");
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
- }
- }
-
- try (FileWriter writer = new FileWriter("fast2.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
-
-// System.out.println("---------------------------------");
-// List<List<Point>> resultsBatchList = new ArrayList<>();
-// // 计算运行时间
-// int cnt = 0;
-// startTime = System.currentTimeMillis();
-// for (Polyline polylineBatch : polylineList) {
-// List<Point> resultsBatch = new ArrayList<>();
-// buildEffectiveArea(polylineBatch, resultsBatch);
-// cnt += resultsBatch.size();
-// resultsBatchList.add(resultsBatch);
-// }
-// // 输出结果
-// endTime = System.currentTimeMillis();
-// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
-// System.out.println(cnt);
-//
-// System.out.println("---------------------------------");
-// // 使用 Stream API 合并所有列表
-// List<Point> mergedList = resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
-// int sameCnt = 0;
-// for (int i = 0; i < mergedList.size(); i++) {
-// if (mergedList.get(i).z == results.get(i).z) {
-// sameCnt += 1;
-// }
-// }
-// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
-//
-// try (FileWriter writer = new FileWriter("batch.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < mergedList.size(); i++) {
-// Point point = mergedList.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
- }
-
- // float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
- // 23089};
- // for (int i = 0; i < vlist.length; i++) {
- // polyline.addVertex(new vPoint(i, vlist[i]));
- // }
-
- // ArrayList<Double> v = new ArrayList<>();
- // String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
- // try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
- // String line;
- // int count = 0;
- // while ((line = br.readLine()) != null && count < 1000) {
- // String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
- // v.add(Double.parseDouble(columns[0])); // 读取第一列数据
- // count++;
- // }
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // // 打印数据长度
- // System.out.println("Data length: " + v.size());
- // for (int i = 0; i < v.size(); i++) {
- // polyline.addVertex(new vPoint(i, v.get(i)));
- // }
-}
(iotdb) 11/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 068c3e6588231a1578d80f52ee2d41519516b551
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jan 21 18:28:23 2025 +0800
add
---
.../org/apache/iotdb/db/query/eBUG/Polyline.java | 4 +-
.../java/org/apache/iotdb/db/query/eBUG/Tool.java | 9 +-
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 120 +++++++++++++--------
3 files changed, 86 insertions(+), 47 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
index b6e2f231c14..9fa11b192cb 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
@@ -12,9 +12,11 @@ public class Polyline {
}
public List<Point> getVertices() {
- return new ArrayList<>(vertices);
+// return new ArrayList<>(vertices);
+ return vertices;
}
+
public int size() {
return vertices.size();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
index 48c1544e76b..0587b2fa39e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
@@ -171,10 +171,11 @@ public class Tool {
points.add(new Point(6, 0));
List<Point> points2 = new ArrayList<>();
- points2.add(points.get(0));
- points2.add(points.get(3));
- points2.add(points.get(6));
-// points2.add(points.get(5));
+// points2.add(points.get(0));
+// points2.add(points.get(3));
+// points2.add(points.get(6));
+ points2.add(new Point(1,-10));
+ points2.add(new Point(3,0));
double area = total_areal_displacement(points, points2, true);
System.out.println("Total area: " + area);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index cc512cbcc9c..12882823356 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -23,6 +23,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.util.*;
+import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
@@ -31,9 +32,9 @@ public class eBUG {
Point pa, Point pi, Point pb) {
// TODO 寻找最近淘汰的e个点，并且是位于pa~pb之间的
// 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
- // pi: point need significance updating
- // pa: left adjacent non-eliminated point
- // pb: right adjacent non-eliminated point
+ // pi: the point whose significance needs updating
+ // pa: left adjacent non-eliminated point of pi
+ // pb: right adjacent non-eliminated point of pi
// return a list of points, containing pa,p,pb and points between pa&pa from the e most recently eliminated points,
// order by time in ascending order
List<Point> res = new ArrayList<>();
@@ -53,9 +54,13 @@ public class eBUG {
return res;
}
- public static void buildEffectiveArea(Polyline lineToSimplify, List<Point> results, int e) {
- results.clear();
- results.addAll(lineToSimplify.getVertices()); // TODO debug
+ public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
+ if (e < 0) {
+ throw new IllegalArgumentException("Parameter e must be non-negative.");
+ }
+
+ List<Point> results = lineToSimplify.getVertices(); // 浅复制
+// results.addAll(lineToSimplify.getVertices()); // TODO debug
// TODO 需要一个东西来记录最近淘汰的点依次是谁
int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
@@ -63,7 +68,7 @@ public class eBUG {
int total = lineToSimplify.size();
if (total < 3) {
- return; // 不足 3 个点无法形成三角形
+ return results; // 不足 3 个点无法形成三角形
}
int nTriangles = total - 2;
@@ -84,7 +89,7 @@ public class eBUG {
// 使用优先队列构建 minHeap
PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO
+ Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
double previousEA = -1;
while (!triangleHeap.isEmpty()) {
@@ -95,27 +100,33 @@ public class eBUG {
// 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
// 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
if (tri.isDeleted) {
+ if (debug) {
+ System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
+ }
continue;
}
// 真正的淘汰点
- //TODO 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
-// popOrder.add(tri.indices[1]);
- System.out.println("eliminate " + lineToSimplify.get(tri.indices[1]));
- if (e > 0) {
+ // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ if (debug) {
+ System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
+ }
+ if (e > 0) { // otherwise e=0
recentEliminated[recentEliminatedIdx] = tri.indices[1];
- recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1
+ recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
+ }
+ if (debug) {
+ System.out.println("the e most recently eliminated points:" + Arrays.toString(recentEliminated));
}
- System.out.println(Arrays.toString(recentEliminated));
// 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
if (tri.area > previousEA) {
previousEA = tri.area;
}
- results.get(tri.indices[1]).z = previousEA;
- // System.out.println(tri.indices[1] + "," + previousEA);
-
-// System.out.println(Arrays.toString(tri.indices) + "," + previousEA);
+ results.get(tri.indices[1]).z = previousEA; // dominated significance
+ if (debug) {
+ System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
+ }
// 更新相邻三角形
if (tri.prev != null) {
@@ -126,26 +137,37 @@ public class eBUG {
tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
-// tri.prev = newPre; // replace TODO can omit, because already done by new Triangle(tri.prev)
+// tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
// 2. 处理pi被淘汰引起tri.prev被更新的事情
// 前一个三角形连到后一个三角形
tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
tri.prev.indices[2] = tri.indices[2];
- // TODO e parameter
+ // e parameter
List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
lineToSimplify.get(tri.prev.indices[0]),
lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2]));
- System.out.println("for updating Point on the left " + lineToSimplify.get(tri.prev.indices[1]));
- for (Point point : pointsForSig) {
- System.out.println(point);
+ lineToSimplify.get(tri.prev.indices[2])); // TODO complexity ?
+ if (debug) {
+ System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
+ for (Point point : pointsForSig) {
+ System.out.println("\t" + point);
+ }
+ }
+ List<Point> baseLine = new ArrayList<>();
+ baseLine.add(pointsForSig.get(0));
+ baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
+ double sig = total_areal_displacement(pointsForSig, baseLine, false);
+ tri.prev.area = sig;
+
+ if (debug) {
+ System.out.println("sig=" + sig);
+ double tmpTri = triArea(lineToSimplify.get(tri.prev.indices[0]),
+ lineToSimplify.get(tri.prev.indices[1]),
+ lineToSimplify.get(tri.prev.indices[2]));
+ System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
}
-
- tri.prev.area = triArea(lineToSimplify.get(tri.prev.indices[0]),
- lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2]));
// 重新加入堆
// 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
@@ -161,7 +183,7 @@ public class eBUG {
tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
-// tri.next = newNext; // replace TODO can omit, because already done by new Triangle(tri.prev)
+// tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
if (tri.prev != null) {
tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
@@ -171,27 +193,41 @@ public class eBUG {
tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
tri.next.indices[0] = tri.indices[0];
- // TODO e parameter
+ // e parameter
List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
lineToSimplify.get(tri.next.indices[0]),
lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2]));
- System.out.println("for updating Point on the right " + lineToSimplify.get(tri.next.indices[1]));
- for (Point point : pointsForSig) {
- System.out.println(point);
+ lineToSimplify.get(tri.next.indices[2])); // TODO complexity
+ if (debug) {
+ System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
+ for (Point point : pointsForSig) {
+ System.out.println("\t" + point);
+ }
+ }
+ List<Point> baseLine = new ArrayList<>();
+ baseLine.add(pointsForSig.get(0));
+ baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
+ double sig = total_areal_displacement(pointsForSig, baseLine, false);
+ tri.next.area = sig;
+
+ if (debug) {
+ System.out.println("sig=" + sig);
+ double tmpTri = triArea(lineToSimplify.get(tri.next.indices[0]),
+ lineToSimplify.get(tri.next.indices[1]),
+ lineToSimplify.get(tri.next.indices[2]));
+ System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
}
-
- tri.next.area = triArea(lineToSimplify.get(tri.next.indices[0]),
- lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2]));
// 重新加入堆
// 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
// 所以必须通过add 来显式重新插入修改后的元素
triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
}
- System.out.println(">>>");
+ if (debug) {
+ System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
+ }
}
+ return results; // 注意这就是lineToSimplify.getVertices()
}
public static void main(String[] args) {
@@ -228,11 +264,11 @@ public class eBUG {
}
System.out.println("---------------------------------");
- List<Point> results = new ArrayList<>();
+// List<Point> results = new ArrayList<>();
// 计算运行时间
- int eParam = 3;
+ int eParam = 1;
long startTime = System.currentTimeMillis();
- buildEffectiveArea(polyline, results, eParam);
+ List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
long endTime = System.currentTimeMillis();
System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
(iotdb) 24/44: eBUG_build
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d545f8db5f54b4a53d7018e446127d1ed1e889c4
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Jan 23 22:35:46 2025 +0800
eBUG_build
---
server/pom.xml | 23 +
.../java/org/apache/iotdb/db/query/eBUG/Point.java | 48 +-
.../org/apache/iotdb/db/query/eBUG/Polyline.java | 53 +-
.../java/org/apache/iotdb/db/query/eBUG/Test1.java | 126 ++--
.../java/org/apache/iotdb/db/query/eBUG/Test2.java | 86 +--
.../java/org/apache/iotdb/db/query/eBUG/Test3.java | 91 +++
.../java/org/apache/iotdb/db/query/eBUG/Tmp.java | 58 +-
.../java/org/apache/iotdb/db/query/eBUG/Tool.java | 365 +++++------
.../org/apache/iotdb/db/query/eBUG/Triangle.java | 76 +--
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 681 +++++++++++----------
.../org/apache/iotdb/db/query/eBUG/eBUG_Build.java | 139 +++++
11 files changed, 1035 insertions(+), 711 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index 3ca3b695285..51f9a56c69a 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -278,6 +278,29 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <finalName>eBUG_Build</finalName>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.iotdb.db.query.eBUG.eBUG_Build</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<profiles>
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
index 6e6fc3aa91e..e9bcf241f0f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
@@ -2,33 +2,33 @@ package org.apache.iotdb.db.query.eBUG;
public class Point {
- double x, y, z;
+ double x, y, z;
- public Point prev; // 指向T'_max{0,k-e}里当前点的前一个点 for eBUG usage
- public Point next; // 指向T'_max{0,k-e}里当前点的后一个点 for eBUG usage
- // 注意Triangle里的prev&next用于最新状态下存在三角形之间的连接关系，
- // 而这里Point的prev&next用于滞后e个淘汰点（也就是最近的e个淘汰点先不施加）状态下的未淘汰点之间的连接关系
+ public Point prev; // 指向T'_max{0,k-e}里当前点的前一个点 for eBUG usage
+ public Point next; // 指向T'_max{0,k-e}里当前点的后一个点 for eBUG usage
+ // 注意Triangle里的prev&next用于最新状态下存在三角形之间的连接关系，
+ // 而这里Point的prev&next用于滞后e个淘汰点（也就是最近的e个淘汰点先不施加）状态下的未淘汰点之间的连接关系
- public Point(double x, double y) {
- this.x = x;
- this.y = y;
- this.z = Double.POSITIVE_INFINITY; // effective area
+ public Point(double x, double y) {
+ this.x = x;
+ this.y = y;
+ this.z = Double.POSITIVE_INFINITY; // effective area
- // for eBUG usage
-// this.eliminated = false;
- this.prev = null;
- this.next = null;
- }
+ // for eBUG usage
+ // this.eliminated = false;
+ this.prev = null;
+ this.next = null;
+ }
- public void markEliminated() {
- // to avoid traversing each point between pa to pb,
- // instead only traversing at most e most recently eliminated points lagged
- prev.next = next;
- next.prev = prev;
- }
+ public void markEliminated() {
+ // to avoid traversing each point between pa to pb,
+ // instead only traversing at most e most recently eliminated points lagged
+ prev.next = next;
+ next.prev = prev;
+ }
- @Override
- public String toString() {
- return "Point: (" + x + ", " + y + ", " + z + ")";
- }
+ @Override
+ public String toString() {
+ return "Point: (" + x + ", " + y + ", " + z + ")";
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
index 1ddfe2ebbe1..534c7a6f67e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
@@ -5,31 +5,30 @@ import java.util.List;
public class Polyline {
- private List<Point> vertices = new ArrayList<>();
-
- public void addVertex(Point point) {
-// if (!vertices.isEmpty()) { //before adding this point
-// vertices.get(vertices.size() - 1).next = point;
-// point.prev = vertices.get(vertices.size() - 1);
-// }
- vertices.add(point);
- }
-
- public List<Point> getVertices() {
-// return new ArrayList<>(vertices);
- return vertices;
- }
-
-
- public int size() {
- return vertices.size();
- }
-
- public Point get(int index) {
- return vertices.get(index);
- }
-
- public void clear() {
- vertices.clear();
- }
+ private List<Point> vertices = new ArrayList<>();
+
+ public void addVertex(Point point) {
+ // if (!vertices.isEmpty()) { //before adding this point
+ // vertices.get(vertices.size() - 1).next = point;
+ // point.prev = vertices.get(vertices.size() - 1);
+ // }
+ vertices.add(point);
+ }
+
+ public List<Point> getVertices() {
+ // return new ArrayList<>(vertices);
+ return vertices;
+ }
+
+ public int size() {
+ return vertices.size();
+ }
+
+ public Point get(int index) {
+ return vertices.get(index);
+ }
+
+ public void clear() {
+ vertices.clear();
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
index 91df7751e50..385362b9e6b 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
@@ -10,74 +10,82 @@ import java.util.Random;
import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
public class Test1 {
- // 用于验证Java eBUG实现和python版本(e=0/1)结果的一致性
- public static void main(String[] args) {
- Polyline polyline = new Polyline();
- List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random(10);
- int n = 10000;
- int eParam = 0;
+ // 用于验证Java eBUG实现和python版本(e=0/1)结果的一致性
+ public static void main(String[] args) {
+ Polyline polyline = new Polyline();
+ List<Polyline> polylineList = new ArrayList<>();
+ Random rand = new Random(10);
+ int n = 10000;
+ int eParam = 1;
- int p = 10;
- for (int i = 0; i < n; i += p) {
- Polyline polylineBatch = new Polyline();
- for (int j = i; j < Math.min(i + p, n); j++) {
- double v = rand.nextInt(1000000);
+ int p = 10;
+ for (int i = 0; i < n; i += p) {
+ Polyline polylineBatch = new Polyline();
+ for (int j = i; j < Math.min(i + p, n); j++) {
+ double v = rand.nextInt(1000000);
- polyline.addVertex(new Point(j, v));
+ polyline.addVertex(new Point(j, v));
- polylineBatch.addVertex(new Point(j, v));
- }
- polylineList.add(polylineBatch);
- }
+ polylineBatch.addVertex(new Point(j, v));
+ }
+ polylineList.add(polylineBatch);
+ }
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
- System.out.println("---------------------------------");
- long startTime = System.currentTimeMillis();
- // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
- List<Point> results = buildEffectiveArea(polyline, eParam, false);
- // 输出结果
- long endTime = System.currentTimeMillis();
- System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(results.size());
+ System.out.println("---------------------------------");
+ long startTime = System.currentTimeMillis();
+ // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
+ List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ // 输出结果
+ long endTime = System.currentTimeMillis();
+ System.out.println(
+ "n="
+ + n
+ + ", e="
+ + eParam
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+ System.out.println(results.size());
- // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
- // 按照时间戳递增排序整理:
- results.sort(Comparator.comparingDouble(point -> point.x));
+ // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
+ // 按照时间戳递增排序整理:
+ System.out.println("make the bottom-up results sorted by time:");
+ results.sort(Comparator.comparingDouble(point -> point.x));
- if (results.size() <= 100) {
- System.out.println("+++++++++++++++++++");
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
- }
- }
+ if (results.size() <= 100) {
+ System.out.println("+++++++++++++++++++");
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ }
+ }
- try (FileWriter writer = new FileWriter("fast.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+ try (FileWriter writer = new FileWriter("fast.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
- // 写入每个点的数据
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ // 写入每个点的数据
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
}
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java
index fb7de582f17..6cb0accefaa 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java
@@ -9,44 +9,52 @@ import java.util.Random;
import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
public class Test2 {
- // 用于测试在线采样
- public static void main(String[] args) {
- int n = 20;
- int eParam = 1;
-
- int m = 2; // >2代表在线采样，<=2代表预计算
-// for (int m = 9900; m <= n; m++) {
-
- Polyline polyline = new Polyline();
- Random rand = new Random(2);
- for (int i = 0; i < n; i++) {
- double v = rand.nextInt(1000000);
- polyline.addVertex(new Point(i, v));
- }
-
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false, m);
- long endTime = System.currentTimeMillis();
- System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
-
- System.out.println(results.size());
- if (m > 2 && m < n) {
- Assert.isTrue(results.size() == m);
- } else {
- Assert.isTrue(results.size() == n);
- }
- if (results.size() <= 100) {
- System.out.println("+++++++++++++++++++");
- for (Point point : results) {
- System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
- }
- // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
- // 按照时间戳递增排序整理:
- results.sort(Comparator.comparingDouble(point -> point.x));
- System.out.println("+++++++++++++++++++");
- for (Point point : results) {
- System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
- }
- }
+ // 用于测试在线采样
+ public static void main(String[] args) {
+ int n = 20;
+ int eParam = 1;
+
+ int m = 2; // >2代表在线采样，<=2代表预计算
+ // for (int m = 9900; m <= n; m++) {
+
+ Polyline polyline = new Polyline();
+ Random rand = new Random(2);
+ for (int i = 0; i < n; i++) {
+ double v = rand.nextInt(1000000);
+ polyline.addVertex(new Point(i, v));
}
+
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false, m);
+ long endTime = System.currentTimeMillis();
+ System.out.println(
+ "n="
+ + n
+ + ", e="
+ + eParam
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+
+ System.out.println(results.size());
+ if (m > 2 && m < n) {
+ Assert.isTrue(results.size() == m);
+ } else {
+ Assert.isTrue(results.size() == n);
+ }
+ if (results.size() <= 100) {
+ System.out.println("+++++++++++++++++++");
+ for (Point point : results) {
+ System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ }
+ // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
+ // 按照时间戳递增排序整理:
+ System.out.println("make the bottom-up results sorted by time:");
+ results.sort(Comparator.comparingDouble(point -> point.x));
+ System.out.println("+++++++++++++++++++");
+ for (Point point : results) {
+ System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ }
+ }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test3.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test3.java
new file mode 100644
index 00000000000..d17ea8fcc65
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test3.java
@@ -0,0 +1,91 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
+
+public class Test3 {
+ // 用于验证Java eBUG实现和python版本(e=0/1)结果的一致性
+ public static void main(String[] args) {
+ Polyline polyline = new Polyline();
+ List<Polyline> polylineList = new ArrayList<>();
+ Random rand = new Random(10);
+ int n = 10000;
+ int eParam = 0;
+
+ int p = 10;
+ for (int i = 0; i < n; i += p) {
+ Polyline polylineBatch = new Polyline();
+ for (int j = i; j < Math.min(i + p, n); j++) {
+ double v = rand.nextInt(1000000);
+
+ polyline.addVertex(new Point(j, v));
+
+ polylineBatch.addVertex(new Point(j, v));
+ }
+ polylineList.add(polylineBatch);
+ }
+
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ System.out.println("---------------------------------");
+ long startTime = System.currentTimeMillis();
+ // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
+ List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ // 输出结果
+ long endTime = System.currentTimeMillis();
+ System.out.println(
+ "n="
+ + n
+ + ", e="
+ + eParam
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+ System.out.println(results.size());
+
+ // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
+ // 按照时间戳递增排序整理:
+ System.out.println("make the bottom-up results sorted by time:");
+ results.sort(Comparator.comparingDouble(point -> point.x));
+
+ if (results.size() <= 100) {
+ System.out.println("+++++++++++++++++++");
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ }
+ }
+
+ try (FileWriter writer = new FileWriter("fast.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < results.size(); i++) {
+ Point point = results.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java
index 9cfb8ac3d5f..270aca1dee7 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java
@@ -8,35 +8,37 @@ import java.util.PriorityQueue;
import java.util.Random;
public class Tmp {
- // 用于debug为什么n>2kw的耗时增大明显
- public static void main(String[] args) {
- int seed = 10;
- Random rand = new Random(seed);
- int eParam = 1;
- try (PrintWriter writer = new PrintWriter(new File("tmp.csv"))) {
- for (int n = 100_0000; n < 100000_0000; n += 5000_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
-// for (int n = 19000000; n < 3000_0000; n +=200_0000) {
- PriorityQueue<Point> heap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.y));
- for (int i = 0; i < n; i++) {
- double v = rand.nextInt(1000000);
- heap.add(new Point(i, v));
- }
-
- long startTime = System.currentTimeMillis();
- long sum = 0;
- while (!heap.isEmpty()) {
- Point p = heap.poll();
- sum += p.x;
- }
- long endTime = System.currentTimeMillis();
- System.out.println(n + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
- writer.println(n + "," + "0" + "," + (endTime - startTime));
- System.out.println(sum);
- }
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
+ // 用于debug为什么n>2kw的耗时增大明显
+ public static void main(String[] args) {
+ int seed = 10;
+ Random rand = new Random(seed);
+ int eParam = 1;
+ try (PrintWriter writer = new PrintWriter(new File("tmp.csv"))) {
+ for (int n = 100_0000;
+ n < 100000_0000;
+ n += 5000_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
+ // for (int n = 19000000; n < 3000_0000; n +=200_0000) {
+ PriorityQueue<Point> heap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.y));
+ for (int i = 0; i < n; i++) {
+ double v = rand.nextInt(1000000);
+ heap.add(new Point(i, v));
}
- System.out.println("finish");
+ long startTime = System.currentTimeMillis();
+ long sum = 0;
+ while (!heap.isEmpty()) {
+ Point p = heap.poll();
+ sum += p.x;
+ }
+ long endTime = System.currentTimeMillis();
+ System.out.println(n + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+ writer.println(n + "," + "0" + "," + (endTime - startTime));
+ System.out.println(sum);
+ }
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
}
+
+ System.out.println("finish");
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
index f7c5bbfb668..86658e24385 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
@@ -1,199 +1,204 @@
package org.apache.iotdb.db.query.eBUG;
-
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class Tool {
- // 计算三角形面积
- public static double triArea(Point d0, Point d1, Point d2) {
- double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
- return (dArea > 0.0) ? dArea : -dArea; // abs
+ // 计算三角形面积
+ public static double triArea(Point d0, Point d1, Point d2) {
+ double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
+ return (dArea > 0.0) ? dArea : -dArea; // abs
+ }
+
+ // 计算简单多边形（边不交叉）的面积（鞋带公式）
+ public static double calculatePolygonArea(List<Point> points) {
+ // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
+ int n = points.size();
+ double area = 0;
+ for (int i = 0; i < n; i++) {
+ int next = (i + 1) % n;
+ area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
}
-
- // 计算简单多边形（边不交叉）的面积（鞋带公式）
- public static double calculatePolygonArea(List<Point> points) {
- // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
- int n = points.size();
- double area = 0;
- for (int i = 0; i < n; i++) {
- int next = (i + 1) % n;
- area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
- }
- return Math.abs(area) / 2.0;
+ return Math.abs(area) / 2.0;
+ }
+
+ // 计算两个向量的叉积
+ public static double crossProduct(double x1, double y1, double x2, double y2) {
+ // >0: (x2,y2)在(x1,y1)的逆时针方向
+ // <0: (x2,y2)在(x1,y1)的顺时针方向
+ // =0: 平行或共线
+ return x1 * y2 - y1 * x2;
+ }
+
+ // 判断两条线段是否相交并计算交点
+ // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
+ public static Object[] lineIntersection(Point[] L1, Point[] L2) {
+ double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
+ double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
+
+ // 判断是否相交（叉积）
+ double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
+ double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
+ double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
+ double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
+
+ // 如果叉积条件满足，表示有交点
+ // d1*d2<0意味着P3、P4分别在L12的两边
+ // d3*d4<0意味着P1、P2分别在L34的两边
+ // 两个同时满足说明有交点
+ if (d1 * d2 < 0 && d3 * d4 < 0) {
+ double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
+ double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
+ double x = x1 + t1 * (x2 - x1);
+ double y = y1 + t1 * (y2 - y1);
+ return new Object[] {true, new Point(x, y)};
}
- // 计算两个向量的叉积
- public static double crossProduct(double x1, double y1, double x2, double y2) {
- // >0: (x2,y2)在(x1,y1)的逆时针方向
- // <0: (x2,y2)在(x1,y1)的顺时针方向
- // =0: 平行或共线
- return x1 * y2 - y1 * x2;
+ // 检查是否起点或终点重合
+ if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
+ return new Object[] {true, new Point(x1, y1)};
}
-
- // 判断两条线段是否相交并计算交点
- // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
- public static Object[] lineIntersection(Point[] L1, Point[] L2) {
- double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
- double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
-
- // 判断是否相交（叉积）
- double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
- double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
- double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
- double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
-
- // 如果叉积条件满足，表示有交点
- // d1*d2<0意味着P3、P4分别在L12的两边
- // d3*d4<0意味着P1、P2分别在L34的两边
- // 两个同时满足说明有交点
- if (d1 * d2 < 0 && d3 * d4 < 0) {
- double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
- double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
- double x = x1 + t1 * (x2 - x1);
- double y = y1 + t1 * (y2 - y1);
- return new Object[]{true, new Point(x, y)};
- }
-
- // 检查是否起点或终点重合
- if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
- return new Object[]{true, new Point(x1, y1)};
- }
- if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
- return new Object[]{true, new Point(x2, y2)};
- }
-
- return new Object[]{false, null};
+ if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
+ return new Object[] {true, new Point(x2, y2)};
}
- // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
- public static double total_areal_displacement(List<Point> points, List<Point> points2, boolean debug) {
- double totalArea = 0;
- int i = 0, j = 0; // i for points, j for points2
- Point prevIntersection = null;
- int prevI = -1, prevJ = -1;
-
-// List<double[]> intersectionCoords = new ArrayList<>();
- List<Double> areaList = new ArrayList<>();
-
- while (i < points.size() - 1 && j < points2.size() - 1) {
- if (debug) {
- System.out.println("--------- " + i + " " + j + " ------------");
- }
-
- // 当前线段
- Point[] L1 = {points.get(i), points.get(i + 1)};
- Point[] L2 = {points2.get(j), points2.get(j + 1)};
-
- // 判断是否有交点
- Object[] result = lineIntersection(L1, L2);
- boolean isIntersect = (boolean) result[0];
- Point intersection = (Point) result[1];
-
- if (isIntersect) {
-// intersectionCoords.add(intersection);
-
- if (prevIntersection != null) {
- // 构造多边形点集
- List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
- polygon.add(prevIntersection);
- if (debug) {
- System.out.println("- start intersection: " + prevIntersection);
- }
- polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
-// polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i + 1))); // 添加当前线段的点，左闭右开
- if (debug) {
- System.out.println("- one side: " + points.subList(prevI, i + 1));
- }
- polygon.add(intersection);
- if (debug) {
- System.out.println("- end intersection: " + intersection);
- }
- List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
- Collections.reverse(tempPoints2); // 添加另一序列的点
- polygon.addAll(tempPoints2);
- if (debug) {
- System.out.println("- another side: " + tempPoints2);
- }
-
-// double[][] polygonArray = new double[polygon.size()][2];
-// for (int k = 0; k < polygon.size(); k++) {
-// polygonArray[k] = polygon.get(k);
-// }
-
- // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
- double area = calculatePolygonArea(polygon);
- if (debug) {
- System.out.println("Area = " + area);
- }
- totalArea += area;
- areaList.add(area);
- }
-
- prevIntersection = intersection;
- prevI = i + 1;
- prevJ = j + 1;
- if (debug) {
- System.out.println("This intersection = " + intersection
- + ", next polygon: side1 = " + prevI + ", side2 = " + prevJ);
- }
- }
-
-
- int currentI = i; // 临时存储i
- int currentJ = j; // 临时存储j
- if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
- i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
- }
- if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
- j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
- }
- } // end of while
+ return new Object[] {false, null};
+ }
+
+ // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
+ public static double total_areal_displacement(
+ List<Point> points, List<Point> points2, boolean debug) {
+ double totalArea = 0;
+ int i = 0, j = 0; // i for points, j for points2
+ Point prevIntersection = null;
+ int prevI = -1, prevJ = -1;
+
+ // List<double[]> intersectionCoords = new ArrayList<>();
+ List<Double> areaList = new ArrayList<>();
+
+ while (i < points.size() - 1 && j < points2.size() - 1) {
+ if (debug) {
+ System.out.println("--------- " + i + " " + j + " ------------");
+ }
+
+ // 当前线段
+ Point[] L1 = {points.get(i), points.get(i + 1)};
+ Point[] L2 = {points2.get(j), points2.get(j + 1)};
+
+ // 判断是否有交点
+ Object[] result = lineIntersection(L1, L2);
+ boolean isIntersect = (boolean) result[0];
+ Point intersection = (Point) result[1];
+
+ if (isIntersect) {
+ // intersectionCoords.add(intersection);
+
+ if (prevIntersection != null) {
+ // 构造多边形点集
+ List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
+ polygon.add(prevIntersection);
+ if (debug) {
+ System.out.println("- start intersection: " + prevIntersection);
+ }
+ polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
+ // polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i +
+ // 1))); // 添加当前线段的点，左闭右开
+ if (debug) {
+ System.out.println("- one side: " + points.subList(prevI, i + 1));
+ }
+ polygon.add(intersection);
+ if (debug) {
+ System.out.println("- end intersection: " + intersection);
+ }
+ List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
+ Collections.reverse(tempPoints2); // 添加另一序列的点
+ polygon.addAll(tempPoints2);
+ if (debug) {
+ System.out.println("- another side: " + tempPoints2);
+ }
+
+ // double[][] polygonArray = new double[polygon.size()][2];
+ // for (int k = 0; k < polygon.size(); k++) {
+ // polygonArray[k] = polygon.get(k);
+ // }
+
+ // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
+ double area = calculatePolygonArea(polygon);
+ if (debug) {
+ System.out.println("Area = " + area);
+ }
+ totalArea += area;
+ areaList.add(area);
+ }
+ prevIntersection = intersection;
+ prevI = i + 1;
+ prevJ = j + 1;
if (debug) {
- System.out.println(areaList);
+ System.out.println(
+ "This intersection = "
+ + intersection
+ + ", next polygon: side1 = "
+ + prevI
+ + ", side2 = "
+ + prevJ);
}
-
- return totalArea;
+ }
+
+ int currentI = i; // 临时存储i
+ int currentJ = j; // 临时存储j
+ if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
+ i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
+ j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ } // end of while
+
+ if (debug) {
+ System.out.println(areaList);
}
- // 测试方法
- public static void main(String[] args) {
- // 示例数据
- List<Point> points = new ArrayList<>();
- points.add(new Point(0, 0));
- points.add(new Point(1, 2));
- points.add(new Point(2, -20));
- points.add(new Point(3, 0));
- points.add(new Point(4, -1));
- points.add(new Point(5, -1.5));
- points.add(new Point(6, 0));
-
- List<Point> points2 = new ArrayList<>();
-// points2.add(points.get(0));
-// points2.add(points.get(3));
-// points2.add(points.get(6));
- points2.add(new Point(1,-10));
- points2.add(new Point(3,0));
-
- double area = total_areal_displacement(points, points2, true);
- System.out.println("Total area: " + area);
-
-// points = new ArrayList<>();
-// points.add(new Point(0, 0));
-// points.add(new Point(1, 2));
-// points.add(new Point(1.5, -10));
-// double area = calculatePolygonArea(points);
-// System.out.println(area);
-//
-// Point[] L1 = new Point[2];
-// L1[0] = new Point(1, 2);
-// L1[1] = new Point(2, -2);
-// Point[] L2 = new Point[2];
-// L2[0] = new Point(0, 0);
-// L2[1] = new Point(3, 0);
-// Object[] res = lineIntersection(L1, L2);
-// System.out.println(res[1]);
- }
+ return totalArea;
+ }
+
+ // 测试方法
+ public static void main(String[] args) {
+ // 示例数据
+ List<Point> points = new ArrayList<>();
+ points.add(new Point(0, 0));
+ points.add(new Point(1, 2));
+ points.add(new Point(2, -20));
+ points.add(new Point(3, 0));
+ points.add(new Point(4, -1));
+ points.add(new Point(5, -1.5));
+ points.add(new Point(6, 0));
+
+ List<Point> points2 = new ArrayList<>();
+ // points2.add(points.get(0));
+ // points2.add(points.get(3));
+ // points2.add(points.get(6));
+ points2.add(new Point(1, -10));
+ points2.add(new Point(3, 0));
+
+ double area = total_areal_displacement(points, points2, true);
+ System.out.println("Total area: " + area);
+
+ // points = new ArrayList<>();
+ // points.add(new Point(0, 0));
+ // points.add(new Point(1, 2));
+ // points.add(new Point(1.5, -10));
+ // double area = calculatePolygonArea(points);
+ // System.out.println(area);
+ //
+ // Point[] L1 = new Point[2];
+ // L1[0] = new Point(1, 2);
+ // L1[1] = new Point(2, -2);
+ // Point[] L2 = new Point[2];
+ // L2[0] = new Point(0, 0);
+ // L2[1] = new Point(3, 0);
+ // Object[] res = lineIntersection(L1, L2);
+ // System.out.println(res[1]);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Triangle.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Triangle.java
index 79f7d440c99..b4c52ebfe3f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Triangle.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Triangle.java
@@ -4,46 +4,46 @@ package org.apache.iotdb.db.query.eBUG;
// https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
public class Triangle {
- int[] indices = new int[3];
- double area;
- Triangle prev;
- Triangle next;
- boolean isDeleted;
-
- public Triangle(int index1, int index2, int index3, double area) {
- this.indices[0] = index1;
- this.indices[1] = index2;
- this.indices[2] = index3;
- this.area = area;
- this.isDeleted = false; // flag for removal. Avoid using heap.remove(x) as it is O(n) complexity
+ int[] indices = new int[3];
+ double area;
+ Triangle prev;
+ Triangle next;
+ boolean isDeleted;
+
+ public Triangle(int index1, int index2, int index3, double area) {
+ this.indices[0] = index1;
+ this.indices[1] = index2;
+ this.indices[2] = index3;
+ this.area = area;
+ this.isDeleted = false; // flag for removal. Avoid using heap.remove(x) as it is O(n) complexity
+ }
+
+ public Triangle(Triangle oldTri) {
+ // deep copy and inherit connection
+
+ this.indices[0] = oldTri.indices[0];
+ this.indices[1] = oldTri.indices[1];
+ this.indices[2] = oldTri.indices[2];
+ this.area = oldTri.area;
+ this.prev = oldTri.prev;
+ this.next = oldTri.next;
+
+ // TODO important! inherit connection relationship to this new point
+ if (this.prev != null) { // previous point to this new point
+ this.prev.next = this;
}
-
- public Triangle(Triangle oldTri) {
- // deep copy and inherit connection
-
- this.indices[0] = oldTri.indices[0];
- this.indices[1] = oldTri.indices[1];
- this.indices[2] = oldTri.indices[2];
- this.area = oldTri.area;
- this.prev = oldTri.prev;
- this.next = oldTri.next;
-
- // TODO important! inherit connection relationship to this new point
- if (this.prev != null) { // previous point to this new point
- this.prev.next = this;
- }
- if (this.next != null) { // next point to this new point
- this.next.prev = this;
- }
-
- this.isDeleted = false; // this new triangle is not deleted
+ if (this.next != null) { // next point to this new point
+ this.next.prev = this;
}
- public boolean isValid() {
- return !isDeleted;
- }
+ this.isDeleted = false; // this new triangle is not deleted
+ }
- public void markDeleted() {
- this.isDeleted = true;
- }
+ public boolean isValid() {
+ return !isDeleted;
+ }
+
+ public void markDeleted() {
+ this.isDeleted = true;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index e177b1dcf20..201f99d2313 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -27,341 +27,390 @@ import java.util.*;
import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-
public class eBUG {
- public static List<Point> findEliminated(Polyline lineToSimplify, int pa_idx, int pb_idx) {
- // pa: left adjacent non-eliminated point of pi
- // pb: right adjacent non-eliminated point of pi
- // return a list of points, T'_max{0,k-e}[a:b] order by time in ascending order
-
- // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！至于其他全局更早淘汰的点则不一定位于a:b之间
- // pa,xxx,pi,xxx,pb
- // when e=0，遍历点数就是3
- // when e>0，遍历点数大于等于4、小于等于e+3
-
- List<Point> res = new ArrayList<>();
- Point start = lineToSimplify.get(pa_idx);
- Point end = lineToSimplify.get(pb_idx);
-// int cnt = 0;
- while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- res.add(start);
- start = start.next; // when e=0, only traversing three points pa pi pb
-// cnt += 1;
- }
- res.add(end);
-// cnt += 1;
-// System.out.println(cnt); // 3<=cnt<=e+3
+ public static List<Point> findEliminated(Polyline lineToSimplify, int pa_idx, int pb_idx) {
+ // pa: left adjacent non-eliminated point of pi
+ // pb: right adjacent non-eliminated point of pi
+ // return a list of points, T'_max{0,k-e}[a:b] order by time in ascending order
+
+ // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！至于其他全局更早淘汰的点则不一定位于a:b之间
+ // pa,xxx,pi,xxx,pb
+ // when e=0，遍历点数就是3
+ // when e>0，遍历点数大于等于4、小于等于e+3
+
+ List<Point> res = new ArrayList<>();
+ Point start = lineToSimplify.get(pa_idx);
+ Point end = lineToSimplify.get(pb_idx);
+ // int cnt = 0;
+ while (start
+ != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ res.add(start);
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ // cnt += 1;
+ }
+ res.add(end);
+ // cnt += 1;
+ // System.out.println(cnt); // 3<=cnt<=e+3
+
+ return res;
+ }
+
+ public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
+ // precomputation mode
+ return buildEffectiveArea(lineToSimplify, e, debug, 0);
+ }
+
+ public static List<Point> buildEffectiveArea(
+ Polyline lineToSimplify, int e, boolean debug, int m) {
+ if (e < 0) {
+ throw new IllegalArgumentException("Parameter e must be non-negative.");
+ }
- return res;
+ if (m > 2) {
+ System.out.println(
+ "online sampling mode, "
+ + "returning "
+ + m
+ + " sampled points sorted by time in ascending order");
+ } else {
+ System.out.println(
+ "offline precomputation mode, "
+ + "returning each point sorted by dominated significance (DS) in ascending order");
}
- public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
- // precomputation mode
- return buildEffectiveArea(lineToSimplify, e, debug, 0);
+ // List<Point> results = lineToSimplify.getVertices(); // 浅复制
+ // TODO 预计算结果改成按照bottom-up逐点淘汰顺序（DS递增）排列而不是按照时间戳，这样省去在线时对DS排序的过程
+ // add的是Point引用，所以没有多用一倍的空间
+ List<Point> resultsBottomUpEliminated = new ArrayList<>();
+
+ // 存储的是点的引用，这样可以修改原来序列里点的淘汰状态
+ // 存储的是距离当前最新状态的滞后的尚未施加、待施加的e个淘汰点
+ LinkedList<Point> laggedEliminatedPoints = new LinkedList<>();
+
+ int total = lineToSimplify.size();
+ if (total < 3) {
+ return lineToSimplify.getVertices(); // 不足 3 个点无法形成三角形
}
- public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug, int m) {
- if (e < 0) {
- throw new IllegalArgumentException("Parameter e must be non-negative.");
- }
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ lineToSimplify.get(0).prev = null;
+ lineToSimplify.get(0).next = lineToSimplify.get(1);
+ lineToSimplify.get(total - 1).next = null;
+ lineToSimplify.get(total - 1).prev = lineToSimplify.get(total - 2);
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+ double area =
+ triArea(
+ lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
+ triangles[i - 1] = new Triangle(index1, index2, index3, area);
+
+ // 初始化点的状态 for eBUG usage
+ lineToSimplify.get(i).prev = lineToSimplify.get(i - 1);
+ lineToSimplify.get(i).next = lineToSimplify.get(i + 1);
+ }
- if (m > 2) {
- System.out.println("online sampling mode, " +
- "returning " + m + " sampled points sorted by time in ascending order");
- } else {
- System.out.println("offline precomputation mode, " +
- "returning each point sorted by dominated significance (DS) in ascending order");
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ }
+
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+
+ double previousEA = -1;
+ // while (!triangleHeap.isEmpty()) { // TODO 注意triangleHeap里装的是non-terminal point对应的三角形
+ // TODO 在线采样m个点，也就是说最后留下m-2个non-terminal point
+ // 注意：triangleHeap里装的包括了标记删除的点！所以triangleHeap.size()不是真正的留下的未被淘汰点数！
+ // 因此需要用fakeCnt来统计heap里的非真实点数，这样triangleHeap.size()-fakeCnt就是真正的留下的未被淘汰点数
+ int remainNonTerminalPoint = Math.max(0, m - 2);
+ int fakeCnt = 0;
+ while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll(); // O(logn)
+
+ // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ if (tri.isDeleted) {
+ fakeCnt--; // 取出了一个被标记删除点
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ continue;
+ }
+
+ // 真正的淘汰点
+ // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ if (debug) {
+ System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
+ }
+ if (e > 0) {
+ if (laggedEliminatedPoints.size() == e) {
+ // 已经有e个滞后淘汰点了，为了把最近淘汰点加入，得先把最老的淘汰点施加上去
+ Point removedPoint = laggedEliminatedPoints.removeFirst(); // 取出最早加入的淘汰点 复杂度1
+ removedPoint.markEliminated(); // 注意是引用，所以改的内容后面可以看到
+ }
+ laggedEliminatedPoints.addLast(lineToSimplify.get(tri.indices[1])); // 加入最新的淘汰点，滞后在这里先不施加
+ } else { // e=0 没有滞后 立即标记删除 T'_k
+ lineToSimplify.get(tri.indices[1]).markEliminated();
+ }
+ if (debug) {
+ System.out.println(
+ "the e most recently eliminated points (lagged):" + laggedEliminatedPoints);
+ }
+
+ // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
+ if (tri.area > previousEA) {
+ previousEA = tri.area;
+ }
+ // results.get(tri.indices[1]).z = previousEA; // dominated significance
+ lineToSimplify.get(tri.indices[1]).z = previousEA; // dominated significance
+ resultsBottomUpEliminated.add(
+ lineToSimplify.get(tri.indices[1])); // TODO add的是Point引用，所以没有多用一倍的空间
+ if (debug) {
+ System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
+ }
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
+
+ Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+
+ // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // 前一个三角形连到后一个三角形
+ tri.prev.next =
+ tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ tri.prev.indices[2] = tri.indices[2];
+
+ // e parameter
+ List<Point> pointsForSig =
+ findEliminated(lineToSimplify, tri.prev.indices[0], tri.prev.indices[2]);
+ if (debug) {
+ System.out.println(
+ "(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
+ System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
+ for (Point point : pointsForSig) {
+ System.out.println("\t" + point);
+ }
+ }
+ List<Point> baseLine = new ArrayList<>();
+ baseLine.add(pointsForSig.get(0));
+ baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
+ double sig = total_areal_displacement(pointsForSig, baseLine, false);
+ tri.prev.area = sig;
+
+ if (debug) {
+ System.out.println("sig=" + sig);
+ double tmpTri =
+ triArea(
+ lineToSimplify.get(tri.prev.indices[0]),
+ lineToSimplify.get(tri.prev.indices[1]),
+ lineToSimplify.get(tri.prev.indices[2]));
+ System.out.println(
+ "\t"
+ + "tri="
+ + tmpTri
+ + ", "
+ + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
}
-// List<Point> results = lineToSimplify.getVertices(); // 浅复制
- // TODO 预计算结果改成按照bottom-up逐点淘汰顺序（DS递增）排列而不是按照时间戳，这样省去在线时对DS排序的过程
- // add的是Point引用，所以没有多用一倍的空间
- List<Point> resultsBottomUpEliminated = new ArrayList<>();
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add来显式重新插入修改后的元素
+ triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
- // 存储的是点的引用，这样可以修改原来序列里点的淘汰状态
- // 存储的是距离当前最新状态的滞后的尚未施加、待施加的e个淘汰点
- LinkedList<Point> laggedEliminatedPoints = new LinkedList<>();
+ if (tri.next != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
- int total = lineToSimplify.size();
- if (total < 3) {
- return lineToSimplify.getVertices(); // 不足 3 个点无法形成三角形
- }
+ // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+ tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- lineToSimplify.get(0).prev = null;
- lineToSimplify.get(0).next = lineToSimplify.get(1);
- lineToSimplify.get(total - 1).next = null;
- lineToSimplify.get(total - 1).prev = lineToSimplify.get(total - 2);
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
- triangles[i - 1] = new Triangle(index1, index2, index3, area);
-
- // 初始化点的状态 for eBUG usage
- lineToSimplify.get(i).prev = lineToSimplify.get(i - 1);
- lineToSimplify.get(i).next = lineToSimplify.get(i + 1);
- }
+ Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ if (tri.prev != null) {
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
}
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
-
- double previousEA = -1;
-// while (!triangleHeap.isEmpty()) { // TODO 注意triangleHeap里装的是non-terminal point对应的三角形
- // TODO 在线采样m个点，也就是说最后留下m-2个non-terminal point
- // 注意：triangleHeap里装的包括了标记删除的点！所以triangleHeap.size()不是真正的留下的未被淘汰点数！
- // 因此需要用fakeCnt来统计heap里的非真实点数，这样triangleHeap.size()-fakeCnt就是真正的留下的未被淘汰点数
- int remainNonTerminalPoint = Math.max(0, m - 2);
- int fakeCnt = 0;
- while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll(); // O(logn)
-
- // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
- // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
- if (tri.isDeleted) {
- fakeCnt--; // 取出了一个被标记删除点
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
- continue;
- }
-
- // 真正的淘汰点
- // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
- if (debug) {
- System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
- }
- if (e > 0) {
- if (laggedEliminatedPoints.size() == e) {
- // 已经有e个滞后淘汰点了，为了把最近淘汰点加入，得先把最老的淘汰点施加上去
- Point removedPoint = laggedEliminatedPoints.removeFirst(); // 取出最早加入的淘汰点 复杂度1
- removedPoint.markEliminated(); // 注意是引用，所以改的内容后面可以看到
- }
- laggedEliminatedPoints.addLast(lineToSimplify.get(tri.indices[1])); // 加入最新的淘汰点，滞后在这里先不施加
- } else { // e=0 没有滞后 立即标记删除 T'_k
- lineToSimplify.get(tri.indices[1]).markEliminated();
- }
- if (debug) {
- System.out.println("the e most recently eliminated points (lagged):" + laggedEliminatedPoints);
- }
-
- // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
- if (tri.area > previousEA) {
- previousEA = tri.area;
- }
-// results.get(tri.indices[1]).z = previousEA; // dominated significance
- lineToSimplify.get(tri.indices[1]).z = previousEA; // dominated significance
- resultsBottomUpEliminated.add(lineToSimplify.get(tri.indices[1])); // TODO add的是Point引用，所以没有多用一倍的空间
- if (debug) {
- System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
- }
-
- // 更新相邻三角形
- if (tri.prev != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-
- Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
- // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
-
- // 2. 处理pi被淘汰引起tri.prev被更新的事情
- // 前一个三角形连到后一个三角形
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
- tri.prev.indices[2] = tri.indices[2];
-
- // e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, tri.prev.indices[0], tri.prev.indices[2]);
- if (debug) {
- System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
- System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- double sig = total_areal_displacement(pointsForSig, baseLine, false);
- tri.prev.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri = triArea(lineToSimplify.get(tri.prev.indices[0]),
- lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2]));
- System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add来显式重新插入修改后的元素
- triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
-
- if (tri.next != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.next被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
- tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
-
- Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
- // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
-
- if (tri.prev != null) {
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
- }
-
- // 2. 处理pi被淘汰引起tri.next被更新的事情
- tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
- tri.next.indices[0] = tri.indices[0];
-
- // e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, tri.next.indices[0], tri.next.indices[2]);
- if (debug) {
- System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
- System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- double sig = total_areal_displacement(pointsForSig, baseLine, false);
- tri.next.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri = triArea(lineToSimplify.get(tri.next.indices[0]),
- lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2]));
- System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
- triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
+ // 2. 处理pi被淘汰引起tri.next被更新的事情
+ tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ tri.next.indices[0] = tri.indices[0];
+
+ // e parameter
+ List<Point> pointsForSig =
+ findEliminated(lineToSimplify, tri.next.indices[0], tri.next.indices[2]);
+ if (debug) {
+ System.out.println(
+ "(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
+ System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
+ for (Point point : pointsForSig) {
+ System.out.println("\t" + point);
+ }
}
-
- if (m > 2) { // online sampling mode
- // 把滞后的淘汰点施加上去，然后返回在线采样结果（也就是返回剩余未被淘汰的点）
- for (Point p : laggedEliminatedPoints) {
- p.markEliminated();
- if (debug) {
- System.out.println("apply lagged elimination of " + p);
- }
- }
- List<Point> onlineSampled = new ArrayList<>();
- Point start = lineToSimplify.get(0);
- Point end = lineToSimplify.get(lineToSimplify.size() - 1);
- while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
- start = start.next; // when e=0, only traversing three points pa pi pb
- }
- onlineSampled.add(end);
- return onlineSampled;
- } else { // offline precomputation mode, for precomputing the dominated significance of each point
-// return results; // 注意这就是lineToSimplify.getVertices()
- // TODO
- resultsBottomUpEliminated.add(lineToSimplify.get(0)); // 全局首点
- resultsBottomUpEliminated.add(lineToSimplify.get(lineToSimplify.size() - 1)); // 全局尾点
- return resultsBottomUpEliminated;
+ List<Point> baseLine = new ArrayList<>();
+ baseLine.add(pointsForSig.get(0));
+ baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
+ double sig = total_areal_displacement(pointsForSig, baseLine, false);
+ tri.next.area = sig;
+
+ if (debug) {
+ System.out.println("sig=" + sig);
+ double tmpTri =
+ triArea(
+ lineToSimplify.get(tri.next.indices[0]),
+ lineToSimplify.get(tri.next.indices[1]),
+ lineToSimplify.get(tri.next.indices[2]));
+ System.out.println(
+ "\t"
+ + "tri="
+ + tmpTri
+ + ", "
+ + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
}
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
}
- public static void main(String[] args) {
-
-// int eParam = 0;
-// int[] eParamList = {0, 0, 1, 2, 3, 4, 5, 10, 14, 15, 16, 20, 30, 40, 50, 100, 200, 500, 1000, 2000, 5000,
-// 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000, 300_0000,
-// 500_0000, 800_0000, 1000_0000,
-// 1500_0000, 2000_0000, 2500_0000, 3000_0000};
- int[] eParamList = {100_0000};
- for (int eParam : eParamList) {
- try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
- for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
- // TODO 注意 要有一个点是n=300w
-
- Polyline polyline = new Polyline();
- int seed = 1;
- Random rand = new Random(seed); // TODO 注意seed在每轮里重设
- for (int i = 0; i < n; i++) {
- double v = rand.nextInt(1000000);
- polyline.addVertex(new Point(i, v));
- }
-
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false, 0);
- long endTime = System.currentTimeMillis();
-
- System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
- writer.println(n + "," + eParam + "," + (endTime - startTime));
- System.out.println(results.size());
-
-// // clear
-// polyline.clear();
-// results.clear();
-// polyline = null;
-// results = null;
-// System.gc();
- }
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
- }
+ if (m > 2) { // online sampling mode
+ // 把滞后的淘汰点施加上去，然后返回在线采样结果（也就是返回剩余未被淘汰的点）
+ for (Point p : laggedEliminatedPoints) {
+ p.markEliminated();
+ if (debug) {
+ System.out.println("apply lagged elimination of " + p);
}
-
-// int n = 300_0000;
-//
-// int seed = 1;
-// Random rand = new Random(seed); // TODO 注意seed在每轮里重设
-// Polyline polyline = new Polyline(); // 注意：如果是固定n变化e实验，这个数据要一次性生成，不然每次随机不一样
-// for (int i = 0; i < n; i++) {
-// double v = rand.nextInt(1000000);
-// polyline.addVertex(new Point(i, v)); // point的状态会在buildEffectiveArea里重置的，所以在下面的e遍历里可以复用这一份数据
-// }
-//
-// try (PrintWriter writer = new PrintWriter(new File("exp_varyE_n" + n + ".csv"))) {
-// int[] eParamList = {0, 1, 2, 10, 1000, 10000, 10_0000, 30_0000, 50_0000, 60_0000, 80_0000, 100_0000, 150_0000, 200_0000};
-// for (int eParam : eParamList) {
-// eParam *= 3; // TODO 注意 因为n=300w而不是100w
-//
-// long startTime = System.currentTimeMillis();
-// List<Point> results = buildEffectiveArea(polyline, eParam, false);
-// long endTime = System.currentTimeMillis();
-//
-// System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
-// writer.println(n + "," + eParam + "," + (endTime - startTime));
-// System.out.println(results.size());
-//
-// // clear 注意不能把原始数据清除，还要接着用
-//// System.gc();
-// }
-// } catch (FileNotFoundException e) {
-// throw new RuntimeException(e);
-// }
-
- System.out.println("finish");
+ }
+ List<Point> onlineSampled = new ArrayList<>();
+ Point start = lineToSimplify.get(0);
+ Point end = lineToSimplify.get(lineToSimplify.size() - 1);
+ while (start
+ != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ }
+ onlineSampled.add(end);
+ return onlineSampled;
+ } else { // offline precomputation mode, for precomputing the dominated significance of each
+ // point
+ // return results; // 注意这就是lineToSimplify.getVertices()
+ // TODO
+ resultsBottomUpEliminated.add(lineToSimplify.get(0)); // 全局首点
+ resultsBottomUpEliminated.add(lineToSimplify.get(lineToSimplify.size() - 1)); // 全局尾点
+ return resultsBottomUpEliminated;
}
+ }
+
+ public static void main(String[] args) {
+ // 实验：预计算耗时关于两个参数e、n的变化规律，是否符合复杂度理论建模
+
+ // int eParam = 0;
+ // int[] eParamList = {0, 0, 1, 2, 3, 4, 5, 10, 14, 15, 16, 20, 30, 40, 50, 100, 200,
+ // 500, 1000, 2000, 5000,
+ // 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000, 300_0000,
+ // 500_0000, 800_0000, 1000_0000,
+ // 1500_0000, 2000_0000, 2500_0000, 3000_0000};
+ int[] eParamList = {100_0000};
+ for (int eParam : eParamList) {
+ try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
+ for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
+ // TODO 注意 要有一个点是n=300w
+
+ Polyline polyline = new Polyline();
+ int seed = 1;
+ Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+ for (int i = 0; i < n; i++) {
+ double v = rand.nextInt(1000000);
+ polyline.addVertex(new Point(i, v));
+ }
+
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false, 0);
+ long endTime = System.currentTimeMillis();
+
+ System.out.println(
+ "n="
+ + n
+ + ", e="
+ + eParam
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+ writer.println(n + "," + eParam + "," + (endTime - startTime));
+ System.out.println(results.size());
+
+ // // clear
+ // polyline.clear();
+ // results.clear();
+ // polyline = null;
+ // results = null;
+ // System.gc();
+ }
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // int n = 300_0000;
+ //
+ // int seed = 1;
+ // Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+ // Polyline polyline = new Polyline(); // 注意：如果是固定n变化e实验，这个数据要一次性生成，不然每次随机不一样
+ // for (int i = 0; i < n; i++) {
+ // double v = rand.nextInt(1000000);
+ // polyline.addVertex(new Point(i, v)); //
+ // point的状态会在buildEffectiveArea里重置的，所以在下面的e遍历里可以复用这一份数据
+ // }
+ //
+ // try (PrintWriter writer = new PrintWriter(new File("exp_varyE_n" + n + ".csv"))) {
+ // int[] eParamList = {0, 1, 2, 10, 1000, 10000, 10_0000, 30_0000, 50_0000, 60_0000,
+ // 80_0000, 100_0000, 150_0000, 200_0000};
+ // for (int eParam : eParamList) {
+ // eParam *= 3; // TODO 注意 因为n=300w而不是100w
+ //
+ // long startTime = System.currentTimeMillis();
+ // List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ // long endTime = System.currentTimeMillis();
+ //
+ // System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce
+ // points: " + (endTime - startTime) + "ms");
+ // writer.println(n + "," + eParam + "," + (endTime - startTime));
+ // System.out.println(results.size());
+ //
+ // // clear 注意不能把原始数据清除，还要接着用
+ //// System.gc();
+ // }
+ // } catch (FileNotFoundException e) {
+ // throw new RuntimeException(e);
+ // }
+
+ System.out.println("finish");
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_Build.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_Build.java
new file mode 100644
index 00000000000..94713f71ec8
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_Build.java
@@ -0,0 +1,139 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Stream;
+
+import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
+
+public class eBUG_Build {
+ // 输入一条时间序列 t,v
+ // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
+ // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
+ public static void main(String[] args) {
+ if (args.length < 5) {
+ System.out.println(
+ "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,eParam,(outDir)");
+ }
+ String input = args[0];
+ boolean hasHeader = Boolean.parseBoolean(args[1]);
+ int timeIdx = Integer.parseInt(args[2]);
+ int valueIdx = Integer.parseInt(args[3]);
+ int eParam = Integer.parseInt(args[4]);
+ String outDir;
+ if (args.length > 5) {
+ outDir = args[5]; // 表示输出文件保存到指定的文件夹
+ } else {
+ outDir = null; // 表示输出文件保存到input文件所在的文件夹
+ }
+ String outputFile = generateOutputFileName(input, eParam, outDir);
+
+ // 打印信息供用户确认
+ System.out.println("Input file: " + input);
+ System.out.println("Has header: " + hasHeader);
+ System.out.println("Time index: " + timeIdx);
+ System.out.println("Value index: " + valueIdx);
+ System.out.println("eParam: " + eParam);
+ System.out.println("outDir: " + outDir);
+ System.out.println("Output file: " + outputFile);
+
+ // 开始运算
+ Polyline polyline = new Polyline();
+
+ // Using Files.lines() for memory-efficient line-by-line reading
+ try (Stream<String> lines = Files.lines(Paths.get(input))) {
+ Iterator<String> iterator = lines.iterator();
+ int lineNumber = 0;
+
+ // Skip the header line if necessary
+ if (hasHeader && iterator.hasNext()) {
+ iterator.next(); // Skip the header
+ }
+
+ // Process each line
+ while (iterator.hasNext()) {
+ lineNumber++;
+ String line = iterator.next();
+ String[] columns = line.split(",");
+
+ if (columns.length > Math.max(timeIdx, valueIdx)) {
+ double time;
+ if (timeIdx < 0) {
+ time = lineNumber;
+ } else {
+ time = Double.parseDouble(columns[timeIdx]);
+ }
+ double value = Double.parseDouble(columns[valueIdx]);
+ polyline.addVertex(new Point(time, value));
+ // System.out.println("Line " + lineNumber + " - Time: " + time + ",
+ // Value: " + value);
+ } else {
+ System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
+ }
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false); // precomputation mode
+ long endTime = System.currentTimeMillis();
+
+ // System.out.println("result point number: " + results.size()); // precomputation
+ // mode所以自然是等于n个点
+ System.out.println(
+ "n="
+ + polyline.getVertices().size()
+ + ", e="
+ + eParam
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+
+ // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
+ // 写入表头
+ writer.write("z,x,y");
+ writer.newLine();
+
+ // 写入数据行，按顺序 z, x, y
+ for (Point point : results) {
+ writer.write(point.z + "," + point.x + "," + point.y);
+ writer.newLine();
+ }
+
+ System.out.println("CSV file written successfully to " + outputFile);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ // Method to generate the output file name based on input path
+ private static String generateOutputFileName(String inputFilePath, int e, String outDir) {
+ // Get the file name without extension
+ Path path = Paths.get(inputFilePath);
+ String fileNameWithoutExtension = path.getFileName().toString();
+ String name = fileNameWithoutExtension.substring(0, fileNameWithoutExtension.lastIndexOf('.'));
+
+ // Get the file extension
+ String extension =
+ path.getFileName().toString().substring(fileNameWithoutExtension.lastIndexOf('.'));
+
+ // Create output file path by appending '-ds' before the extension
+ String outputFile = name + "-ds-e" + e + extension;
+
+ if (outDir == null) { // 表示使用input文件所在的文件夹
+ // Handle path compatibility for different operating systems (Linux/Windows)
+ return path.getParent().resolve(outputFile).toString();
+ } else {
+ Path out = Paths.get(outDir);
+ return out.resolve(outputFile).toString();
+ }
+ }
+}
(iotdb) 19/44: Add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 98cdae627c79981b57c75de2eceec1913d99d1f9
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Jan 23 02:56:14 2025 +0800
Add
---
.../java/org/apache/iotdb/db/query/eBUG/Point.java | 20 ++
.../org/apache/iotdb/db/query/eBUG/Polyline.java | 4 +
.../java/org/apache/iotdb/db/query/eBUG/Test1.java | 6 +-
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 175 +++++------
.../eBUG/eBUG_circulateArrayForEliminated.java | 325 +++++++++++++++++++++
5 files changed, 439 insertions(+), 91 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
index 77ebdb4cb39..70aa19ea531 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
@@ -4,10 +4,30 @@ public class Point {
double x, y, z;
+// public boolean eliminated; // 滞后e个淘汰点 for eBUG usage
+ public Point prev; // 指向T'_max{0,k-e}里当前点的前一个点 for eBUG usage
+ public Point next; // 指向T'_max{0,k-e}里当前点的后一个点 for eBUG usage
+ // 注意Triangle里的prev&next用于最新状态下存在三角形之间的连接关系，
+ // 而这里Point的prev&next用于滞后e个淘汰点（也就是最近的e个淘汰点先不施加）状态下的未淘汰点之间的连接关系
+
public Point(double x, double y) {
this.x = x;
this.y = y;
this.z = Double.POSITIVE_INFINITY; // effective area
+
+ // for eBUG usage
+// this.eliminated = false;
+ this.prev = null;
+ this.next = null;
+ }
+
+ public void markEliminated() {
+// eliminated = true;
+
+ // to avoid traversing each point between pa to pb,
+ // instead only traversing at most e most recently eliminated points lagged
+ prev.next = next;
+ next.prev = prev;
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
index bed14731832..1ddfe2ebbe1 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
@@ -8,6 +8,10 @@ public class Polyline {
private List<Point> vertices = new ArrayList<>();
public void addVertex(Point point) {
+// if (!vertices.isEmpty()) { //before adding this point
+// vertices.get(vertices.size() - 1).next = point;
+// point.prev = vertices.get(vertices.size() - 1);
+// }
vertices.add(point);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
index 56e73cc17e0..28b0eebd53e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
@@ -43,11 +43,7 @@ public class Test1 {
}
System.out.println("---------------------------------");
-// List<Point> results = new ArrayList<>();
- // 计算运行时间
-// int eParam = 10;
-// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
- int eParam = 2;
+ int eParam = 1000000;
long startTime = System.currentTimeMillis();
List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 77e21a27067..906bc4e2366 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -29,56 +29,35 @@ import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
public class eBUG {
- public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
- int pa_idx, int pi_idx, int pb_idx) {
- // 复杂度分析 记c=b-a。
- // 如果e<c，那么从最近淘汰的e个点里寻找位于pa~pb之间的，找到的点和pa pi pb一起按照时间戳递增排序，后面鞋带公式计算面积：O(e+eloge)
- // 否则e>=c，直接取T[a:b]，已经排序号了，后面鞋带公式计算面积：O(c)
-
- // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
-
- // pi: the point whose significance needs updating
+ public static List<Point> findEliminated(Polyline lineToSimplify, int pa_idx, int pb_idx) {
// pa: left adjacent non-eliminated point of pi
// pb: right adjacent non-eliminated point of pi
- // return a list of points, containing pa,p,pb and points between pa&pa from the e most recently eliminated points,
- // order by time in ascending order
+ // return a list of points, T'_max{0,k-e}[a:b] order by time in ascending order
- Point pa = lineToSimplify.get(pa_idx);
- Point pi = lineToSimplify.get(pi_idx);
- Point pb = lineToSimplify.get(pb_idx);
+ // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！至于其他全局更早淘汰的点则不一定位于a:b之间
+ // pa,xxx,pi,xxx,pb
+ // when e=0，遍历点数就是3
+ // when e>0，遍历点数大于等于4、小于等于e+3
List<Point> res = new ArrayList<>();
-
- // TODO note: here may happen because e=c=0 or just 0<c<=e
- if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
-// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
- res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
- return res; // 已经按照时间戳递增
+ Point start = lineToSimplify.get(pa_idx);
+ Point end = lineToSimplify.get(pb_idx);
+// int cnt = 0;
+ while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ res.add(start);
+ start = start.next; // when e=0, only traversing three points pa pi pb
+// cnt += 1;
}
-
- // TODO note: here may happen because c>e=0 or c>e>0. the latter needs search among recentEliminated and sort by timestamp
- // 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
-// System.out.println("k>=[c=(b-a-2)]>e, search among e");
- res.add(pa); // 注意pa pi pb已经按照时间戳递增
- res.add(pi);
- res.add(pb);
-
- // 包括了c>e=0的情况
- for (int idx : recentEliminated) { // e points
- if (idx == 0) { // init, not filled by real eliminated points yet
- break;
- }
- Point p = lineToSimplify.get(idx);
- if (p.x > pa.x && p.x < pb.x) {
- res.add(p);
- }
- }
-
- // 按照时间戳递增排序，这是为了后面计算total areal displacement需要
- if (recentEliminated.length > 0) { // 否则e=0，不需要排序pa pi pb三个点
- res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 这样的话时间复杂度变成e*log(e)
- }
-
+ res.add(end);
+// cnt += 1;
+// System.out.println(cnt); // 3<=cnt<=e+3
+
+// for (int i = pa_idx; i <= pb_idx; i++) { // 左闭右闭
+// Point p = lineToSimplify.get(i);
+// if (!p.eliminated) {
+// res.add(p);
+// }
+// }
return res;
}
@@ -90,8 +69,12 @@ public class eBUG {
List<Point> results = lineToSimplify.getVertices(); // 浅复制
// 用循环数组来记录最近淘汰的点
- int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
- int recentEliminatedIdx = 0; // index in the array, circulate
+// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
+// int recentEliminatedIdx = 0; // index in the array, circulate
+
+ // 存储的是点的引用，这样可以修改原来序列里点的淘汰状态
+ // 存储的是距离当前最新状态的滞后的尚未施加、待施加的e个淘汰点
+ LinkedList<Point> laggedEliminatedPoints = new LinkedList<>();
int total = lineToSimplify.size();
if (total < 3) {
@@ -102,10 +85,21 @@ public class eBUG {
Triangle[] triangles = new Triangle[nTriangles];
// 创建所有三角形并计算初始面积
+// lineToSimplify.get(0).eliminated = false; // 初始化点的状态 for eBUG usage
+ lineToSimplify.get(0).prev = null;
+ lineToSimplify.get(0).next = lineToSimplify.get(1);
+// lineToSimplify.get(total - 1).eliminated = false; // 初始化点的状态 for eBUG usage
+ lineToSimplify.get(total - 1).next = null;
+ lineToSimplify.get(total - 1).prev = lineToSimplify.get(total - 2);
for (int i = 1; i < total - 1; i++) {
int index1 = i - 1, index2 = i, index3 = i + 1;
double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
triangles[i - 1] = new Triangle(index1, index2, index3, area);
+
+ // 初始化点的状态 for eBUG usage
+// lineToSimplify.get(i).eliminated = false;
+ lineToSimplify.get(i).prev = lineToSimplify.get(i - 1);
+ lineToSimplify.get(i).next = lineToSimplify.get(i + 1);
}
// 设置三角形的前后关系
@@ -138,12 +132,18 @@ public class eBUG {
if (debug) {
System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
}
- if (e > 0) { // otherwise e=0
- recentEliminated[recentEliminatedIdx] = tri.indices[1];
- recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
+ if (e > 0) {
+ if (laggedEliminatedPoints.size() == e) {
+ // 已经有e个滞后淘汰点了，为了把最近淘汰点加入，得先把最老的淘汰点施加上去
+ Point removedPoint = laggedEliminatedPoints.removeFirst(); // 取出最早加入的淘汰点 复杂度1
+ removedPoint.markEliminated(); // 注意是引用，所以改的内容后面可以看到
+ }
+ laggedEliminatedPoints.addLast(lineToSimplify.get(tri.indices[1])); // 加入最新的淘汰点，滞后在这里先不施加
+ } else { // e=0 没有滞后 立即标记删除 T'_k
+ lineToSimplify.get(tri.indices[1]).markEliminated();
}
if (debug) {
- System.out.println("the e most recently eliminated points:" + Arrays.toString(recentEliminated));
+ System.out.println("the e most recently eliminated points (lagged):" + laggedEliminatedPoints);
}
// 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
@@ -172,12 +172,10 @@ public class eBUG {
tri.prev.indices[2] = tri.indices[2];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
- tri.prev.indices[0],
- tri.prev.indices[1],
- tri.prev.indices[2]); // TODO complexity ?
+ List<Point> pointsForSig = findEliminated(lineToSimplify, tri.prev.indices[0], tri.prev.indices[2]);
if (debug) {
System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
+ System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
for (Point point : pointsForSig) {
System.out.println("\t" + point);
}
@@ -221,12 +219,10 @@ public class eBUG {
tri.next.indices[0] = tri.indices[0];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
- tri.next.indices[0],
- tri.next.indices[1],
- tri.next.indices[2]); // TODO complexity
+ List<Point> pointsForSig = findEliminated(lineToSimplify, tri.next.indices[0], tri.next.indices[2]);
if (debug) {
System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
+ System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
for (Point point : pointsForSig) {
System.out.println("\t" + point);
}
@@ -259,45 +255,52 @@ public class eBUG {
public static void main(String[] args) {
- int eParam = 2000_0000;
- try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
- for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
- // TODO 注意 要有一个点是n=300w
-
- Polyline polyline = new Polyline();
- int seed = 1;
- Random rand = new Random(seed); // TODO 注意seed在每轮里重设
- for (int i = 0; i < n; i++) {
- double v = rand.nextInt(1000000);
- polyline.addVertex(new Point(i, v));
- }
+// int eParam = 0;
+ int[] eParamList = {0, 1, 2, 3, 4, 5, 10, 14, 15, 16, 20, 30, 40, 50, 100, 200, 500, 1000, 2000, 5000,
+ 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000, 300_0000,
+ 500_0000, 800_0000, 1000_0000,
+ 1500_0000, 2000_0000, 2500_0000, 3000_0000};
+ for (int eParam : eParamList) {
+ try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
+ for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
+ // TODO 注意 要有一个点是n=300w
+
+ Polyline polyline = new Polyline();
+ int seed = 1;
+ Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+ for (int i = 0; i < n; i++) {
+ double v = rand.nextInt(1000000);
+ polyline.addVertex(new Point(i, v));
+ }
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false);
- long endTime = System.currentTimeMillis();
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ long endTime = System.currentTimeMillis();
- System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
- writer.println(n + "," + eParam + "," + (endTime - startTime));
- System.out.println(results.size());
+ System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+ writer.println(n + "," + eParam + "," + (endTime - startTime));
+ System.out.println(results.size());
- // clear
- polyline.clear();
- results.clear();
- polyline = null;
- results = null;
- System.gc();
+// // clear
+// polyline.clear();
+// results.clear();
+// polyline = null;
+// results = null;
+// System.gc();
+ }
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
}
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
}
// int n = 300_0000;
+//
// int seed = 1;
// Random rand = new Random(seed); // TODO 注意seed在每轮里重设
// Polyline polyline = new Polyline(); // 注意：如果是固定n变化e实验，这个数据要一次性生成，不然每次随机不一样
// for (int i = 0; i < n; i++) {
// double v = rand.nextInt(1000000);
-// polyline.addVertex(new Point(i, v));
+// polyline.addVertex(new Point(i, v)); // point的状态会在buildEffectiveArea里重置的，所以在下面的e遍历里可以复用这一份数据
// }
//
// try (PrintWriter writer = new PrintWriter(new File("exp_varyE_n" + n + ".csv"))) {
@@ -314,7 +317,7 @@ public class eBUG {
// System.out.println(results.size());
//
// // clear 注意不能把原始数据清除，还要接着用
-// System.gc();
+//// System.gc();
// }
// } catch (FileNotFoundException e) {
// throw new RuntimeException(e);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_circulateArrayForEliminated.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_circulateArrayForEliminated.java
new file mode 100644
index 00000000000..38e9c96ad4e
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_circulateArrayForEliminated.java
@@ -0,0 +1,325 @@
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+//
+//package org.apache.iotdb.db.query.eBUG;
+//
+//import java.io.File;
+//import java.io.FileNotFoundException;
+//import java.io.PrintWriter;
+//import java.util.*;
+//
+//import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
+//import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
+//
+//
+//public class eBUG_circulateArrayForEliminated {
+// public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
+// int pa_idx, int pi_idx, int pb_idx) {
+// // 复杂度分析 记c=b-a。
+// // 如果e<c，那么从最近淘汰的e个点里寻找位于pa~pb之间的，找到的点和pa pi pb一起按照时间戳递增排序，后面鞋带公式计算面积：O(e+eloge)
+// // 否则e>=c，直接取T[a:b]，已经排序号了，后面鞋带公式计算面积：O(c)
+//
+// // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
+//
+// // pi: the point whose significance needs updating
+// // pa: left adjacent non-eliminated point of pi
+// // pb: right adjacent non-eliminated point of pi
+// // return a list of points, containing pa,p,pb and points between pa&pa from the e most recently eliminated points,
+// // order by time in ascending order
+//
+// Point pa = lineToSimplify.get(pa_idx);
+// Point pi = lineToSimplify.get(pi_idx);
+// Point pb = lineToSimplify.get(pb_idx);
+//
+// List<Point> res = new ArrayList<>();
+//
+// // TODO note: here may happen because e=c=0 or just 0<c<=e
+// if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
+//// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
+// res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
+// return res; // 已经按照时间戳递增
+// }
+//
+// // TODO note: here may happen because c>e=0 or c>e>0. the latter needs search among recentEliminated and sort by timestamp
+// // 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
+//// System.out.println("k>=[c=(b-a-2)]>e, search among e");
+// res.add(pa); // 注意pa pi pb已经按照时间戳递增
+// res.add(pi);
+// res.add(pb);
+//
+// // 包括了c>e=0的情况
+// for (int idx : recentEliminated) { // e points
+// if (idx == 0) { // init, not filled by real eliminated points yet
+// break;
+// }
+// Point p = lineToSimplify.get(idx);
+// if (p.x > pa.x && p.x < pb.x) {
+// res.add(p);
+// }
+// }
+//
+// // 按照时间戳递增排序，这是为了后面计算total areal displacement需要
+// if (recentEliminated.length > 0) { // 否则e=0，不需要排序pa pi pb三个点
+// res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 这样的话时间复杂度变成e*log(e)
+// }
+//
+// return res;
+// }
+//
+// public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
+// if (e < 0) {
+// throw new IllegalArgumentException("Parameter e must be non-negative.");
+// }
+//
+// List<Point> results = lineToSimplify.getVertices(); // 浅复制
+//
+// // 用循环数组来记录最近淘汰的点
+// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
+// int recentEliminatedIdx = 0; // index in the array, circulate
+//
+// int total = lineToSimplify.size();
+// if (total < 3) {
+// return results; // 不足 3 个点无法形成三角形
+// }
+//
+// int nTriangles = total - 2;
+// Triangle[] triangles = new Triangle[nTriangles];
+//
+// // 创建所有三角形并计算初始面积
+// for (int i = 1; i < total - 1; i++) {
+// int index1 = i - 1, index2 = i, index3 = i + 1;
+// double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
+// triangles[i - 1] = new Triangle(index1, index2, index3, area);
+// }
+//
+// // 设置三角形的前后关系
+// for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+// triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+// triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+// }
+//
+// // 使用优先队列构建 minHeap
+// PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+// Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+//
+// double previousEA = -1;
+// while (!triangleHeap.isEmpty()) {
+// // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+// // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+// Triangle tri = triangleHeap.poll(); // O(logn)
+//
+// // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+// // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+// if (tri.isDeleted) {
+// if (debug) {
+// System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
+// }
+// continue;
+// }
+//
+// // 真正的淘汰点
+// // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+// if (debug) {
+// System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
+// }
+// if (e > 0) { // otherwise e=0
+// recentEliminated[recentEliminatedIdx] = tri.indices[1];
+// recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
+// }
+// if (debug) {
+// System.out.println("the e most recently eliminated points:" + Arrays.toString(recentEliminated));
+// }
+//
+// // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
+// if (tri.area > previousEA) {
+// previousEA = tri.area;
+// }
+// results.get(tri.indices[1]).z = previousEA; // dominated significance
+// if (debug) {
+// System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
+// }
+//
+// // 更新相邻三角形
+// if (tri.prev != null) {
+// // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+//
+// // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+// // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+// tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
+//
+// Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+// // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+//
+// // 2. 处理pi被淘汰引起tri.prev被更新的事情
+// // 前一个三角形连到后一个三角形
+// tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+// tri.prev.indices[2] = tri.indices[2];
+//
+// // e parameter
+// List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
+// tri.prev.indices[0],
+// tri.prev.indices[1],
+// tri.prev.indices[2]); // TODO complexity ?
+// if (debug) {
+// System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
+// for (Point point : pointsForSig) {
+// System.out.println("\t" + point);
+// }
+// }
+// List<Point> baseLine = new ArrayList<>();
+// baseLine.add(pointsForSig.get(0));
+// baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
+// double sig = total_areal_displacement(pointsForSig, baseLine, false);
+// tri.prev.area = sig;
+//
+// if (debug) {
+// System.out.println("sig=" + sig);
+// double tmpTri = triArea(lineToSimplify.get(tri.prev.indices[0]),
+// lineToSimplify.get(tri.prev.indices[1]),
+// lineToSimplify.get(tri.prev.indices[2]));
+// System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
+// }
+//
+// // 重新加入堆
+// // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+// // 所以必须通过add来显式重新插入修改后的元素
+// triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+// }
+//
+// if (tri.next != null) {
+// // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+//
+// // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+// // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+// tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+//
+// Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+// // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
+//
+// if (tri.prev != null) {
+// tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
+// }
+//
+// // 2. 处理pi被淘汰引起tri.next被更新的事情
+// tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+// tri.next.indices[0] = tri.indices[0];
+//
+// // e parameter
+// List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
+// tri.next.indices[0],
+// tri.next.indices[1],
+// tri.next.indices[2]); // TODO complexity
+// if (debug) {
+// System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
+// for (Point point : pointsForSig) {
+// System.out.println("\t" + point);
+// }
+// }
+// List<Point> baseLine = new ArrayList<>();
+// baseLine.add(pointsForSig.get(0));
+// baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
+// double sig = total_areal_displacement(pointsForSig, baseLine, false);
+// tri.next.area = sig;
+//
+// if (debug) {
+// System.out.println("sig=" + sig);
+// double tmpTri = triArea(lineToSimplify.get(tri.next.indices[0]),
+// lineToSimplify.get(tri.next.indices[1]),
+// lineToSimplify.get(tri.next.indices[2]));
+// System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
+// }
+//
+// // 重新加入堆
+// // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+// // 所以必须通过add 来显式重新插入修改后的元素
+// triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+// }
+// if (debug) {
+// System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
+// }
+// }
+// return results; // 注意这就是lineToSimplify.getVertices()
+// }
+//
+// public static void main(String[] args) {
+//
+// int eParam = 2000_0000;
+// try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
+// for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
+// // TODO 注意 要有一个点是n=300w
+//
+// Polyline polyline = new Polyline();
+// int seed = 1;
+// Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+// for (int i = 0; i < n; i++) {
+// double v = rand.nextInt(1000000);
+// polyline.addVertex(new Point(i, v));
+// }
+//
+// long startTime = System.currentTimeMillis();
+// List<Point> results = buildEffectiveArea(polyline, eParam, false);
+// long endTime = System.currentTimeMillis();
+//
+// System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+// writer.println(n + "," + eParam + "," + (endTime - startTime));
+// System.out.println(results.size());
+//
+// // clear
+// polyline.clear();
+// results.clear();
+// polyline = null;
+// results = null;
+// System.gc();
+// }
+// } catch (FileNotFoundException e) {
+// throw new RuntimeException(e);
+// }
+//
+//// int n = 300_0000;
+//// int seed = 1;
+//// Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+//// Polyline polyline = new Polyline(); // 注意：如果是固定n变化e实验，这个数据要一次性生成，不然每次随机不一样
+//// for (int i = 0; i < n; i++) {
+//// double v = rand.nextInt(1000000);
+//// polyline.addVertex(new Point(i, v));
+//// }
+////
+//// try (PrintWriter writer = new PrintWriter(new File("exp_varyE_n" + n + ".csv"))) {
+//// int[] eParamList = {0, 1, 2, 10, 1000, 10000, 10_0000, 30_0000, 50_0000, 60_0000, 80_0000, 100_0000, 150_0000, 200_0000};
+//// for (int eParam : eParamList) {
+//// eParam *= 3; // TODO 注意 因为n=300w而不是100w
+////
+//// long startTime = System.currentTimeMillis();
+//// List<Point> results = buildEffectiveArea(polyline, eParam, false);
+//// long endTime = System.currentTimeMillis();
+////
+//// System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+//// writer.println(n + "," + eParam + "," + (endTime - startTime));
+//// System.out.println(results.size());
+////
+//// // clear 注意不能把原始数据清除，还要接着用
+//// System.gc();
+//// }
+//// } catch (FileNotFoundException e) {
+//// throw new RuntimeException(e);
+//// }
+//
+// System.out.println("finish");
+// }
+//}
(iotdb) 20/44: ok
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 33a6441223684353d90fae91c22c9e80c5d8b267
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Jan 23 16:31:50 2025 +0800
ok
---
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 9 +-
.../eBUG/eBUG_circulateArrayForEliminated.java | 325 ---------------------
2 files changed, 5 insertions(+), 329 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 906bc4e2366..408d440a46f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -256,10 +256,11 @@ public class eBUG {
public static void main(String[] args) {
// int eParam = 0;
- int[] eParamList = {0, 1, 2, 3, 4, 5, 10, 14, 15, 16, 20, 30, 40, 50, 100, 200, 500, 1000, 2000, 5000,
- 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000, 300_0000,
- 500_0000, 800_0000, 1000_0000,
- 1500_0000, 2000_0000, 2500_0000, 3000_0000};
+// int[] eParamList = {0, 0, 1, 2, 3, 4, 5, 10, 14, 15, 16, 20, 30, 40, 50, 100, 200, 500, 1000, 2000, 5000,
+// 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000, 300_0000,
+// 500_0000, 800_0000, 1000_0000,
+// 1500_0000, 2000_0000, 2500_0000, 3000_0000};
+ int[] eParamList = {100_0000};
for (int eParam : eParamList) {
try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_circulateArrayForEliminated.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_circulateArrayForEliminated.java
deleted file mode 100644
index 38e9c96ad4e..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_circulateArrayForEliminated.java
+++ /dev/null
@@ -1,325 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing,
-// * software distributed under the License is distributed on an
-// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// * KIND, either express or implied. See the License for the
-// * specific language governing permissions and limitations
-// * under the License.
-// */
-//
-//package org.apache.iotdb.db.query.eBUG;
-//
-//import java.io.File;
-//import java.io.FileNotFoundException;
-//import java.io.PrintWriter;
-//import java.util.*;
-//
-//import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
-//import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-//
-//
-//public class eBUG_circulateArrayForEliminated {
-// public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
-// int pa_idx, int pi_idx, int pb_idx) {
-// // 复杂度分析 记c=b-a。
-// // 如果e<c，那么从最近淘汰的e个点里寻找位于pa~pb之间的，找到的点和pa pi pb一起按照时间戳递增排序，后面鞋带公式计算面积：O(e+eloge)
-// // 否则e>=c，直接取T[a:b]，已经排序号了，后面鞋带公式计算面积：O(c)
-//
-// // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
-//
-// // pi: the point whose significance needs updating
-// // pa: left adjacent non-eliminated point of pi
-// // pb: right adjacent non-eliminated point of pi
-// // return a list of points, containing pa,p,pb and points between pa&pa from the e most recently eliminated points,
-// // order by time in ascending order
-//
-// Point pa = lineToSimplify.get(pa_idx);
-// Point pi = lineToSimplify.get(pi_idx);
-// Point pb = lineToSimplify.get(pb_idx);
-//
-// List<Point> res = new ArrayList<>();
-//
-// // TODO note: here may happen because e=c=0 or just 0<c<=e
-// if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
-//// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
-// res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
-// return res; // 已经按照时间戳递增
-// }
-//
-// // TODO note: here may happen because c>e=0 or c>e>0. the latter needs search among recentEliminated and sort by timestamp
-// // 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
-//// System.out.println("k>=[c=(b-a-2)]>e, search among e");
-// res.add(pa); // 注意pa pi pb已经按照时间戳递增
-// res.add(pi);
-// res.add(pb);
-//
-// // 包括了c>e=0的情况
-// for (int idx : recentEliminated) { // e points
-// if (idx == 0) { // init, not filled by real eliminated points yet
-// break;
-// }
-// Point p = lineToSimplify.get(idx);
-// if (p.x > pa.x && p.x < pb.x) {
-// res.add(p);
-// }
-// }
-//
-// // 按照时间戳递增排序，这是为了后面计算total areal displacement需要
-// if (recentEliminated.length > 0) { // 否则e=0，不需要排序pa pi pb三个点
-// res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 这样的话时间复杂度变成e*log(e)
-// }
-//
-// return res;
-// }
-//
-// public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
-// if (e < 0) {
-// throw new IllegalArgumentException("Parameter e must be non-negative.");
-// }
-//
-// List<Point> results = lineToSimplify.getVertices(); // 浅复制
-//
-// // 用循环数组来记录最近淘汰的点
-// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
-// int recentEliminatedIdx = 0; // index in the array, circulate
-//
-// int total = lineToSimplify.size();
-// if (total < 3) {
-// return results; // 不足 3 个点无法形成三角形
-// }
-//
-// int nTriangles = total - 2;
-// Triangle[] triangles = new Triangle[nTriangles];
-//
-// // 创建所有三角形并计算初始面积
-// for (int i = 1; i < total - 1; i++) {
-// int index1 = i - 1, index2 = i, index3 = i + 1;
-// double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
-// triangles[i - 1] = new Triangle(index1, index2, index3, area);
-// }
-//
-// // 设置三角形的前后关系
-// for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
-// triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
-// triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
-// }
-//
-// // 使用优先队列构建 minHeap
-// PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
-// Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
-//
-// double previousEA = -1;
-// while (!triangleHeap.isEmpty()) {
-// // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
-// // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
-// Triangle tri = triangleHeap.poll(); // O(logn)
-//
-// // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
-// // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
-// if (tri.isDeleted) {
-// if (debug) {
-// System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
-// }
-// continue;
-// }
-//
-// // 真正的淘汰点
-// // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
-// if (debug) {
-// System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
-// }
-// if (e > 0) { // otherwise e=0
-// recentEliminated[recentEliminatedIdx] = tri.indices[1];
-// recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
-// }
-// if (debug) {
-// System.out.println("the e most recently eliminated points:" + Arrays.toString(recentEliminated));
-// }
-//
-// // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
-// if (tri.area > previousEA) {
-// previousEA = tri.area;
-// }
-// results.get(tri.indices[1]).z = previousEA; // dominated significance
-// if (debug) {
-// System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
-// }
-//
-// // 更新相邻三角形
-// if (tri.prev != null) {
-// // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-//
-// // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
-// // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
-// tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-//
-// Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
-// // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
-//
-// // 2. 处理pi被淘汰引起tri.prev被更新的事情
-// // 前一个三角形连到后一个三角形
-// tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
-// tri.prev.indices[2] = tri.indices[2];
-//
-// // e parameter
-// List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
-// tri.prev.indices[0],
-// tri.prev.indices[1],
-// tri.prev.indices[2]); // TODO complexity ?
-// if (debug) {
-// System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
-// for (Point point : pointsForSig) {
-// System.out.println("\t" + point);
-// }
-// }
-// List<Point> baseLine = new ArrayList<>();
-// baseLine.add(pointsForSig.get(0));
-// baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
-// double sig = total_areal_displacement(pointsForSig, baseLine, false);
-// tri.prev.area = sig;
-//
-// if (debug) {
-// System.out.println("sig=" + sig);
-// double tmpTri = triArea(lineToSimplify.get(tri.prev.indices[0]),
-// lineToSimplify.get(tri.prev.indices[1]),
-// lineToSimplify.get(tri.prev.indices[2]));
-// System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
-// }
-//
-// // 重新加入堆
-// // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
-// // 所以必须通过add来显式重新插入修改后的元素
-// triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
-// }
-//
-// if (tri.next != null) {
-// // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-//
-// // 1. 处理旧的tri.next被标记删除的事情（角色替代）
-// // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
-// tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
-//
-// Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
-// // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
-//
-// if (tri.prev != null) {
-// tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
-// }
-//
-// // 2. 处理pi被淘汰引起tri.next被更新的事情
-// tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
-// tri.next.indices[0] = tri.indices[0];
-//
-// // e parameter
-// List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
-// tri.next.indices[0],
-// tri.next.indices[1],
-// tri.next.indices[2]); // TODO complexity
-// if (debug) {
-// System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
-// for (Point point : pointsForSig) {
-// System.out.println("\t" + point);
-// }
-// }
-// List<Point> baseLine = new ArrayList<>();
-// baseLine.add(pointsForSig.get(0));
-// baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
-// double sig = total_areal_displacement(pointsForSig, baseLine, false);
-// tri.next.area = sig;
-//
-// if (debug) {
-// System.out.println("sig=" + sig);
-// double tmpTri = triArea(lineToSimplify.get(tri.next.indices[0]),
-// lineToSimplify.get(tri.next.indices[1]),
-// lineToSimplify.get(tri.next.indices[2]));
-// System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
-// }
-//
-// // 重新加入堆
-// // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
-// // 所以必须通过add 来显式重新插入修改后的元素
-// triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
-// }
-// if (debug) {
-// System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
-// }
-// }
-// return results; // 注意这就是lineToSimplify.getVertices()
-// }
-//
-// public static void main(String[] args) {
-//
-// int eParam = 2000_0000;
-// try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
-// for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
-// // TODO 注意 要有一个点是n=300w
-//
-// Polyline polyline = new Polyline();
-// int seed = 1;
-// Random rand = new Random(seed); // TODO 注意seed在每轮里重设
-// for (int i = 0; i < n; i++) {
-// double v = rand.nextInt(1000000);
-// polyline.addVertex(new Point(i, v));
-// }
-//
-// long startTime = System.currentTimeMillis();
-// List<Point> results = buildEffectiveArea(polyline, eParam, false);
-// long endTime = System.currentTimeMillis();
-//
-// System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
-// writer.println(n + "," + eParam + "," + (endTime - startTime));
-// System.out.println(results.size());
-//
-// // clear
-// polyline.clear();
-// results.clear();
-// polyline = null;
-// results = null;
-// System.gc();
-// }
-// } catch (FileNotFoundException e) {
-// throw new RuntimeException(e);
-// }
-//
-//// int n = 300_0000;
-//// int seed = 1;
-//// Random rand = new Random(seed); // TODO 注意seed在每轮里重设
-//// Polyline polyline = new Polyline(); // 注意：如果是固定n变化e实验，这个数据要一次性生成，不然每次随机不一样
-//// for (int i = 0; i < n; i++) {
-//// double v = rand.nextInt(1000000);
-//// polyline.addVertex(new Point(i, v));
-//// }
-////
-//// try (PrintWriter writer = new PrintWriter(new File("exp_varyE_n" + n + ".csv"))) {
-//// int[] eParamList = {0, 1, 2, 10, 1000, 10000, 10_0000, 30_0000, 50_0000, 60_0000, 80_0000, 100_0000, 150_0000, 200_0000};
-//// for (int eParam : eParamList) {
-//// eParam *= 3; // TODO 注意 因为n=300w而不是100w
-////
-//// long startTime = System.currentTimeMillis();
-//// List<Point> results = buildEffectiveArea(polyline, eParam, false);
-//// long endTime = System.currentTimeMillis();
-////
-//// System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
-//// writer.println(n + "," + eParam + "," + (endTime - startTime));
-//// System.out.println(results.size());
-////
-//// // clear 注意不能把原始数据清除，还要接着用
-//// System.gc();
-//// }
-//// } catch (FileNotFoundException e) {
-//// throw new RuntimeException(e);
-//// }
-//
-// System.out.println("finish");
-// }
-//}
(iotdb) 03/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a74bbeefdea05eda798cf803e64c4cfe118f623e
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Dec 20 00:47:31 2024 +0800
add
---
.../iotdb/db/query/simpiece/Visval_standard.java | 55 +++++++++++++++++-----
1 file changed, 42 insertions(+), 13 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
index 98fdb790057..095a8e234f7 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
@@ -1,13 +1,12 @@
package org.apache.iotdb.db.query.simpiece;
-import java.io.BufferedReader;
-import java.io.FileReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
import java.util.Random;
+import java.util.stream.Collectors;
// adapted from the open source C++ code https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
class Triangle {
@@ -146,15 +145,18 @@ public class Visval_standard {
public static void main(String[] args) {
Polyline polyline = new Polyline();
-// polyline.addVertex(new ofPoint(0, 0));
-// polyline.addVertex(new ofPoint(1, 0));
-// polyline.addVertex(new ofPoint(1, 1));
-// polyline.addVertex(new ofPoint(0, 1));
-
+ List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random();
- int n = 10000;
- for (int i = 0; i < n; i++) {
- polyline.addVertex(new vPoint(i, rand.nextInt(1000)));
+ int n = 1000_000;
+ int p = 1000;
+ for (int i = 0; i < n; i += p) {
+ Polyline polylineBatch = new Polyline();
+ for (int j = i; j < Math.min(i + p, n); j++) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new vPoint(j, v));
+ polylineBatch.addVertex(new vPoint(j, v));
+ }
+ polylineList.add(polylineBatch);
}
// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842, 23089};
@@ -181,13 +183,11 @@ public class Visval_standard {
// polyline.addVertex(new vPoint(i, v.get(i)));
// }
+ System.out.println("---------------------------------");
List<vPoint> results = new ArrayList<>();
-
// 计算运行时间
long startTime = System.currentTimeMillis();
-
buildEffectiveArea(polyline, results);
-
// 输出结果
long endTime = System.currentTimeMillis();
System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
@@ -197,6 +197,35 @@ public class Visval_standard {
// vPoint point = results.get(i);
// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
// }
+
+ System.out.println("---------------------------------");
+ List<List<vPoint>> resultsBatchList = new ArrayList<>();
+ // 计算运行时间
+ int cnt = 0;
+ startTime = System.currentTimeMillis();
+ for (Polyline polylineBatch : polylineList) {
+ List<vPoint> resultsBatch = new ArrayList<>();
+ buildEffectiveArea(polylineBatch, resultsBatch);
+ cnt += resultsBatch.size();
+ resultsBatchList.add(resultsBatch);
+ }
+ // 输出结果
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println(cnt);
+
+ System.out.println("---------------------------------");
+ // 使用 Stream API 合并所有列表
+ List<vPoint> mergedList = resultsBatchList.stream().flatMap(List::stream)
+ .collect(Collectors.toList());
+ int sameCnt = 0;
+ for (int i = 0; i < mergedList.size(); i++) {
+ if (mergedList.get(i).z == results.get(i).z) {
+ sameCnt += 1;
+ }
+ }
+ System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
+
}
}
(iotdb) 02/44: add standard implementation
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b9109823f632b659bc1c3a1a1077765c1a7d1864
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Dec 19 23:40:45 2024 +0800
add standard implementation
---
.../groupby/LocalGroupByExecutorTri_Visval.java | 4 +-
.../query/simpiece/{Visval.java => VisvalOld.java} | 24 ++-
.../iotdb/db/query/simpiece/Visval_standard.java | 202 +++++++++++++++++++++
3 files changed, 227 insertions(+), 3 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java
index 8cb330812a6..40793c6651b 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/LocalGroupByExecutorTri_Visval.java
@@ -32,7 +32,7 @@ import org.apache.iotdb.db.query.control.QueryResourceManager;
import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.query.reader.series.SeriesReader;
import org.apache.iotdb.db.query.simpiece.TimeSeriesReader;
-import org.apache.iotdb.db.query.simpiece.Visval;
+import org.apache.iotdb.db.query.simpiece.VisvalOld;
import org.apache.iotdb.db.query.simpiece.VisvalPoint;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.MinMaxInfo;
@@ -135,7 +135,7 @@ public class LocalGroupByExecutorTri_Visval implements GroupByExecutor {
}
// Visval
- List<VisvalPoint> reducedPoints = Visval.reducePoints(points, m);
+ List<VisvalPoint> reducedPoints = VisvalOld.reducePoints(points, m);
for (VisvalPoint p : reducedPoints) {
series.append(p.y).append("[").append(p.x).append("]").append(",");
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalOld.java
similarity index 70%
rename from server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
rename to server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalOld.java
index 62b3019edac..9f6e80897b0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/VisvalOld.java
@@ -19,10 +19,12 @@
package org.apache.iotdb.db.query.simpiece;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Random;
import java.util.concurrent.ConcurrentSkipListMap;
-public class Visval {
+public class VisvalOld {
public static List<VisvalPoint> reducePoints(List<VisvalPoint> points, int targetCount) {
if (points.size() <= targetCount) {
@@ -39,6 +41,8 @@ public class Visval {
}
while (points.size() > targetCount) {
+ // 这里有个问题：points remove和heap不对齐，因为有可能heap里有多个点的area相同就一起被remove了，但是points里还没有remove
+ System.out.println(points.size());
VisvalPoint p = minHeap.pollFirstEntry().getValue();
if (p == null) {
return points;
@@ -67,4 +71,22 @@ public class Visval {
private static double calculateArea(VisvalPoint a, VisvalPoint b, VisvalPoint c) {
return Math.abs(a.x * (b.y - c.y) + b.x * (c.y - a.y) + c.x * (a.y - b.y)) / 2.0;
}
+
+ public static void main(String[] args) {
+ List<VisvalPoint> points = new ArrayList<>();
+ // 添加点到列表 points 100万数据
+ Random rand = new Random();
+ int n = 10000;
+ int m = 1;
+ for (int i = 0; i < n; i++) {
+ points.add(new VisvalPoint(i, rand.nextInt(1000)));
+ }
+ // 计算运行时间
+ long startTime = System.currentTimeMillis();
+ List<VisvalPoint> reducedPoints = VisvalOld.reducePoints(points, m);
+ // 输出结果
+ System.out.println(reducedPoints.size());
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
new file mode 100644
index 00000000000..98fdb790057
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
@@ -0,0 +1,202 @@
+package org.apache.iotdb.db.query.simpiece;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Random;
+
+// adapted from the open source C++ code https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
+class Triangle {
+
+ int[] indices = new int[3];
+ double area;
+ Triangle prev;
+ Triangle next;
+
+ public Triangle(int index1, int index2, int index3, double area) {
+ this.indices[0] = index1;
+ this.indices[1] = index2;
+ this.indices[2] = index3;
+ this.area = area;
+ }
+}
+
+class vPoint {
+
+ double x, y, z;
+
+ public vPoint(double x, double y) {
+ this.x = x;
+ this.y = y;
+ this.z = Double.POSITIVE_INFINITY;// effective area
+ }
+}
+
+class Polyline {
+
+ private List<vPoint> vertices = new ArrayList<>();
+
+ public void addVertex(vPoint point) {
+ vertices.add(point);
+ }
+
+ public List<vPoint> getVertices() {
+ return new ArrayList<>(vertices);
+ }
+
+ public int size() {
+ return vertices.size();
+ }
+
+ public vPoint get(int index) {
+ return vertices.get(index);
+ }
+}
+
+public class Visval_standard {
+
+ // 计算三角形面积
+ private static double triArea(vPoint d0, vPoint d1, vPoint d2) {
+ double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
+ return (double) ((dArea > 0.0) ? dArea : -dArea); // abs
+ }
+
+ public static void buildEffectiveArea(Polyline lineToSimplify, List<vPoint> results) {
+ results.clear();
+ results.addAll(lineToSimplify.getVertices());
+
+ int total = lineToSimplify.size();
+ if (total < 3) {
+ return; // 不足 3 个点无法形成三角形
+ }
+
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+ double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2),
+ lineToSimplify.get(index3));
+ triangles[i - 1] = new Triangle(index1, index2, index3, area);
+ }
+
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) {
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ }
+
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(
+ Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles);
+
+ double previousEA = -1;
+ while (!triangleHeap.isEmpty()) {
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll();
+
+ // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
+ if (tri.area > previousEA) {
+ previousEA = tri.area;
+ }
+ results.get(tri.indices[1]).z = previousEA;
+// System.out.println(tri.indices[1] + "," + previousEA);
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 前一个三角形连到后一个三角形
+ tri.prev.next = tri.next;
+ tri.prev.indices[2] = tri.indices[2];
+ tri.prev.area = triArea(
+ lineToSimplify.get(tri.prev.indices[0]),
+ lineToSimplify.get(tri.prev.indices[1]),
+ lineToSimplify.get(tri.prev.indices[2])
+ );
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过 remove 和 add 来显式重新插入修改后的元素
+ triangleHeap.remove(tri.prev);
+ triangleHeap.add(tri.prev);
+ }
+
+ if (tri.next != null) {
+ // 后一个三角形连到前一个三角形
+ tri.next.prev = tri.prev;
+ tri.next.indices[0] = tri.indices[0];
+ tri.next.area = triArea(
+ lineToSimplify.get(tri.next.indices[0]),
+ lineToSimplify.get(tri.next.indices[1]),
+ lineToSimplify.get(tri.next.indices[2])
+ );
+
+ // 重新加入堆
+ triangleHeap.remove(tri.next);
+ triangleHeap.add(tri.next);
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+ Polyline polyline = new Polyline();
+// polyline.addVertex(new ofPoint(0, 0));
+// polyline.addVertex(new ofPoint(1, 0));
+// polyline.addVertex(new ofPoint(1, 1));
+// polyline.addVertex(new ofPoint(0, 1));
+
+ Random rand = new Random();
+ int n = 10000;
+ for (int i = 0; i < n; i++) {
+ polyline.addVertex(new vPoint(i, rand.nextInt(1000)));
+ }
+
+// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842, 23089};
+// for (int i = 0; i < vlist.length; i++) {
+// polyline.addVertex(new vPoint(i, vlist[i]));
+// }
+
+// ArrayList<Double> v = new ArrayList<>();
+// String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
+// try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+// String line;
+// int count = 0;
+// while ((line = br.readLine()) != null && count < 1000) {
+// String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
+// v.add(Double.parseDouble(columns[0])); // 读取第一列数据
+// count++;
+// }
+// } catch (Exception e) {
+// e.printStackTrace();
+// }
+// // 打印数据长度
+// System.out.println("Data length: " + v.size());
+// for (int i = 0; i < v.size(); i++) {
+// polyline.addVertex(new vPoint(i, v.get(i)));
+// }
+
+ List<vPoint> results = new ArrayList<>();
+
+ // 计算运行时间
+ long startTime = System.currentTimeMillis();
+
+ buildEffectiveArea(polyline, results);
+
+ // 输出结果
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println(results.size());
+
+// for (int i = 0; i < results.size(); i++) {
+// vPoint point = results.get(i);
+// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+// }
+ }
+}
+
(iotdb) 23/44: return bottom-up order
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5ee9e320f4c473516ff3cea64a2d07824f2af757
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Jan 23 18:34:02 2025 +0800
return bottom-up order
---
.../java/org/apache/iotdb/db/query/eBUG/Test1.java | 10 ++++-
.../java/org/apache/iotdb/db/query/eBUG/Test2.java | 51 +++++++++++++---------
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 25 ++++++++---
3 files changed, 57 insertions(+), 29 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
index f53f5719217..91df7751e50 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
@@ -3,6 +3,7 @@ package org.apache.iotdb.db.query.eBUG;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import java.util.Random;
@@ -13,8 +14,9 @@ public class Test1 {
public static void main(String[] args) {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random(3);
+ Random rand = new Random(10);
int n = 10000;
+ int eParam = 0;
int p = 10;
for (int i = 0; i < n; i += p) {
@@ -44,14 +46,18 @@ public class Test1 {
}
System.out.println("---------------------------------");
- int eParam = 1;
long startTime = System.currentTimeMillis();
+ // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
long endTime = System.currentTimeMillis();
System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
System.out.println(results.size());
+ // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
+ // 按照时间戳递增排序整理:
+ results.sort(Comparator.comparingDouble(point -> point.x));
+
if (results.size() <= 100) {
System.out.println("+++++++++++++++++++");
for (int i = 0; i < results.size(); i++) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java
index 589603525e5..fb7de582f17 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test2.java
@@ -2,6 +2,7 @@ package org.apache.iotdb.db.query.eBUG;
import io.jsonwebtoken.lang.Assert;
+import java.util.Comparator;
import java.util.List;
import java.util.Random;
@@ -10,32 +11,42 @@ import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
public class Test2 {
// 用于测试在线采样
public static void main(String[] args) {
- int n = 100000;
+ int n = 20;
int eParam = 1;
-// int m = 100; // >2代表在线采样，<=2代表预计算
- for (int m = 9900; m <= n; m++) {
+ int m = 2; // >2代表在线采样，<=2代表预计算
+// for (int m = 9900; m <= n; m++) {
- Polyline polyline = new Polyline();
- Random rand = new Random(2);
- for (int i = 0; i < n; i++) {
- double v = rand.nextInt(1000000);
- polyline.addVertex(new Point(i, v));
- }
+ Polyline polyline = new Polyline();
+ Random rand = new Random(2);
+ for (int i = 0; i < n; i++) {
+ double v = rand.nextInt(1000000);
+ polyline.addVertex(new Point(i, v));
+ }
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false, m);
- long endTime = System.currentTimeMillis();
- System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false, m);
+ long endTime = System.currentTimeMillis();
+ System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(results.size());
+ System.out.println(results.size());
+ if (m > 2 && m < n) {
Assert.isTrue(results.size() == m);
+ } else {
+ Assert.isTrue(results.size() == n);
+ }
+ if (results.size() <= 100) {
+ System.out.println("+++++++++++++++++++");
+ for (Point point : results) {
+ System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ }
+ // 注意现在返回的结果是按照Sig递增也就是bottom-up淘汰的顺序排列的，而不是按照时间戳递增排列
+ // 按照时间戳递增排序整理:
+ results.sort(Comparator.comparingDouble(point -> point.x));
+ System.out.println("+++++++++++++++++++");
+ for (Point point : results) {
+ System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ }
}
-// if (results.size() <= 100) {
-// System.out.println("+++++++++++++++++++");
-// for (Point point : results) {
-// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
-// }
-// }
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index b9147e4c574..e177b1dcf20 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -66,12 +66,17 @@ public class eBUG {
}
if (m > 2) {
- System.out.println("online sampling mode, returning " + m + " sampled points");
+ System.out.println("online sampling mode, " +
+ "returning " + m + " sampled points sorted by time in ascending order");
} else {
- System.out.println("offline precomputation mode, returning each point with dominated significance");
+ System.out.println("offline precomputation mode, " +
+ "returning each point sorted by dominated significance (DS) in ascending order");
}
- List<Point> results = lineToSimplify.getVertices(); // 浅复制
+// List<Point> results = lineToSimplify.getVertices(); // 浅复制
+ // TODO 预计算结果改成按照bottom-up逐点淘汰顺序（DS递增）排列而不是按照时间戳，这样省去在线时对DS排序的过程
+ // add的是Point引用，所以没有多用一倍的空间
+ List<Point> resultsBottomUpEliminated = new ArrayList<>();
// 存储的是点的引用，这样可以修改原来序列里点的淘汰状态
// 存储的是距离当前最新状态的滞后的尚未施加、待施加的e个淘汰点
@@ -79,7 +84,7 @@ public class eBUG {
int total = lineToSimplify.size();
if (total < 3) {
- return results; // 不足 3 个点无法形成三角形
+ return lineToSimplify.getVertices(); // 不足 3 个点无法形成三角形
}
int nTriangles = total - 2;
@@ -155,7 +160,9 @@ public class eBUG {
if (tri.area > previousEA) {
previousEA = tri.area;
}
- results.get(tri.indices[1]).z = previousEA; // dominated significance
+// results.get(tri.indices[1]).z = previousEA; // dominated significance
+ lineToSimplify.get(tri.indices[1]).z = previousEA; // dominated significance
+ resultsBottomUpEliminated.add(lineToSimplify.get(tri.indices[1])); // TODO add的是Point引用，所以没有多用一倍的空间
if (debug) {
System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
}
@@ -270,13 +277,17 @@ public class eBUG {
Point start = lineToSimplify.get(0);
Point end = lineToSimplify.get(lineToSimplify.size() - 1);
while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- onlineSampled.add(start);
+ onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
start = start.next; // when e=0, only traversing three points pa pi pb
}
onlineSampled.add(end);
return onlineSampled;
} else { // offline precomputation mode, for precomputing the dominated significance of each point
- return results; // 注意这就是lineToSimplify.getVertices()
+// return results; // 注意这就是lineToSimplify.getVertices()
+ // TODO
+ resultsBottomUpEliminated.add(lineToSimplify.get(0)); // 全局首点
+ resultsBottomUpEliminated.add(lineToSimplify.get(lineToSimplify.size() - 1)); // 全局尾点
+ return resultsBottomUpEliminated;
}
}
(iotdb) 01/44: disable license check
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 584658ee74bf1fab635da20734635379129a0b2e
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Dec 19 16:51:35 2024 +0800
disable license check
---
pom.xml | 146 ++++++++++++++++++++++++++++++++--------------------------------
1 file changed, 73 insertions(+), 73 deletions(-)
diff --git a/pom.xml b/pom.xml
index 241fc71fcfd..d3b84e3199c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -613,66 +613,66 @@
<artifactId>sonar-maven-plugin</artifactId>
<version>3.9.0.2155</version>
</plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <consoleOutput>true</consoleOutput>
- <excludes>
- <!-- Git related files -->
- <exclude>**/.git/**</exclude>
- <exclude>**/.mvn/**</exclude>
- <exclude>**/.gitignore</exclude>
- <exclude>**/.gitmodules</exclude>
- <exclude>**/.git-blame-ignore-revs</exclude>
- <!-- Maven related files -->
- <exclude>**/target/**</exclude>
- <!-- Eclipse related files -->
- <exclude>**/.project</exclude>
- <exclude>**/.settings/**</exclude>
- <exclude>**/.classpath</exclude>
- <!-- IntelliJ related files -->
- <exclude>**/.idea/**</exclude>
- <exclude>**/*.iml</exclude>
- <!-- Runtime log -->
- <exclude>**/*.log</exclude>
- <!-- Exclude CVS files -->
- <exclude>**/*.cvs</exclude>
- <!-- licenses -->
- <exclude>licenses/*</exclude>
- <!-- only for Travis CI with WinOS-->
- <exclude>hadoopbin</exclude>
- <exclude>windowssystem32</exclude>
- <!-- generated by Github -->
- <exclude>**/.github/**</exclude>
- <!-- figures -->
- <exclude>**/.eps</exclude>
- <exclude>**/.png</exclude>
- <exclude>**/.jpg</exclude>
- <exclude>**/.jpeg</exclude>
- <!--Generated by Apache Release -->
- <exclude>local-snapshots-dir/**</exclude>
- <!-- JSON can't contain comments and therefore no Apache header -->
- <exclude>*.json</exclude>
- <!-- visualization plans -->
- <exclude>**/*.plan</exclude>
- <exclude>**/NOTICE-binary</exclude>
- <exclude>**/LICENSE-binary</exclude>
- <!-- json does not support comments-->
- <exclude>**/*.json</exclude>
- <!-- the zeppelin export file format-->
- <exclude>**/*.zpln</exclude>
- <!-- exclude go.mod and go.sum in iotdb-client-go submodule-->
- <exclude>**/go.mod</exclude>
- <exclude>**/go.sum</exclude>
- <!-- python -->
- <exclude>.pytest_cache/**</exclude>
- <exclude>venv/**</exclude>
- <exclude>apache_iotdb.egg-info/**</exclude>
- <exclude>client-py/iotdb/thrift/**</exclude>
- </excludes>
- </configuration>
- </plugin>
+ <!-- <plugin>-->
+ <!-- <groupId>org.apache.rat</groupId>-->
+ <!-- <artifactId>apache-rat-plugin</artifactId>-->
+ <!-- <configuration>-->
+ <!-- <consoleOutput>true</consoleOutput>-->
+ <!-- <excludes>-->
+ <!-- <!– Git related files –>-->
+ <!-- <exclude>**/.git/**</exclude>-->
+ <!-- <exclude>**/.mvn/**</exclude>-->
+ <!-- <exclude>**/.gitignore</exclude>-->
+ <!-- <exclude>**/.gitmodules</exclude>-->
+ <!-- <exclude>**/.git-blame-ignore-revs</exclude>-->
+ <!-- <!– Maven related files –>-->
+ <!-- <exclude>**/target/**</exclude>-->
+ <!-- <!– Eclipse related files –>-->
+ <!-- <exclude>**/.project</exclude>-->
+ <!-- <exclude>**/.settings/**</exclude>-->
+ <!-- <exclude>**/.classpath</exclude>-->
+ <!-- <!– IntelliJ related files –>-->
+ <!-- <exclude>**/.idea/**</exclude>-->
+ <!-- <exclude>**/*.iml</exclude>-->
+ <!-- <!– Runtime log –>-->
+ <!-- <exclude>**/*.log</exclude>-->
+ <!-- <!– Exclude CVS files –>-->
+ <!-- <exclude>**/*.cvs</exclude>-->
+ <!-- <!– licenses –>-->
+ <!-- <exclude>licenses/*</exclude>-->
+ <!-- <!– only for Travis CI with WinOS–>-->
+ <!-- <exclude>hadoopbin</exclude>-->
+ <!-- <exclude>windowssystem32</exclude>-->
+ <!-- <!– generated by Github –>-->
+ <!-- <exclude>**/.github/**</exclude>-->
+ <!-- <!– figures –>-->
+ <!-- <exclude>**/.eps</exclude>-->
+ <!-- <exclude>**/.png</exclude>-->
+ <!-- <exclude>**/.jpg</exclude>-->
+ <!-- <exclude>**/.jpeg</exclude>-->
+ <!-- <!–Generated by Apache Release –>-->
+ <!-- <exclude>local-snapshots-dir/**</exclude>-->
+ <!-- <!– JSON can't contain comments and therefore no Apache header –>-->
+ <!-- <exclude>*.json</exclude>-->
+ <!-- <!– visualization plans –>-->
+ <!-- <exclude>**/*.plan</exclude>-->
+ <!-- <exclude>**/NOTICE-binary</exclude>-->
+ <!-- <exclude>**/LICENSE-binary</exclude>-->
+ <!-- <!– json does not support comments–>-->
+ <!-- <exclude>**/*.json</exclude>-->
+ <!-- <!– the zeppelin export file format–>-->
+ <!-- <exclude>**/*.zpln</exclude>-->
+ <!-- <!– exclude go.mod and go.sum in iotdb-client-go submodule–>-->
+ <!-- <exclude>**/go.mod</exclude>-->
+ <!-- <exclude>**/go.sum</exclude>-->
+ <!-- <!– python –>-->
+ <!-- <exclude>.pytest_cache/**</exclude>-->
+ <!-- <exclude>venv/**</exclude>-->
+ <!-- <exclude>apache_iotdb.egg-info/**</exclude>-->
+ <!-- <exclude>client-py/iotdb/thrift/**</exclude>-->
+ <!-- </excludes>-->
+ <!-- </configuration>-->
+ <!-- </plugin>-->
<plugin>
<groupId>com.diffplug.spotless</groupId>
<artifactId>spotless-maven-plugin</artifactId>
@@ -885,19 +885,19 @@
Check if all files contain Apache headers in them.
Ignore this plugin, we use license-maven-plugin to check apache header.
-->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <executions>
- <execution>
- <id>license-check</id>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
+ <!-- <plugin>-->
+ <!-- <groupId>org.apache.rat</groupId>-->
+ <!-- <artifactId>apache-rat-plugin</artifactId>-->
+ <!-- <executions>-->
+ <!-- <execution>-->
+ <!-- <id>license-check</id>-->
+ <!-- <phase>verify</phase>-->
+ <!-- <goals>-->
+ <!-- <goal>check</goal>-->
+ <!-- </goals>-->
+ <!-- </execution>-->
+ <!-- </executions>-->
+ <!-- </plugin>-->
<plugin>
<groupId>au.com.acegi</groupId>
<artifactId>xml-format-maven-plugin</artifactId>
(iotdb) 27/44: bottomUpL2;FSW
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dcb711d269a42d6f207fa3b1152b9d65dd1c850f
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jan 27 15:09:45 2025 +0800
bottomUpL2;FSW
---
server/pom.xml | 8 +-
server/sample_BUL2-jar-with-dependencies.jar | Bin 0 -> 38716084 bytes
server/sample_eBUG-jar-with-dependencies.jar | Bin 0 -> 38715994 bytes
server/sample_fsw-jar-with-dependencies.jar | Bin 0 -> 38715995 bytes
server/server-changelist.md | 53 -
.../java/org/apache/iotdb/db/query/eBUG/FSW.java | 150 +++
.../java/org/apache/iotdb/db/query/eBUG/Point.java | 70 +-
.../org/apache/iotdb/db/query/eBUG/Polyline.java | 60 +-
.../java/org/apache/iotdb/db/query/eBUG/SWAB.java | 1203 +++++++++++++-------
.../java/org/apache/iotdb/db/query/eBUG/Test3.java | 2 +-
.../java/org/apache/iotdb/db/query/eBUG/Test4.java | 118 +-
.../java/org/apache/iotdb/db/query/eBUG/Test5.java | 95 +-
.../java/org/apache/iotdb/db/query/eBUG/Test6.java | 140 +--
.../java/org/apache/iotdb/db/query/eBUG/Test7.java | 99 +-
.../java/org/apache/iotdb/db/query/eBUG/Test8.java | 97 +-
.../java/org/apache/iotdb/db/query/eBUG/Test9.java | 64 ++
.../java/org/apache/iotdb/db/query/eBUG/Tool.java | 567 +++++----
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 715 ++++++------
.../org/apache/iotdb/db/query/eBUG/eBUG_Build.java | 139 ---
.../org/apache/iotdb/db/query/eBUG/sample_FSW.java | 82 ++
.../apache/iotdb/db/query/eBUG/sample_SWAB.java | 83 ++
.../iotdb/db/query/eBUG/sample_bottomUpL2.java | 74 ++
.../apache/iotdb/db/query/eBUG/sample_eBUG.java | 100 ++
23 files changed, 2412 insertions(+), 1507 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index 51f9a56c69a..66ccebdb43a 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -281,10 +281,14 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
- <finalName>eBUG_Build</finalName>
+ <finalName>sample_fsw</finalName>
+ <!-- <finalName>sample_eBUG</finalName>-->
+ <!-- <finalName>sample_BUL2</finalName>-->
<archive>
<manifest>
- <mainClass>org.apache.iotdb.db.query.eBUG.eBUG_Build</mainClass>
+ <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>-->
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpL2</mainClass>-->
</manifest>
</archive>
<descriptorRefs>
diff --git a/server/sample_BUL2-jar-with-dependencies.jar b/server/sample_BUL2-jar-with-dependencies.jar
new file mode 100644
index 00000000000..d592875251c
Binary files /dev/null and b/server/sample_BUL2-jar-with-dependencies.jar differ
diff --git a/server/sample_eBUG-jar-with-dependencies.jar b/server/sample_eBUG-jar-with-dependencies.jar
new file mode 100644
index 00000000000..a37acfd0590
Binary files /dev/null and b/server/sample_eBUG-jar-with-dependencies.jar differ
diff --git a/server/sample_fsw-jar-with-dependencies.jar b/server/sample_fsw-jar-with-dependencies.jar
new file mode 100644
index 00000000000..c1981d2a984
Binary files /dev/null and b/server/sample_fsw-jar-with-dependencies.jar differ
diff --git a/server/server-changelist.md b/server/server-changelist.md
deleted file mode 100644
index c186e231b34..00000000000
--- a/server/server-changelist.md
+++ /dev/null
@@ -1,53 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-# 0.10.0 -> 0.11.0
-
-## Upgrade configuration files
-
-# 0.9.0 -> 0.10.0
-
-## Upgrade TsFile from 000001 to 000002
-
-## upgrade mlog, append ,,, behind each create timeseries log
-
-## Add system.properties
-
-## Upgrade configuration files
-
-# 0.8.0 -> 0.9.0
-
-## User profile
-Last Updated on November 14th, 2019 by EJTTianYu.
-
-| PR# | Name | Author | Changes |
-| ---- | ------------------------------------------------------------ | --------------- | ------------------------------------------------------------ |
-| 299 | Turn watermark-demo into watermark feature | LeiRui | Add watermark in the end of the profile. |
-
-## Upgrading Log
-Last Updated on November 14th, 2019 by EJTTianYu.
-
-| PR# | Name | Author | Changes |
-| ---- | ------------------------------------------------------------ | --------------- | ------------------------------------------------------------ |
-| 467 | [IOTDB-294]online upgrade from v0.8.0 to current version | EJTTianYu | Add upgrade/upgrade.txt in the system dir to record the upgrading process. |
-
-
-
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/FSW.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/FSW.java
new file mode 100644
index 00000000000..90ad2e25353
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/FSW.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.Tool.getParam;
+
+// Liu, Xiaoyan, Lin, Zhenjiang, Wang, Huaiqing, 2008. Novel online methods for time series
+// segmentation.
+// FSW: feasible space window
+// adapted from https://github.com/zhourongleiden/AEPLA.git
+public class FSW {
+
+ public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
+ int length = points.size();
+
+ // Precompute upper and lower bounds
+ double[] p_upper = new double[length];
+ double[] p_lower = new double[length];
+ for (int j = 0; j < length; j++) {
+ double v = points.get(j).getValue();
+ p_upper[j] = v + epsilon;
+ p_lower[j] = v - epsilon;
+ }
+
+ // init the first segment
+ int i = 0;
+ int seg_no = 0;
+ int csp_id = 0;
+ List<int[]> segmentPoint = new ArrayList<>();
+ segmentPoint.add(new int[] {0, 0});
+ double upSlope = Double.POSITIVE_INFINITY;
+ double lowSlope = Double.NEGATIVE_INFINITY;
+
+ while (i < length - 1) {
+ // IOMonitor2.DCP_D_getAllSatisfiedPageData_traversedPointNum++;
+ i++;
+ upSlope =
+ Math.min(
+ upSlope,
+ ((p_upper[i] - points.get(segmentPoint.get(seg_no)[1]).getValue())
+ / (points.get(i).getTimestamp()
+ - points.get(segmentPoint.get(seg_no)[1]).getTimestamp())));
+ lowSlope =
+ Math.max(
+ lowSlope,
+ ((p_lower[i] - points.get(segmentPoint.get(seg_no)[1]).getValue())
+ / (points.get(i).getTimestamp()
+ - points.get(segmentPoint.get(seg_no)[1]).getTimestamp())));
+ if (upSlope < lowSlope) {
+ seg_no += 1;
+ segmentPoint.add(new int[] {seg_no, csp_id});
+ i = csp_id;
+ upSlope = Double.POSITIVE_INFINITY;
+ lowSlope = Double.NEGATIVE_INFINITY;
+ } else {
+ double s =
+ (points.get(i).getValue() - points.get(segmentPoint.get(seg_no)[1]).getValue())
+ / (points.get(i).getTimestamp()
+ - points.get(segmentPoint.get(seg_no)[1]).getTimestamp());
+ if (s >= lowSlope && s <= upSlope) {
+ csp_id = i;
+ }
+ }
+ }
+
+ // deal with the last segment
+ if (segmentPoint.get(seg_no)[1] < length - 1) {
+ // IOMonitor2.DCP_D_getAllSatisfiedPageData_traversedPointNum++;
+ seg_no += 1;
+ segmentPoint.add(new int[] {seg_no, length - 1});
+ }
+
+ List<Point> result = new ArrayList<>();
+ for (int[] ints : segmentPoint) {
+ result.add(points.get(ints[1]));
+ }
+
+ return result;
+ }
+
+ public static void main(String[] args) throws IOException {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 100000_0;
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ // Polyline polyline = new Polyline();
+ // for (int i = 0; i < N; i += 1) {
+ // double v = rand.nextInt(1000);
+ // polyline.addVertex(new Point(i, v));
+ // }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < points.size(); i++) {
+ Point point = points.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ int m = 119;
+
+ long startTime = System.currentTimeMillis();
+ double epsilon = getParam(points, m, FSW::reducePoints, m * 0.01);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ List<Point> sampled = reducePoints(points, epsilon);
+ System.out.println(sampled.size());
+ System.out.println(epsilon);
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
index 3d47e88f422..712d1b94e01 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
@@ -2,35 +2,43 @@ package org.apache.iotdb.db.query.eBUG;
public class Point {
- double x, y, z;
-
- public Point prev; // 指向T'_max{0,k-e}里当前点的前一个点 for eBUG usage
- public Point next; // 指向T'_max{0,k-e}里当前点的后一个点 for eBUG usage
- // 注意Triangle里的prev&next用于最新状态下存在三角形之间的连接关系，
- // 而这里Point的prev&next用于滞后e个淘汰点（也就是最近的e个淘汰点先不施加）状态下的未淘汰点之间的连接关系
-
- public int index; // for swab usage
-
- public Point(double x, double y) {
- this.x = x;
- this.y = y;
- this.z = Double.POSITIVE_INFINITY; // effective area
-
- // for eBUG usage
- // this.eliminated = false;
- this.prev = null;
- this.next = null;
- }
-
- public void markEliminated() {
- // to avoid traversing each point between pa to pb,
- // instead only traversing at most e most recently eliminated points lagged
- prev.next = next;
- next.prev = prev;
- }
-
- @Override
- public String toString() {
- return "Point: (" + x + ", " + y + ", " + z + ")";
- }
+ double x, y, z;
+
+ public Point prev; // 指向T'_max{0,k-e}里当前点的前一个点 for eBUG usage
+ public Point next; // 指向T'_max{0,k-e}里当前点的后一个点 for eBUG usage
+ // 注意Triangle里的prev&next用于最新状态下存在三角形之间的连接关系，
+ // 而这里Point的prev&next用于滞后e个淘汰点（也就是最近的e个淘汰点先不施加）状态下的未淘汰点之间的连接关系
+
+ public int index; // for swab usage
+
+ public Point(double x, double y) {
+ this.x = x;
+ this.y = y;
+ this.z = Double.POSITIVE_INFINITY; // effective area
+
+ // for eBUG usage
+ // this.eliminated = false;
+ this.prev = null;
+ this.next = null;
+ }
+
+ public void markEliminated() {
+ // to avoid traversing each point between pa to pb,
+ // instead only traversing at most e most recently eliminated points lagged
+ prev.next = next;
+ next.prev = prev;
+ }
+
+ @Override
+ public String toString() {
+ return "Point: (" + x + ", " + y + ", " + z + ")";
+ }
+
+ public double getTimestamp() { // used by FSW
+ return x;
+ }
+
+ public double getValue() { // used by FSW
+ return y;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
index 5b1f5354871..a97eba75d08 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
@@ -5,34 +5,34 @@ import java.util.List;
public class Polyline {
- private List<Point> vertices = new ArrayList<>();
-
- public void addVertex(Point point) {
- // if (!vertices.isEmpty()) { //before adding this point
- // vertices.get(vertices.size() - 1).next = point;
- // point.prev = vertices.get(vertices.size() - 1);
- // }
- vertices.add(point);
- }
-
- public List<Point> getVertices() {
- // return new ArrayList<>(vertices);
- return vertices;
- }
-
- public void setVertices(List<Point> points) {
- this.vertices = points;
- }
-
- public int size() {
- return vertices.size();
- }
-
- public Point get(int index) {
- return vertices.get(index);
- }
-
- public void clear() {
- vertices.clear();
- }
+ private List<Point> vertices = new ArrayList<>();
+
+ public void addVertex(Point point) {
+ // if (!vertices.isEmpty()) { //before adding this point
+ // vertices.get(vertices.size() - 1).next = point;
+ // point.prev = vertices.get(vertices.size() - 1);
+ // }
+ vertices.add(point);
+ }
+
+ public List<Point> getVertices() {
+ // return new ArrayList<>(vertices);
+ return vertices;
+ }
+
+ public void setVertices(List<Point> points) {
+ this.vertices = points;
+ }
+
+ public int size() {
+ return vertices.size();
+ }
+
+ public Point get(int index) {
+ return vertices.get(index);
+ }
+
+ public void clear() {
+ vertices.clear();
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
index 31b2f02ea86..f6d5e2fc6a3 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
@@ -1,481 +1,834 @@
package org.apache.iotdb.db.query.eBUG;
+import java.io.*;
import java.util.*;
-
public class SWAB {
- public static Object[] prefixSum(List<Point> points) {
- int n = points.size();
-
- // 预计算prefix sum从而可以加速L2误差计算
- double[] prefixSumX = new double[n];
- double[] prefixSumY = new double[n];
- double[] prefixSumX2 = new double[n];
- double[] prefixSumY2 = new double[n];
- double[] prefixSumXY = new double[n];
-
- Point pend = points.get(points.size() - 1); // 最后一个点，按照时间戳递增排序所以应该baseX是最大时间戳，从而归一化
- double baseX = pend.x; // 归一化避免数值溢出，主要就是这个时间戳太大
- double baseY = pend.y; // 归一化避免数值溢出
- if (baseX == 0) {
- baseX = 1;
- }
- if (baseY == 0) {
- baseY = 1;
- }
+ public static Object[] prefixSum(List<Point> points) {
+ int n = points.size();
+
+ // 预计算prefix sum从而可以加速L2误差计算
+ double[] prefixSumX = new double[n];
+ double[] prefixSumY = new double[n];
+ double[] prefixSumX2 = new double[n];
+ double[] prefixSumY2 = new double[n];
+ double[] prefixSumXY = new double[n];
+
+ Point pend = points.get(points.size() - 1); // 最后一个点，按照时间戳递增排序所以应该baseX是最大时间戳，从而归一化
+ double baseX = pend.x; // 归一化避免数值溢出，主要就是这个时间戳太大
+ double baseY = pend.y; // 归一化避免数值溢出
+ if (baseX == 0) {
+ baseX = 1;
+ }
+ if (baseY == 0) {
+ baseY = 1;
+ }
- Point p0 = points.get(0);
- prefixSumX[0] = p0.x / baseX;
- prefixSumY[0] = p0.y / baseY;
- prefixSumX2[0] = (p0.x * p0.x) / (baseX * baseX);
- prefixSumY2[0] = (p0.y * p0.y) / (baseY * baseY);
- prefixSumXY[0] = (p0.x * p0.y) / (baseX * baseY);
-
- for (int i = 1; i < n; i++) {
- Point p = points.get(i);
- prefixSumX[i] = prefixSumX[i - 1] + (p.x / baseX);
- prefixSumY[i] = prefixSumY[i - 1] + (p.y / baseY);
- prefixSumX2[i] = prefixSumX2[i - 1] + (p.x * p.x) / (baseX * baseX);
- prefixSumY2[i] = prefixSumY2[i - 1] + (p.y * p.y) / (baseY * baseY);
- prefixSumXY[i] = prefixSumXY[i - 1] + (p.x * p.y) / (baseX * baseY);
- }
+ Point p0 = points.get(0);
+ prefixSumX[0] = p0.x / baseX;
+ prefixSumY[0] = p0.y / baseY;
+ prefixSumX2[0] = (p0.x * p0.x) / (baseX * baseX);
+ prefixSumY2[0] = (p0.y * p0.y) / (baseY * baseY);
+ prefixSumXY[0] = (p0.x * p0.y) / (baseX * baseY);
+
+ for (int i = 1; i < n; i++) {
+ Point p = points.get(i);
+ prefixSumX[i] = prefixSumX[i - 1] + (p.x / baseX);
+ prefixSumY[i] = prefixSumY[i - 1] + (p.y / baseY);
+ prefixSumX2[i] = prefixSumX2[i - 1] + (p.x * p.x) / (baseX * baseX);
+ prefixSumY2[i] = prefixSumY2[i - 1] + (p.y * p.y) / (baseY * baseY);
+ prefixSumXY[i] = prefixSumXY[i - 1] + (p.x * p.y) / (baseX * baseY);
+ }
- return new Object[]{prefixSumX, prefixSumY, prefixSumX2, prefixSumY2, prefixSumXY, baseX, baseY};
+ return new Object[] {
+ prefixSumX, prefixSumY, prefixSumX2, prefixSumY2, prefixSumXY, baseX, baseY
+ };
+ }
+
+ public static double myLA_withTimestamps(List<Point> points, Object[] prefixSum, int lx, int rx) {
+ // 默认joint=true, residual=true，即使用linear interpolation近似分段，且使用L2误差
+ // lx~rx 左闭右闭
+ // 使用prefix sum加速，但是注意防止溢出处理
+ double x1 = points.get(lx).x;
+ double y1 = points.get(lx).y;
+ double x2 = points.get(rx).x;
+ double y2 = points.get(rx).y;
+
+ double k = (y2 - y1) / (x2 - x1);
+ double b = (y1 * x2 - y2 * x1) / (x2 - x1);
+
+ // 计算L2误差，使用prefix sum加速计算
+ double[] prefixSumX = (double[]) prefixSum[0];
+ double[] prefixSumY = (double[]) prefixSum[1];
+ double[] prefixSumX2 = (double[]) prefixSum[2];
+ double[] prefixSumY2 = (double[]) prefixSum[3];
+ double[] prefixSumXY = (double[]) prefixSum[4];
+ double baseX = (double) prefixSum[5];
+ double baseY = (double) prefixSum[6];
+
+ double sumX, sumY, sumX2, sumY2, sumXY;
+
+ if (lx > 0) {
+ sumX = prefixSumX[rx] - prefixSumX[lx - 1];
+ sumY = prefixSumY[rx] - prefixSumY[lx - 1];
+ sumX2 = prefixSumX2[rx] - prefixSumX2[lx - 1];
+ sumY2 = prefixSumY2[rx] - prefixSumY2[lx - 1];
+ sumXY = prefixSumXY[rx] - prefixSumXY[lx - 1];
+ } else { // lx=0
+ sumX = prefixSumX[rx];
+ sumY = prefixSumY[rx];
+ sumX2 = prefixSumX2[rx];
+ sumY2 = prefixSumY2[rx];
+ sumXY = prefixSumXY[rx];
}
- public static double myLA_withTimestamps(List<Point> points, Object[] prefixSum, int lx, int rx) {
- // 默认joint=true, residual=true，即使用linear interpolation近似分段，且使用L2误差
- // lx~rx 左闭右闭
- // 使用prefix sum加速，但是注意防止溢出处理
- double x1 = points.get(lx).x;
- double y1 = points.get(lx).y;
- double x2 = points.get(rx).x;
- double y2 = points.get(rx).y;
-
- double k = (y2 - y1) / (x2 - x1);
- double b = (y1 * x2 - y2 * x1) / (x2 - x1);
-
- // 计算L2误差，使用prefix sum加速计算
- double[] prefixSumX = (double[]) prefixSum[0];
- double[] prefixSumY = (double[]) prefixSum[1];
- double[] prefixSumX2 = (double[]) prefixSum[2];
- double[] prefixSumY2 = (double[]) prefixSum[3];
- double[] prefixSumXY = (double[]) prefixSum[4];
- double baseX = (double) prefixSum[5];
- double baseY = (double) prefixSum[6];
-
- double sumX, sumY, sumX2, sumY2, sumXY;
-
- if (lx > 0) {
- sumX = prefixSumX[rx] - prefixSumX[lx - 1];
- sumY = prefixSumY[rx] - prefixSumY[lx - 1];
- sumX2 = prefixSumX2[rx] - prefixSumX2[lx - 1];
- sumY2 = prefixSumY2[rx] - prefixSumY2[lx - 1];
- sumXY = prefixSumXY[rx] - prefixSumXY[lx - 1];
- } else { // lx=0
- sumX = prefixSumX[rx];
- sumY = prefixSumY[rx];
- sumX2 = prefixSumX2[rx];
- sumY2 = prefixSumY2[rx];
- sumXY = prefixSumXY[rx];
- }
+ // 计算L2误差，注意归一化还原
+ return sumY2 * baseY * baseY
+ + k * k * sumX2 * baseX * baseX
+ + 2 * k * b * sumX * baseX
+ - 2 * b * sumY * baseY
+ - 2 * k * sumXY * baseX * baseY
+ + b * b * (rx - lx + 1);
+ }
+
+ public static double myLA_withTimestamps(List<Point> points, int lx, int rx) {
+ // 默认joint=true, residual=true，即使用linear interpolation近似分段，且使用L2误差
+ // lx~rx 左闭右闭
+ // 没有使用prefix sum加速
+ double x1 = points.get(lx).x;
+ double y1 = points.get(lx).y;
+ double x2 = points.get(rx).x;
+ double y2 = points.get(rx).y;
+
+ double k = (y2 - y1) / (x2 - x1);
+ double b = (y1 * x2 - y2 * x1) / (x2 - x1);
+
+ double tmp = 0;
+ for (int i = lx; i <= rx; i++) {
+ double e =
+ (k * points.get(i).x + b - points.get(i).y) * (k * points.get(i).x + b - points.get(i).y);
+ tmp += e;
+ }
+ return tmp;
+ }
+
+ public static List<Point> seg_bottomUp_maxerror_withTimestamps(
+ List<Point> points, double maxError, Object[] prefixSum, boolean debug) {
+ int total = points.size();
+ if (total < 3) {
+ return points; // 不足 3 个点无法形成三角形
+ }
- // 计算L2误差，注意归一化还原
- return sumY2 * baseY * baseY + k * k * sumX2 * baseX * baseX + 2 * k * b * sumX * baseX - 2 * b * sumY * baseY - 2 * k * sumXY * baseX * baseY + b * b * (rx - lx + 1);
+ // 每个triangle对应两个相邻的待合并的分段
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ points.get(0).prev = null;
+ points.get(0).next = points.get(1);
+ // points.get(0).index = 0;
+ points.get(total - 1).next = null;
+ points.get(total - 1).prev = points.get(total - 2);
+ // points.get(total - 1).index = points.size() - 1;
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+ double mc;
+ if (prefixSum == null) {
+ mc = myLA_withTimestamps(points, index1, index3);
+ } else {
+ mc = myLA_withTimestamps(points, prefixSum, index1, index3);
+ }
+ triangles[i - 1] = new Triangle(index1, index2, index3, mc);
+
+ // 初始化点的状态 用于最后在线采样结果顺序输出未淘汰点
+ points.get(i).prev = points.get(i - 1);
+ points.get(i).next = points.get(i + 1);
+ // points.get(i).index = i; // for swab usage
}
- public static double myLA_withTimestamps(List<Point> points, int lx, int rx) {
- // 默认joint=true, residual=true，即使用linear interpolation近似分段，且使用L2误差
- // lx~rx 左闭右闭
- // 没有使用prefix sum加速
- double x1 = points.get(lx).x;
- double y1 = points.get(lx).y;
- double x2 = points.get(rx).x;
- double y2 = points.get(rx).y;
-
- double k = (y2 - y1) / (x2 - x1);
- double b = (y1 * x2 - y2 * x1) / (x2 - x1);
-
- double tmp = 0;
- for (int i = lx; i <= rx; i++) {
- double e = (k * points.get(i).x + b - points.get(i).y) * (k * points.get(i).x + b - points.get(i).y);
- tmp += e;
- }
- return tmp;
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
}
- public static List<Point> seg_bottomUp_maxerror_withTimestamps(List<Point> points, double maxError, Object[] prefixSum, boolean debug) {
- int total = points.size();
- if (total < 3) {
- return points; // 不足 3 个点无法形成三角形
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+
+ int fakeCnt = 0;
+ while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll(); // O(logn)
+
+ // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ if (tri.isDeleted) {
+ fakeCnt--; // 取出了一个被标记删除点
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ continue;
+ }
+
+ // 真正的淘汰点
+ // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ if (debug) {
+ System.out.println("(1) eliminate " + points.get(tri.indices[1]));
+ }
+ points.get(tri.indices[1]).markEliminated();
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
+
+ Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+
+ // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // 前一个三角形连到后一个三角形
+ tri.prev.next =
+ tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ tri.prev.indices[2] = tri.indices[2];
+
+ double mc;
+ if (prefixSum == null) {
+ mc = myLA_withTimestamps(points, tri.prev.indices[0], tri.prev.indices[2]);
+ } else {
+ mc = myLA_withTimestamps(points, prefixSum, tri.prev.indices[0], tri.prev.indices[2]);
+ }
+ tri.prev.area = mc;
+ if (debug) {
+ System.out.println(
+ "(2) updating point on the left "
+ + points.get(tri.prev.indices[1])
+ + ", ranging ["
+ + tri.prev.indices[0]
+ + ","
+ + tri.prev.indices[2]
+ + "]");
}
- // 每个triangle对应两个相邻的待合并的分段
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- points.get(0).prev = null;
- points.get(0).next = points.get(1);
-// points.get(0).index = 0;
- points.get(total - 1).next = null;
- points.get(total - 1).prev = points.get(total - 2);
-// points.get(total - 1).index = points.size() - 1;
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- double mc;
- if (prefixSum == null) {
- mc = myLA_withTimestamps(points, index1, index3);
- } else {
- mc = myLA_withTimestamps(points, prefixSum, index1, index3);
- }
- triangles[i - 1] = new Triangle(index1, index2, index3, mc);
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add来显式重新插入修改后的元素
+ triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+
+ if (tri.next != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
- // 初始化点的状态 用于最后在线采样结果顺序输出未淘汰点
- points.get(i).prev = points.get(i - 1);
- points.get(i).next = points.get(i + 1);
-// points.get(i).index = i; // for swab usage
+ // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+ tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+
+ Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
+
+ if (tri.prev != null) {
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
}
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ // 2. 处理pi被淘汰引起tri.next被更新的事情
+ tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ tri.next.indices[0] = tri.indices[0];
+
+ double mc;
+ if (prefixSum == null) {
+ mc = myLA_withTimestamps(points, tri.next.indices[0], tri.next.indices[2]);
+ } else {
+ mc = myLA_withTimestamps(points, prefixSum, tri.next.indices[0], tri.next.indices[2]);
+ }
+ tri.next.area = mc;
+ if (debug) {
+ System.out.println(
+ "(3) updating point on the right "
+ + points.get(tri.next.indices[1])
+ + ", ranging ["
+ + tri.next.indices[0]
+ + ","
+ + tri.next.indices[2]
+ + "]");
}
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
-
- int fakeCnt = 0;
- while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll(); // O(logn)
-
- // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
- // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
- if (tri.isDeleted) {
- fakeCnt--; // 取出了一个被标记删除点
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
- continue;
- }
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ }
- // 真正的淘汰点
- // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
- if (debug) {
- System.out.println("(1) eliminate " + points.get(tri.indices[1]));
- }
- points.get(tri.indices[1]).markEliminated();
-
- // 更新相邻三角形
- if (tri.prev != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-
- Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
- // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
-
- // 2. 处理pi被淘汰引起tri.prev被更新的事情
- // 前一个三角形连到后一个三角形
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
- tri.prev.indices[2] = tri.indices[2];
-
- double mc;
- if (prefixSum == null) {
- mc = myLA_withTimestamps(points, tri.prev.indices[0], tri.prev.indices[2]);
- } else {
- mc = myLA_withTimestamps(points, prefixSum, tri.prev.indices[0], tri.prev.indices[2]);
- }
- tri.prev.area = mc;
- if (debug) {
- System.out.println("(2) updating point on the left " + points.get(tri.prev.indices[1]) + ", ranging [" + tri.prev.indices[0] + "," + tri.prev.indices[2] + "]");
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add来显式重新插入修改后的元素
- triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
+ List<Point> onlineSampled = new ArrayList<>();
+ Point start = points.get(0);
+ Point end = points.get(points.size() - 1);
+ while (start
+ != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ }
+ onlineSampled.add(end);
+ return onlineSampled;
+ }
+
+ public static int nextSlidingWindowWithTimestamps(
+ List<Point> points, double maxError, Object[] prefixSum) {
+ if (points.size() <= 1) {
+ return points.size() - 1;
+ }
+ int i = 2; // 从第二个点开始
+ if (prefixSum == null) {
+ while (i < points.size() && myLA_withTimestamps(points, 0, i) < maxError) {
+ i++; // 窗口扩大
+ }
+ } else {
+ while (i < points.size() && myLA_withTimestamps(points, prefixSum, 0, i) < maxError) {
+ i++; // 窗口扩大
+ }
+ }
+ return i - 1; // 从0开始，并且是从0到i-1的左闭右闭
+ }
+
+ public static List<Point> swab_framework(List<Point> points, double maxError, int m) {
+ return swab_framework(points, maxError, m, null, false);
+ }
+
+ public static List<Point> reducePoints(List<Point> points, double maxError, Object... kwargs)
+ throws IOException {
+ int m = -1; // 默认值
+ for (Object arg : kwargs) {
+ if (arg instanceof Integer) {
+ m = (Integer) arg; // 获取 m 参数
+ }
+ }
+ if (m <= 0) {
+ throw new IOException("please set m>0");
+ }
+ return swab_framework(points, maxError, m, null, false);
+ }
+
+ public static List<Point> swab_framework(
+ List<Point> points, double maxError, int m, Object[] prefixSum, boolean debug) {
+ if (debug) {
+ if (prefixSum != null) {
+ // deprecated
+ System.out.println("enable prefix sum for accelerating L2 error computation");
+ }
+ }
- if (tri.next != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.next被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
- tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
-
- Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
- // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
-
- if (tri.prev != null) {
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
- }
-
- // 2. 处理pi被淘汰引起tri.next被更新的事情
- tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
- tri.next.indices[0] = tri.indices[0];
-
- double mc;
- if (prefixSum == null) {
- mc = myLA_withTimestamps(points, tri.next.indices[0], tri.next.indices[2]);
- } else {
- mc = myLA_withTimestamps(points, prefixSum, tri.next.indices[0], tri.next.indices[2]);
- }
- tri.next.area = mc;
- if (debug) {
- System.out.println("(3) updating point on the right " + points.get(tri.next.indices[1]) + ", ranging [" + tri.next.indices[0] + "," + tri.next.indices[2] + "]");
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
- triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
- }
+ // 在这里给Point添加全局排序idx, 用于全局定位
+ for (int i = 0; i < points.size(); i++) {
+ points.get(i).index = i;
+ }
- List<Point> onlineSampled = new ArrayList<>();
- Point start = points.get(0);
- Point end = points.get(points.size() - 1);
- while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
- start = start.next; // when e=0, only traversing three points pa pi pb
- }
- onlineSampled.add(end);
- return onlineSampled;
+ if (debug) {
+ System.out.println("data length=" + points.size() + ", max_error=" + maxError + ", m=" + m);
+ }
+ // m在这里的含义是分段数，虽然SWAB不能直接控制分段数，但是也需要这个参数来确定buffer大小
+ int bs = (int) Math.floor((double) points.size() / m * 6); // buffer size
+ int lb = bs / 2, ub = bs * 2;
+ if (lb == 0) {
+ lb += 2; // lb最小为2，否则陷入循环，因为joint buffer里至少会留一个点
+ ub += 2;
+ }
+ if (debug) {
+ System.out.println("bs=" + bs + ", lb=" + lb + ", ub=" + ub);
}
- public static int nextSlidingWindowWithTimestamps(List<Point> points, double maxError, Object[] prefixSum) {
- if (points.size() <= 1) {
- return points.size() - 1;
+ int buffer_startIdx = 0; // 左闭
+ int buffer_endIdx = bs; // 右开
+ // 注意虽然joint，但是buffer和remaining points是buffer_endIdx处是不重叠的
+ // buffer: [0:bs) 左闭右开
+ // remaining: [bs:] 左闭
+
+ List<Point> segTs = new ArrayList<>();
+ segTs.add(points.get(0)); // 全局首点
+
+ boolean needBottomUp = true; // false when only output, no input
+ int reuseIdx = 0; // 注意这里当needBottomUp=false的时候是直接取用上一轮的bottomup points但是base从reuseIdx开始
+ List<Point> sampled = Collections.emptyList();
+ while (true) {
+ if (debug) {
+ System.out.println("#####################################################");
+ System.out.println(
+ "buffer data: ["
+ + buffer_startIdx
+ + ","
+ + buffer_endIdx
+ + ")"
+ + ", remaining data: ["
+ + buffer_endIdx
+ + ","
+ + points.size()
+ + ")");
+ System.out.println("reBottomUp:" + needBottomUp);
+ }
+
+ if (needBottomUp) {
+ if (debug) {
+ System.out.println(">>>>bottom up on the buffer");
}
- int i = 2; // 从第二个点开始
+ reuseIdx = 0; // 重置
if (prefixSum == null) {
- while (i < points.size() && myLA_withTimestamps(points, 0, i) < maxError) {
- i++; // 窗口扩大
- }
+ sampled =
+ seg_bottomUp_maxerror_withTimestamps(
+ points.subList(buffer_startIdx, buffer_endIdx), maxError, null, false);
} else {
- while (i < points.size() && myLA_withTimestamps(points, prefixSum, 0, i) < maxError) {
- i++; // 窗口扩大
+ // deprecated
+ double[] prefixSumX =
+ Arrays.copyOfRange((double[]) prefixSum[0], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumY =
+ Arrays.copyOfRange((double[]) prefixSum[1], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumX2 =
+ Arrays.copyOfRange((double[]) prefixSum[2], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumY2 =
+ Arrays.copyOfRange((double[]) prefixSum[3], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumXY =
+ Arrays.copyOfRange((double[]) prefixSum[4], buffer_startIdx, buffer_endIdx);
+ if (buffer_startIdx > 0) {
+ for (int i = 0; i < prefixSumX.length; i++) {
+ prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY.length; i++) {
+ prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumX2.length; i++) {
+ prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY2.length; i++) {
+ prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_startIdx - 1];
}
+ for (int i = 0; i < prefixSumXY.length; i++) {
+ prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_startIdx - 1];
+ }
+ }
+ Object[] prefixSumPart =
+ new Object[] {
+ prefixSumX,
+ prefixSumY,
+ prefixSumX2,
+ prefixSumY2,
+ prefixSumXY,
+ prefixSum[5],
+ prefixSum[6]
+ };
+ sampled =
+ seg_bottomUp_maxerror_withTimestamps(
+ points.subList(buffer_startIdx, buffer_endIdx), maxError, prefixSumPart, false);
+ // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
}
- return i - 1; // 从0开始，并且是从0到i-1的左闭右闭
- }
- public static List<Point> swab_framework(List<Point> points, double maxError, int m, boolean debug) {
- return swab_framework(points, maxError, m, null, debug);
- }
+ // TODO eBUG also
+ // sampled = eBUG.buildEffectiveArea(points.subList(buffer_startIdx,
+ // buffer_endIdx), 0, false, m);
+ }
+
+ if (debug) {
+ System.out.println("number of bottomUp points=" + (sampled.size() - reuseIdx));
+ System.out.println("---------------------------------");
+ }
- public static List<Point> swab_framework(List<Point> points, double maxError, int m, Object[] prefixSum, boolean debug) {
+ if (sampled.size() - reuseIdx >= 2) { // 左边输出一个segment
if (debug) {
- if (prefixSum != null) {
- // deprecated
- System.out.println("enable prefix sum for accelerating L2 error computation");
+ System.out.println(
+ ">>>>left output a segment:["
+ + sampled.get(reuseIdx).index
+ + ","
+ + sampled.get(1 + reuseIdx).index
+ + "]");
+ }
+ segTs.add(sampled.get(1 + reuseIdx)); // 分段右端点
+ buffer_startIdx = sampled.get(1 + reuseIdx).index; // buffer起点右移
+ }
+
+ if (debug) {
+ System.out.println(
+ "buffer data: ["
+ + buffer_startIdx
+ + ","
+ + buffer_endIdx
+ + ")"
+ + ", remaining data: ["
+ + buffer_endIdx
+ + ","
+ + points.size()
+ + ")");
+ System.out.println("---------------------------------");
+ }
+
+ if (buffer_endIdx >= points.size()) { // no more point added into buffer w
+ if (debug) {
+ System.out.println(">>>>no more data remaining");
+ }
+ // Add remaining segments and break if no more input data
+ if (sampled.size() - reuseIdx
+ > 2) { // more than one segment, 其中第一段segment已经在上面被记录了，但是没有被移出去
+ if (debug) {
+ System.out.println(">>>>so just left output all remaining segments");
+ }
+ for (int k = 2 + reuseIdx; k < sampled.size(); k++) { // 全局尾点就在这里输出
+ segTs.add(sampled.get(k));
+ if (debug) {
+ System.out.println("output: " + sampled.get(k));
}
+ }
}
+ break;
+ }
- // 在这里给Point添加全局排序idx, 用于全局定位
- for (int i = 0; i < points.size(); i++) {
- points.get(i).index = i;
+ if ((buffer_endIdx - buffer_startIdx) >= lb) {
+ if (debug) {
+ System.out.println(
+ ">>>>no need adding the sliding segment from right because len(w) >= lb");
}
+ needBottomUp = false;
+ reuseIdx++; // 这样简单，不需要sampled.remove(0)复杂度
+ if (debug) {
+ System.out.println(
+ "next iteration uses the remaining segment without re-bottomUp, reuseIdx="
+ + reuseIdx);
+ }
+ } else {
if (debug) {
- System.out.println("data length=" + points.size() + ", max_error=" + maxError + ", m=" + m);
+ System.out.println(">>>>adding the sliding segment from right because len(w) < lb");
}
- // m在这里的含义是分段数，虽然SWAB不能直接控制分段数，但是也需要这个参数来确定buffer大小
- int bs = (int) Math.floor((double) points.size() / m * 6); // buffer size
- int lb = bs / 2, ub = bs * 2;
- if (lb == 0) {
- lb += 2; // lb最小为2，否则陷入循环，因为joint buffer里至少会留一个点
- ub += 2;
+ needBottomUp = true; // 加了新的数据进来意味着下一轮里就要re-bottomUp
+
+ while ((buffer_endIdx - buffer_startIdx) < lb && (points.size() - buffer_endIdx) > 0) {
+ // 从0开始，并且rx是从0开始的右闭 [0,rx]一共rx+1个点
+ List<Point> remaining_points = points.subList(buffer_endIdx, points.size());
+ int rx;
+ if (prefixSum == null) {
+ rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, null);
+ } else {
+ // deprecated
+ double[] prefixSumX =
+ Arrays.copyOfRange((double[]) prefixSum[0], buffer_endIdx, points.size());
+ double[] prefixSumY =
+ Arrays.copyOfRange((double[]) prefixSum[1], buffer_endIdx, points.size());
+ double[] prefixSumX2 =
+ Arrays.copyOfRange((double[]) prefixSum[2], buffer_endIdx, points.size());
+ double[] prefixSumY2 =
+ Arrays.copyOfRange((double[]) prefixSum[3], buffer_endIdx, points.size());
+ double[] prefixSumXY =
+ Arrays.copyOfRange((double[]) prefixSum[4], buffer_endIdx, points.size());
+ if (buffer_endIdx > 0) {
+ for (int i = 0; i < prefixSumX.length; i++) {
+ prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY.length; i++) {
+ prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumX2.length; i++) {
+ prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY2.length; i++) {
+ prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumXY.length; i++) {
+ prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_endIdx - 1];
+ }
+ }
+ Object[] prefixSumPart =
+ new Object[] {
+ prefixSumX,
+ prefixSumY,
+ prefixSumX2,
+ prefixSumY2,
+ prefixSumXY,
+ prefixSum[5],
+ prefixSum[6]
+ };
+ rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, prefixSumPart);
+ // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
+ }
+ if ((buffer_endIdx - buffer_startIdx) + (rx + 1) > ub) {
+ // # avoid more than ub
+ rx = ub - (buffer_endIdx - buffer_startIdx) - 1;
+ }
+ buffer_endIdx += (rx + 1); // buffer终点右移
+ if (debug) {
+ System.out.println("input len=" + (rx + 1));
+ }
}
+
if (debug) {
- System.out.println("bs=" + bs + ", lb=" + lb + ", ub=" + ub);
+ System.out.println(
+ "buffer data: ["
+ + buffer_startIdx
+ + ","
+ + buffer_endIdx
+ + ")"
+ + ", remaining data: ["
+ + buffer_endIdx
+ + ","
+ + points.size()
+ + ")");
+ if ((buffer_endIdx - buffer_startIdx) < lb) {
+ System.out.println("warn less");
+ }
+ if ((buffer_endIdx - buffer_startIdx) > ub) {
+ System.out.println("warn more");
+ }
}
+ }
+ }
+ return segTs;
+ }
- int buffer_startIdx = 0; // 左闭
- int buffer_endIdx = bs; // 右开
- // 注意虽然joint，但是buffer和remaining points是buffer_endIdx处是不重叠的
- // buffer: [0:bs) 左闭右开
- // remaining: [bs:] 左闭
+ public static List<Point> seg_bottomUp_m_withTimestamps(
+ List<Point> points, int m, Object[] prefixSum, boolean debug) throws IOException {
+ if (m <= 2) {
+ throw new IOException("please make m>2");
+ }
+ // 直接控制采样点数
+ int total = points.size();
+ if (total < 3) {
+ return points; // 不足 3 个点无法形成三角形
+ }
- List<Point> segTs = new ArrayList<>();
- segTs.add(points.get(0)); //全局首点
+ // 每个triangle对应两个相邻的待合并的分段
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ points.get(0).prev = null;
+ points.get(0).next = points.get(1);
+ // points.get(0).index = 0;
+ points.get(total - 1).next = null;
+ points.get(total - 1).prev = points.get(total - 2);
+ // points.get(total - 1).index = points.size() - 1;
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+ double mc;
+ if (prefixSum == null) {
+ mc = myLA_withTimestamps(points, index1, index3);
+ } else {
+ mc = myLA_withTimestamps(points, prefixSum, index1, index3);
+ }
+ triangles[i - 1] = new Triangle(index1, index2, index3, mc);
+
+ // 初始化点的状态 用于最后在线采样结果顺序输出未淘汰点
+ points.get(i).prev = points.get(i - 1);
+ points.get(i).next = points.get(i + 1);
+ // points.get(i).index = i; // for swab usage
+ }
- boolean needBottomUp = true; // false when only output, no input
- int reuseIdx = 0; // 注意这里当needBottomUp=false的时候是直接取用上一轮的bottomup points但是base从reuseIdx开始
- List<Point> sampled = Collections.emptyList();
- while (true) {
- if (debug) {
- System.out.println("#####################################################");
- System.out.println("buffer data: [" + buffer_startIdx + "," + buffer_endIdx + ")" + ", remaining data: [" + buffer_endIdx + "," + points.size() + ")");
- System.out.println("reBottomUp:" + needBottomUp);
- }
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ }
- if (needBottomUp) {
- if (debug) {
- System.out.println(">>>>bottom up on the buffer");
- }
- reuseIdx = 0; // 重置
- if (prefixSum == null) {
- sampled = seg_bottomUp_maxerror_withTimestamps(points.subList(buffer_startIdx, buffer_endIdx),
- maxError, null, false);
- } else {
- // deprecated
- double[] prefixSumX = Arrays.copyOfRange((double[]) prefixSum[0], buffer_startIdx, buffer_endIdx);
- double[] prefixSumY = Arrays.copyOfRange((double[]) prefixSum[1], buffer_startIdx, buffer_endIdx);
- double[] prefixSumX2 = Arrays.copyOfRange((double[]) prefixSum[2], buffer_startIdx, buffer_endIdx);
- double[] prefixSumY2 = Arrays.copyOfRange((double[]) prefixSum[3], buffer_startIdx, buffer_endIdx);
- double[] prefixSumXY = Arrays.copyOfRange((double[]) prefixSum[4], buffer_startIdx, buffer_endIdx);
- if (buffer_startIdx > 0) {
- for (int i = 0; i < prefixSumX.length; i++) {
- prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_startIdx - 1];
- }
- for (int i = 0; i < prefixSumY.length; i++) {
- prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_startIdx - 1];
- }
- for (int i = 0; i < prefixSumX2.length; i++) {
- prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_startIdx - 1];
- }
- for (int i = 0; i < prefixSumY2.length; i++) {
- prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_startIdx - 1];
- }
- for (int i = 0; i < prefixSumXY.length; i++) {
- prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_startIdx - 1];
- }
- }
- Object[] prefixSumPart = new Object[]{prefixSumX, prefixSumY, prefixSumX2, prefixSumY2, prefixSumXY,
- prefixSum[5], prefixSum[6]};
- sampled = seg_bottomUp_maxerror_withTimestamps(points.subList(buffer_startIdx, buffer_endIdx),
- maxError, prefixSumPart, false);
- // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
- }
-
-
- // TODO eBUG also
-// sampled = eBUG.buildEffectiveArea(points.subList(buffer_startIdx, buffer_endIdx), 0, false, m);
- }
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+
+ int remainNonTerminalPoint = m - 2;
+ int fakeCnt = 0;
+ // while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
+ while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll(); // O(logn)
+
+ // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ if (tri.isDeleted) {
+ fakeCnt--; // 取出了一个被标记删除点
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ continue;
+ }
+
+ // 真正的淘汰点
+ // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ if (debug) {
+ System.out.println("(1) eliminate " + points.get(tri.indices[1]));
+ }
+ points.get(tri.indices[1]).markEliminated();
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
+
+ Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+
+ // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // 前一个三角形连到后一个三角形
+ tri.prev.next =
+ tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ tri.prev.indices[2] = tri.indices[2];
+
+ double mc;
+ if (prefixSum == null) {
+ mc = myLA_withTimestamps(points, tri.prev.indices[0], tri.prev.indices[2]);
+ } else {
+ mc = myLA_withTimestamps(points, prefixSum, tri.prev.indices[0], tri.prev.indices[2]);
+ }
+ tri.prev.area = mc;
+ if (debug) {
+ System.out.println(
+ "(2) updating point on the left "
+ + points.get(tri.prev.indices[1])
+ + ", ranging ["
+ + tri.prev.indices[0]
+ + ","
+ + tri.prev.indices[2]
+ + "]");
+ }
- if (debug) {
- System.out.println("number of bottomUp points=" + (sampled.size() - reuseIdx));
- System.out.println("---------------------------------");
- }
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add来显式重新插入修改后的元素
+ triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
- if (sampled.size() - reuseIdx >= 2) { // 左边输出一个segment
- if (debug) {
- System.out.println(">>>>left output a segment:["
- + sampled.get(reuseIdx).index + "," + sampled.get(1 + reuseIdx).index + "]");
- }
- segTs.add(sampled.get(1 + reuseIdx)); // 分段右端点
- buffer_startIdx = sampled.get(1 + reuseIdx).index; // buffer起点右移
- }
+ if (tri.next != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
- if (debug) {
- System.out.println("buffer data: [" + buffer_startIdx + "," + buffer_endIdx + ")" + ", remaining data: [" + buffer_endIdx + "," + points.size() + ")");
- System.out.println("---------------------------------");
- }
+ // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+ tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
- if (buffer_endIdx >= points.size()) { // no more point added into buffer w
- if (debug) {
- System.out.println(">>>>no more data remaining");
- }
- // Add remaining segments and break if no more input data
- if (sampled.size() - reuseIdx > 2) { // more than one segment, 其中第一段segment已经在上面被记录了，但是没有被移出去
- if (debug) {
- System.out.println(">>>>so just left output all remaining segments");
- }
- for (int k = 2 + reuseIdx; k < sampled.size(); k++) { // 全局尾点就在这里输出
- segTs.add(sampled.get(k));
- if (debug) {
- System.out.println("output: " + sampled.get(k));
- }
- }
- }
- break;
- }
+ Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
- if ((buffer_endIdx - buffer_startIdx) >= lb) {
- if (debug) {
- System.out.println(">>>>no need adding the sliding segment from right because len(w) >= lb");
- }
- needBottomUp = false;
-
- reuseIdx++; // 这样简单，不需要sampled.remove(0)复杂度
- if (debug) {
- System.out.println("next iteration uses the remaining segment without re-bottomUp, reuseIdx=" + reuseIdx);
- }
- } else {
- if (debug) {
- System.out.println(">>>>adding the sliding segment from right because len(w) < lb");
- }
- needBottomUp = true; // 加了新的数据进来意味着下一轮里就要re-bottomUp
-
- while ((buffer_endIdx - buffer_startIdx) < lb && (points.size() - buffer_endIdx) > 0) {
- // 从0开始，并且rx是从0开始的右闭 [0,rx]一共rx+1个点
- List<Point> remaining_points = points.subList(buffer_endIdx, points.size());
- int rx;
- if (prefixSum == null) {
- rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, null);
- } else {
- // deprecated
- double[] prefixSumX = Arrays.copyOfRange((double[]) prefixSum[0], buffer_endIdx, points.size());
- double[] prefixSumY = Arrays.copyOfRange((double[]) prefixSum[1], buffer_endIdx, points.size());
- double[] prefixSumX2 = Arrays.copyOfRange((double[]) prefixSum[2], buffer_endIdx, points.size());
- double[] prefixSumY2 = Arrays.copyOfRange((double[]) prefixSum[3], buffer_endIdx, points.size());
- double[] prefixSumXY = Arrays.copyOfRange((double[]) prefixSum[4], buffer_endIdx, points.size());
- if (buffer_endIdx > 0) {
- for (int i = 0; i < prefixSumX.length; i++) {
- prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_endIdx - 1];
- }
- for (int i = 0; i < prefixSumY.length; i++) {
- prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_endIdx - 1];
- }
- for (int i = 0; i < prefixSumX2.length; i++) {
- prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_endIdx - 1];
- }
- for (int i = 0; i < prefixSumY2.length; i++) {
- prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_endIdx - 1];
- }
- for (int i = 0; i < prefixSumXY.length; i++) {
- prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_endIdx - 1];
- }
- }
- Object[] prefixSumPart = new Object[]{prefixSumX, prefixSumY, prefixSumX2, prefixSumY2, prefixSumXY,
- prefixSum[5], prefixSum[6]};
- rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, prefixSumPart);
- // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
- }
- if ((buffer_endIdx - buffer_startIdx) + (rx + 1) > ub) {
- // # avoid more than ub
- rx = ub - (buffer_endIdx - buffer_startIdx) - 1;
- }
- buffer_endIdx += (rx + 1); // buffer终点右移
- if (debug) {
- System.out.println("input len=" + (rx + 1));
- }
- }
-
- if (debug) {
- System.out.println("buffer data: [" + buffer_startIdx + "," + buffer_endIdx + ")" + ", remaining data: [" + buffer_endIdx + "," + points.size() + ")");
- if ((buffer_endIdx - buffer_startIdx) < lb) {
- System.out.println("warn less");
- }
- if ((buffer_endIdx - buffer_startIdx) > ub) {
- System.out.println("warn more");
- }
- }
- }
+ if (tri.prev != null) {
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
+ }
+
+ // 2. 处理pi被淘汰引起tri.next被更新的事情
+ tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ tri.next.indices[0] = tri.indices[0];
+
+ double mc;
+ if (prefixSum == null) {
+ mc = myLA_withTimestamps(points, tri.next.indices[0], tri.next.indices[2]);
+ } else {
+ mc = myLA_withTimestamps(points, prefixSum, tri.next.indices[0], tri.next.indices[2]);
}
- return segTs;
+ tri.next.area = mc;
+ if (debug) {
+ System.out.println(
+ "(3) updating point on the right "
+ + points.get(tri.next.indices[1])
+ + ", ranging ["
+ + tri.next.indices[0]
+ + ","
+ + tri.next.indices[2]
+ + "]");
+ }
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ }
+
+ List<Point> onlineSampled = new ArrayList<>();
+ Point start = points.get(0);
+ Point end = points.get(points.size() - 1);
+ while (start
+ != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ }
+ onlineSampled.add(end);
+ return onlineSampled;
+ }
+
+ public static void main(String[] args) throws IOException {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 2000;
+ // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < points.size(); i++) {
+ Point point = points.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ // long startTime = System.currentTimeMillis();
+ Object[] prefixSum = prefixSum(polyline.getVertices());
+ // long endTime = System.currentTimeMillis();
+ // System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) +
+ // "ms");
+
+ int m = 100;
+
+ // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
+ long startTime = System.currentTimeMillis();
+ List<Point> sampled = seg_bottomUp_m_withTimestamps(points, m, null, false);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ System.out.println(sampled.size());
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
}
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test3.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test3.java
index d17ea8fcc65..868d58acf8a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test3.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test3.java
@@ -14,7 +14,7 @@ public class Test3 {
public static void main(String[] args) {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random(10);
+ Random rand = new Random(11);
int n = 10000;
int eParam = 0;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test4.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test4.java
index ada4b47b14e..31e358d21de 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test4.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test4.java
@@ -11,69 +11,71 @@ import static org.apache.iotdb.db.query.eBUG.SWAB.myLA_withTimestamps;
import static org.apache.iotdb.db.query.eBUG.SWAB.prefixSum;
public class Test4 {
- // 用于测试使用prefix sum加速L2误差计算的加速效果
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = false;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 2000_0000;
-// Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
-// try (FileWriter writer = new FileWriter("raw.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < polyline.size(); i++) {
-// Point point = polyline.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println(polyline.size() + " Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
+ // 用于测试使用prefix sum加速L2误差计算的加速效果
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 2000_0000;
+ // Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ // try (FileWriter writer = new FileWriter("raw.csv")) {
+ // // 写入CSV头部
+ // writer.append("x,y,z\n");
+ //
+ // // 写入每个点的数据
+ // for (int i = 0; i < polyline.size(); i++) {
+ // Point point = polyline.get(i);
+ // writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ // }
+ // System.out.println(polyline.size() + " Data has been written");
+ // } catch (IOException e) {
+ // System.out.println("Error writing to CSV file: " + e.getMessage());
+ // }
- long startTime = System.currentTimeMillis();
- Object[] prefixSum = prefixSum(polyline.getVertices());
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
+ long startTime = System.currentTimeMillis();
+ Object[] prefixSum = prefixSum(polyline.getVertices());
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
- double error;
+ double error;
- List<Double> error1 = new ArrayList<>();
- startTime = System.currentTimeMillis();
- for (int lx = 0; lx < polyline.size() / 2; lx += 10000) {
- error = myLA_withTimestamps(polyline.getVertices(), prefixSum, lx, polyline.size() - 1);
- System.out.println(lx + "," + error);
- error1.add(error);
- }
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to compute L2 error with prefix sum: " + (endTime - startTime) + "ms");
+ List<Double> error1 = new ArrayList<>();
+ startTime = System.currentTimeMillis();
+ for (int lx = 0; lx < polyline.size() / 2; lx += 10000) {
+ error = myLA_withTimestamps(polyline.getVertices(), prefixSum, lx, polyline.size() - 1);
+ System.out.println(lx + "," + error);
+ error1.add(error);
+ }
+ endTime = System.currentTimeMillis();
+ System.out.println(
+ "Time taken to compute L2 error with prefix sum: " + (endTime - startTime) + "ms");
- List<Double> error2 = new ArrayList<>();
- startTime = System.currentTimeMillis();
- for (int lx = 0; lx < polyline.size() / 2; lx += 10000) {
- error = myLA_withTimestamps(polyline.getVertices(), lx, polyline.size() - 1);
- System.out.println(lx + "," + error);
- error2.add(error);
- }
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to compute L2 error without prefix sum: " + (endTime - startTime) + "ms");
+ List<Double> error2 = new ArrayList<>();
+ startTime = System.currentTimeMillis();
+ for (int lx = 0; lx < polyline.size() / 2; lx += 10000) {
+ error = myLA_withTimestamps(polyline.getVertices(), lx, polyline.size() - 1);
+ System.out.println(lx + "," + error);
+ error2.add(error);
+ }
+ endTime = System.currentTimeMillis();
+ System.out.println(
+ "Time taken to compute L2 error without prefix sum: " + (endTime - startTime) + "ms");
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < error1.size(); i++) {
- writer.println(error1.get(i) + "," + error2.get(i) + "," + (error2.get(i) - error1.get(i)));
- }
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < error1.size(); i++) {
+ writer.println(error1.get(i) + "," + error2.get(i) + "," + (error2.get(i) - error1.get(i)));
+ }
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
}
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test5.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test5.java
index c9ac846a0b2..e7caffeb46c 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test5.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test5.java
@@ -7,58 +7,59 @@ import java.util.Random;
import static org.apache.iotdb.db.query.eBUG.SWAB.seg_bottomUp_maxerror_withTimestamps;
public class Test5 {
- // 用于验证java bottomUpMaxError实现正确性
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = false;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 2000;
-// Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+ // 用于验证java bottomUpMaxError实现正确性
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 2000;
+ // Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(polyline.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
- long startTime = System.currentTimeMillis();
- Object[] prefixSum = SWAB.prefixSum(polyline.getVertices());
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
+ long startTime = System.currentTimeMillis();
+ Object[] prefixSum = SWAB.prefixSum(polyline.getVertices());
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
- startTime = System.currentTimeMillis();
- double maxError = 5000;
- List<Point> sampled = seg_bottomUp_maxerror_withTimestamps(polyline.getVertices(), maxError, prefixSum, false);
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to : " + (endTime - startTime) + "ms");
+ startTime = System.currentTimeMillis();
+ double maxError = 5000;
+ List<Point> sampled =
+ seg_bottomUp_maxerror_withTimestamps(polyline.getVertices(), maxError, prefixSum, false);
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to : " + (endTime - startTime) + "ms");
- for (Point p : sampled) {
- System.out.println(p);
- }
- System.out.println(sampled.size());
+ for (Point p : sampled) {
+ System.out.println(p);
+ }
+ System.out.println(sampled.size());
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < sampled.size(); i++) {
- writer.println(sampled.get(i).x + "," + sampled.get(i).y);
- }
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
}
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test6.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test6.java
index 75f288a061c..c82da4d1e9d 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test6.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test6.java
@@ -8,82 +8,88 @@ import static org.apache.iotdb.db.query.eBUG.SWAB.prefixSum;
import static org.apache.iotdb.db.query.eBUG.SWAB.seg_bottomUp_maxerror_withTimestamps;
public class Test6 {
- // 用于实测bottomUp-L2-with/without prefix sum acceleration和eBUG算法的耗时比较
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = false;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 1000_0000;
-// Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+ // 用于实测bottomUp-L2-with/without prefix sum acceleration和eBUG算法的耗时比较
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 1000_0000;
+ // Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(polyline.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
- long startTime = System.currentTimeMillis();
- Object[] prefixSum = prefixSum(polyline.getVertices());
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
+ long startTime = System.currentTimeMillis();
+ Object[] prefixSum = prefixSum(polyline.getVertices());
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
- double maxError = 500000;
+ double maxError = 500000;
- // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
- startTime = System.currentTimeMillis();
- List<Point> sampled = seg_bottomUp_maxerror_withTimestamps(polyline.getVertices(), maxError, null, false);
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to bottomUp-L2 without prefix sum acceleration: " + (endTime - startTime) + "ms");
- System.out.println(sampled.size());
+ // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
+ startTime = System.currentTimeMillis();
+ List<Point> sampled =
+ seg_bottomUp_maxerror_withTimestamps(polyline.getVertices(), maxError, null, false);
+ endTime = System.currentTimeMillis();
+ System.out.println(
+ "Time taken to bottomUp-L2 without prefix sum acceleration: "
+ + (endTime - startTime)
+ + "ms");
+ System.out.println(sampled.size());
- // 30s worst case O(nlogn)主要就是heap的操作耗时了
- startTime = System.currentTimeMillis();
- sampled = seg_bottomUp_maxerror_withTimestamps(polyline.getVertices(), maxError, prefixSum, false);
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to bottomUp-L2 with prefix sum acceleration:" + (endTime - startTime) + "ms");
- System.out.println(sampled.size());
+ // 30s worst case O(nlogn)主要就是heap的操作耗时了
+ startTime = System.currentTimeMillis();
+ sampled =
+ seg_bottomUp_maxerror_withTimestamps(polyline.getVertices(), maxError, prefixSum, false);
+ endTime = System.currentTimeMillis();
+ System.out.println(
+ "Time taken to bottomUp-L2 with prefix sum acceleration:" + (endTime - startTime) + "ms");
+ System.out.println(sampled.size());
- // 39s worst-case O(n2)但是实际达不到worst-case所以实际运行不会那么大耗时。
- // 不过还是会比L2误差的耗时要大，毕竟AD计算比L2更复杂一些虽然都是线性，但是常数大一些
- startTime = System.currentTimeMillis();
- sampled = eBUG.buildEffectiveArea(polyline, 1000_0000, false, 3591486);
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to eBUG with e>n-3: " + (endTime - startTime) + "ms");
- System.out.println(sampled.size());
+ // 39s worst-case O(n2)但是实际达不到worst-case所以实际运行不会那么大耗时。
+ // 不过还是会比L2误差的耗时要大，毕竟AD计算比L2更复杂一些虽然都是线性，但是常数大一些
+ startTime = System.currentTimeMillis();
+ sampled = eBUG.buildEffectiveArea(polyline, 1000_0000, false, 3591486);
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to eBUG with e>n-3: " + (endTime - startTime) + "ms");
+ System.out.println(sampled.size());
- // 33s worst case O(nlogn)主要就是heap的操作耗时了
- startTime = System.currentTimeMillis();
- sampled = eBUG.buildEffectiveArea(polyline, 0, false, 3591486);
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to eBUG with e=0: " + (endTime - startTime) + "ms");
- System.out.println(sampled.size());
+ // 33s worst case O(nlogn)主要就是heap的操作耗时了
+ startTime = System.currentTimeMillis();
+ sampled = eBUG.buildEffectiveArea(polyline, 0, false, 3591486);
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to eBUG with e=0: " + (endTime - startTime) + "ms");
+ System.out.println(sampled.size());
-// for (Point p : sampled) {
-// System.out.println(p);
-// }
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < sampled.size(); i++) {
- writer.println(sampled.get(i).x + "," + sampled.get(i).y);
- }
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
}
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test7.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test7.java
index 7590d40914d..d8d4a9a06f1 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test7.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test7.java
@@ -7,61 +7,62 @@ import java.util.Random;
import static org.apache.iotdb.db.query.eBUG.SWAB.swab_framework;
public class Test7 {
- // 用于验证java swab实现正确性
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = false;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 10000;
- Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
-// Polyline polyline = new Polyline();
-// for (int i = 0; i < N; i += 1) {
-// double v = rand.nextInt(1000);
-// polyline.addVertex(new Point(i, v));
-// }
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+ // 用于验证java swab实现正确性
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 10000;
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ // Polyline polyline = new Polyline();
+ // for (int i = 0; i < N; i += 1) {
+ // double v = rand.nextInt(1000);
+ // polyline.addVertex(new Point(i, v));
+ // }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(polyline.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ // 写入每个点的数据
+ for (int i = 0; i < points.size(); i++) {
+ Point point = points.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
-// long startTime = System.currentTimeMillis();
-// Object[] prefixSum = prefixSum(polyline.getVertices());
-// long endTime = System.currentTimeMillis();
-// System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
+ // long startTime = System.currentTimeMillis();
+ // Object[] prefixSum = prefixSum(polyline.getVertices());
+ // long endTime = System.currentTimeMillis();
+ // System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) +
+ // "ms");
- double maxError = 0.2;
- int m = 1000;
+ double maxError = 0.2;
+ int m = 1000;
- // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
- long startTime = System.currentTimeMillis();
- List<Point> sampled = swab_framework(polyline.getVertices(), maxError, m, null, true);
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
+ long startTime = System.currentTimeMillis();
+ List<Point> sampled = swab_framework(points, maxError, m, null, true);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
-// for (Point p : sampled) {
-// System.out.println(p);
-// }
- System.out.println(sampled.size());
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ System.out.println(sampled.size());
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < sampled.size(); i++) {
- writer.println(sampled.get(i).x + "," + sampled.get(i).y);
- }
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
}
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test8.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test8.java
index 93b2ddf531c..62550e6178e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test8.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test8.java
@@ -4,62 +4,63 @@ import java.io.*;
import java.util.List;
import java.util.Random;
-import static org.apache.iotdb.db.query.eBUG.SWAB.prefixSum;
-import static org.apache.iotdb.db.query.eBUG.SWAB.swab_framework;
+import static org.apache.iotdb.db.query.eBUG.SWAB.reducePoints;
public class Test8 {
- // 用于测试SWAB速度
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = false;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 1000_0000;
-// Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+ // 用于测试SWAB速度
+ public static void main(String[] args) throws IOException {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 1000_0000;
+ // Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(polyline.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
-// Object[] prefixSum = prefixSum(polyline.getVertices());
+ // Object[] prefixSum = prefixSum(polyline.getVertices());
- double maxError = 20000000;
- int m = 10000;
+ double maxError = 20000000;
+ int m = 10000;
- // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
- long startTime = System.currentTimeMillis();
- List<Point> sampled = swab_framework(polyline.getVertices(), maxError, m, null, false);
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
+ long startTime = System.currentTimeMillis();
+ // List<Point> sampled = swab_framework(polyline.getVertices(), maxError, m, null,
+ // false);
+ List<Point> sampled = reducePoints(polyline.getVertices(), maxError, m);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
-// for (Point p : sampled) {
-// System.out.println(p);
-// }
- System.out.println(sampled.size());
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ System.out.println(sampled.size());
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < sampled.size(); i++) {
- writer.println(sampled.get(i).x + "," + sampled.get(i).y);
- }
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
}
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test9.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test9.java
new file mode 100644
index 00000000000..f6c963ec870
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test9.java
@@ -0,0 +1,64 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.Tool.getParam;
+
+public class Test9 {
+ // 测试swab二分查找maxError参数效果
+ public static void main(String[] args) throws IOException {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 1000_0000;
+ // Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ // epsilon不能太小，否则每次input进来很少的点，循环太多次
+ // 另外buffer不能太大，也就是n/m不能太大，否则每次bottomUp在一个很大的buffer上
+ int m = 1000;
+ // double epsilon = 1000000000; // 这个大了没关系，buffer size会控制住进来的点数的
+
+ long startTime = System.currentTimeMillis();
+ double epsilon = getParam(polyline.getVertices(), m, SWAB::reducePoints, m * 0.1, m);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ startTime = System.currentTimeMillis();
+ List<Point> sampled = SWAB.reducePoints(polyline.getVertices(), epsilon, m);
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ System.out.println(sampled.size());
+ System.out.println(epsilon);
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
index 2f76097918e..a61f091198a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
@@ -2,6 +2,7 @@ package org.apache.iotdb.db.query.eBUG;
import java.io.IOException;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
@@ -10,245 +11,363 @@ import java.util.List;
import java.util.stream.Stream;
public class Tool {
-
- public static Polyline readFromFile(String input, boolean hasHeader, int timeIdx, int valueIdx, int N) {
- Polyline polyline = new Polyline();
-
- // Using Files.lines() for memory-efficient line-by-line reading
- try (Stream<String> lines = Files.lines(Paths.get(input))) {
- Iterator<String> iterator = lines.iterator();
- int lineNumber = 0;
-
- // Skip the header line if necessary
- if (hasHeader && iterator.hasNext()) {
- iterator.next(); // Skip the header
- }
-
- // Process each line
- while (iterator.hasNext()) {
- lineNumber++;
- String line = iterator.next();
- String[] columns = line.split(",");
-
- if (columns.length > Math.max(timeIdx, valueIdx)) {
- double time;
- if (timeIdx < 0) {
- time = lineNumber;
- } else {
- time = Double.parseDouble(columns[timeIdx]);
- }
- double value = Double.parseDouble(columns[valueIdx]);
- polyline.addVertex(new Point(time, value));
- // System.out.println("Line " + lineNumber + " - Time: " + time + ",
- // Value: " + value);
- } else {
- System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
- }
- if (N > 0 && lineNumber >= N) {
- // N>0控制点数，否则N<=0表示读全部数据
- break;
- }
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- return polyline;
- }
-
- // 计算三角形面积
- public static double triArea(Point d0, Point d1, Point d2) {
- double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
- return (dArea > 0.0) ? dArea : -dArea; // abs
+ // Assumed interface for the simplification function
+ interface SimplifyFunction {
+ List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs)
+ throws IOException;
+ }
+
+ // Method to generate the output file name based on input path
+ public static String generateOutputFileName(
+ String inputFilePath, String outDir, String customize) {
+ // Get the file name without extension
+ Path path = Paths.get(inputFilePath);
+ String fileNameWithoutExtension = path.getFileName().toString();
+ String name = fileNameWithoutExtension.substring(0, fileNameWithoutExtension.lastIndexOf('.'));
+
+ // Get the file extension
+ String extension =
+ path.getFileName().toString().substring(fileNameWithoutExtension.lastIndexOf('.'));
+
+ // Create output file path by appending '-ds' before the extension
+ // String outputFile = name + "-ds-e" + e + extension;
+ String outputFile = name + customize + extension;
+
+ if (outDir == null) { // 表示使用input文件所在的文件夹
+ // Handle path compatibility for different operating systems (Linux/Windows)
+ return path.getParent().resolve(outputFile).toString();
+ } else {
+ Path out = Paths.get(outDir);
+ return out.resolve(outputFile).toString();
}
+ }
+
+ public static double getParam(
+ List<Point> points,
+ int m,
+ SimplifyFunction simplifyFunc,
+ double tolerantPts,
+ Object... kwargs)
+ throws IOException {
+ // double x = 1; 不要从1开始，因为对于swab来说使用如此小的maxError会很慢，宁可让maxError从大到小
+ double x = 10;
+ boolean directLess = false;
+ boolean directMore = false;
+
+ int iterNum = 0;
+ int maxIterNum = 50;
+
+ // First binary search loop to find the initial range
+ double base = 2;
+ while (true) {
+ // System.out.println("x=" + x);
+ iterNum++;
+ if (iterNum > maxIterNum) {
+ // System.out.println("reach maxIterNum1");
+ break; // Avoid infinite loops for special cases
+ }
+ List<Point> res = simplifyFunc.reducePoints(points, x, kwargs);
+ int length = res.size();
+ // System.out.println(length);
+
+ if (Math.abs(length - m) <= tolerantPts) {
+ return x; // Return the found parameter
+ }
+
+ if (length > m) {
+ if (directMore) {
+ break; // Reach the first more
+ }
+ if (!directLess) {
+ directLess = true;
+ }
- // 计算简单多边形（边不交叉）的面积（鞋带公式）
- public static double calculatePolygonArea(List<Point> points) {
- // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
- int n = points.size();
- double area = 0;
- for (int i = 0; i < n; i++) {
- int next = (i + 1) % n;
- area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
+ x *= base; // Need less points, increase x
+ } else {
+ if (directLess) {
+ break; // Reach the first less
+ }
+ if (!directMore) {
+ directMore = true;
}
- return Math.abs(area) / 2.0;
- }
- // 计算两个向量的叉积
- public static double crossProduct(double x1, double y1, double x2, double y2) {
- // >0: (x2,y2)在(x1,y1)的逆时针方向
- // <0: (x2,y2)在(x1,y1)的顺时针方向
- // =0: 平行或共线
- return x1 * y2 - y1 * x2;
+ x /= base; // Need more points, decrease x
+ }
}
- // 判断两条线段是否相交并计算交点
- // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
- public static Object[] lineIntersection(Point[] L1, Point[] L2) {
- double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
- double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
-
- // 判断是否相交（叉积）
- double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
- double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
- double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
- double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
-
- // 如果叉积条件满足，表示有交点
- // d1*d2<0意味着P3、P4分别在L12的两边
- // d3*d4<0意味着P1、P2分别在L34的两边
- // 两个同时满足说明有交点
- if (d1 * d2 < 0 && d3 * d4 < 0) {
- double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
- double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
- double x = x1 + t1 * (x2 - x1);
- double y = y1 + t1 * (y2 - y1);
- return new Object[]{true, new Point(x, y)};
- }
+ // Determine the initial left and right bounds
+ double left = (directLess) ? x / base : x;
+ double right = (directMore) ? x * base : x;
+
+ // Refine the range with binary search
+ iterNum = 0;
+ while (true) {
+ iterNum++;
+ if (iterNum > maxIterNum) {
+ // System.out.println("reach maxIterNum");
+ break; // Avoid infinite loops
+ }
+
+ double mid = (left + right) / 2;
+
+ List<Point> res = simplifyFunc.reducePoints(points, mid, kwargs);
+ int length = res.size();
+ // System.out.println(length);
+
+ if (Math.abs(length - m) <= tolerantPts) {
+ return mid; // Return the refined parameter
+ }
+
+ if (length > m) {
+ left = mid; // Need less, narrow range to the left
+ } else {
+ right = mid; // Need more, narrow range to the right
+ }
+ // System.out.println(left + "," + right);
+ }
- // 检查是否起点或终点重合
- if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
- return new Object[]{true, new Point(x1, y1)};
+ return (left + right) / 2; // Return the average of left and right as the final parameter
+ }
+
+ public static List<Point> readFromFile(
+ String input, boolean hasHeader, int timeIdx, int valueIdx, int N) {
+ List<Point> res = new ArrayList<>();
+
+ // Using Files.lines() for memory-efficient line-by-line reading
+ try (Stream<String> lines = Files.lines(Paths.get(input))) {
+ Iterator<String> iterator = lines.iterator();
+ int lineNumber = 0;
+
+ // Skip the header line if necessary
+ if (hasHeader && iterator.hasNext()) {
+ iterator.next(); // Skip the header
+ }
+
+ // Process each line
+ while (iterator.hasNext()) {
+ lineNumber++;
+ String line = iterator.next();
+ String[] columns = line.split(",");
+
+ if (columns.length > Math.max(timeIdx, valueIdx)) {
+ double time;
+ if (timeIdx < 0) {
+ time = lineNumber;
+ } else {
+ time = Double.parseDouble(columns[timeIdx]);
+ }
+ double value = Double.parseDouble(columns[valueIdx]);
+ res.add(new Point(time, value));
+ // System.out.println("Line " + lineNumber + " - Time: " + time + ",
+ // Value: " + value);
+ } else {
+ System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
}
- if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
- return new Object[]{true, new Point(x2, y2)};
+ if (N > 0 && lineNumber >= N) {
+ // N>0控制点数，否则N<=0表示读全部数据
+ break;
}
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return res;
+ }
+
+ // 计算三角形面积
+ public static double triArea(Point d0, Point d1, Point d2) {
+ double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
+ return (dArea > 0.0) ? dArea : -dArea; // abs
+ }
+
+ // 计算简单多边形（边不交叉）的面积（鞋带公式）
+ public static double calculatePolygonArea(List<Point> points) {
+ // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
+ int n = points.size();
+ double area = 0;
+ for (int i = 0; i < n; i++) {
+ int next = (i + 1) % n;
+ area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
+ }
+ return Math.abs(area) / 2.0;
+ }
+
+ // 计算两个向量的叉积
+ public static double crossProduct(double x1, double y1, double x2, double y2) {
+ // >0: (x2,y2)在(x1,y1)的逆时针方向
+ // <0: (x2,y2)在(x1,y1)的顺时针方向
+ // =0: 平行或共线
+ return x1 * y2 - y1 * x2;
+ }
+
+ // 判断两条线段是否相交并计算交点
+ // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
+ public static Object[] lineIntersection(Point[] L1, Point[] L2) {
+ double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
+ double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
+
+ // 判断是否相交（叉积）
+ double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
+ double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
+ double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
+ double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
+
+ // 如果叉积条件满足，表示有交点
+ // d1*d2<0意味着P3、P4分别在L12的两边
+ // d3*d4<0意味着P1、P2分别在L34的两边
+ // 两个同时满足说明有交点
+ if (d1 * d2 < 0 && d3 * d4 < 0) {
+ double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
+ double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
+ double x = x1 + t1 * (x2 - x1);
+ double y = y1 + t1 * (y2 - y1);
+ return new Object[] {true, new Point(x, y)};
+ }
- return new Object[]{false, null};
+ // 检查是否起点或终点重合
+ if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
+ return new Object[] {true, new Point(x1, y1)};
+ }
+ if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
+ return new Object[] {true, new Point(x2, y2)};
}
- // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
- public static double total_areal_displacement(
- List<Point> points, List<Point> points2, boolean debug) {
- double totalArea = 0;
- int i = 0, j = 0; // i for points, j for points2
- Point prevIntersection = null;
- int prevI = -1, prevJ = -1;
-
- // List<double[]> intersectionCoords = new ArrayList<>();
- List<Double> areaList = new ArrayList<>();
-
- while (i < points.size() - 1 && j < points2.size() - 1) {
- if (debug) {
- System.out.println("--------- " + i + " " + j + " ------------");
- }
-
- // 当前线段
- Point[] L1 = {points.get(i), points.get(i + 1)};
- Point[] L2 = {points2.get(j), points2.get(j + 1)};
-
- // 判断是否有交点
- Object[] result = lineIntersection(L1, L2);
- boolean isIntersect = (boolean) result[0];
- Point intersection = (Point) result[1];
-
- if (isIntersect) {
- // intersectionCoords.add(intersection);
-
- if (prevIntersection != null) {
- // 构造多边形点集
- List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
- polygon.add(prevIntersection);
- if (debug) {
- System.out.println("- start intersection: " + prevIntersection);
- }
- polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
- // polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i +
- // 1))); // 添加当前线段的点，左闭右开
- if (debug) {
- System.out.println("- one side: " + points.subList(prevI, i + 1));
- }
- polygon.add(intersection);
- if (debug) {
- System.out.println("- end intersection: " + intersection);
- }
- List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
- Collections.reverse(tempPoints2); // 添加另一序列的点
- polygon.addAll(tempPoints2);
- if (debug) {
- System.out.println("- another side: " + tempPoints2);
- }
-
- // double[][] polygonArray = new double[polygon.size()][2];
- // for (int k = 0; k < polygon.size(); k++) {
- // polygonArray[k] = polygon.get(k);
- // }
-
- // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
- double area = calculatePolygonArea(polygon);
- if (debug) {
- System.out.println("Area = " + area);
- }
- totalArea += area;
- areaList.add(area);
- }
-
- prevIntersection = intersection;
- prevI = i + 1;
- prevJ = j + 1;
- if (debug) {
- System.out.println(
- "This intersection = "
- + intersection
- + ", next polygon: side1 = "
- + prevI
- + ", side2 = "
- + prevJ);
- }
- }
-
- int currentI = i; // 临时存储i
- int currentJ = j; // 临时存储j
- if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
- i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
- }
- if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
- j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
- }
- } // end of while
+ return new Object[] {false, null};
+ }
+
+ // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
+ public static double total_areal_displacement(
+ List<Point> points, List<Point> points2, boolean debug) {
+ double totalArea = 0;
+ int i = 0, j = 0; // i for points, j for points2
+ Point prevIntersection = null;
+ int prevI = -1, prevJ = -1;
+
+ // List<double[]> intersectionCoords = new ArrayList<>();
+ List<Double> areaList = new ArrayList<>();
+
+ while (i < points.size() - 1 && j < points2.size() - 1) {
+ if (debug) {
+ System.out.println("--------- " + i + " " + j + " ------------");
+ }
+
+ // 当前线段
+ Point[] L1 = {points.get(i), points.get(i + 1)};
+ Point[] L2 = {points2.get(j), points2.get(j + 1)};
+
+ // 判断是否有交点
+ Object[] result = lineIntersection(L1, L2);
+ boolean isIntersect = (boolean) result[0];
+ Point intersection = (Point) result[1];
+
+ if (isIntersect) {
+ // intersectionCoords.add(intersection);
+
+ if (prevIntersection != null) {
+ // 构造多边形点集
+ List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
+ polygon.add(prevIntersection);
+ if (debug) {
+ System.out.println("- start intersection: " + prevIntersection);
+ }
+ polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
+ // polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i +
+ // 1))); // 添加当前线段的点，左闭右开
+ if (debug) {
+ System.out.println("- one side: " + points.subList(prevI, i + 1));
+ }
+ polygon.add(intersection);
+ if (debug) {
+ System.out.println("- end intersection: " + intersection);
+ }
+ List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
+ Collections.reverse(tempPoints2); // 添加另一序列的点
+ polygon.addAll(tempPoints2);
+ if (debug) {
+ System.out.println("- another side: " + tempPoints2);
+ }
+
+ // double[][] polygonArray = new double[polygon.size()][2];
+ // for (int k = 0; k < polygon.size(); k++) {
+ // polygonArray[k] = polygon.get(k);
+ // }
+
+ // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
+ double area = calculatePolygonArea(polygon);
+ if (debug) {
+ System.out.println("Area = " + area);
+ }
+ totalArea += area;
+ areaList.add(area);
+ }
+ prevIntersection = intersection;
+ prevI = i + 1;
+ prevJ = j + 1;
if (debug) {
- System.out.println(areaList);
+ System.out.println(
+ "This intersection = "
+ + intersection
+ + ", next polygon: side1 = "
+ + prevI
+ + ", side2 = "
+ + prevJ);
}
-
- return totalArea;
+ }
+
+ int currentI = i; // 临时存储i
+ int currentJ = j; // 临时存储j
+ if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
+ i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
+ j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ } // end of while
+
+ if (debug) {
+ System.out.println(areaList);
}
- // 测试方法
- public static void main(String[] args) {
- // 示例数据
- List<Point> points = new ArrayList<>();
- points.add(new Point(0, 0));
- points.add(new Point(1, 2));
- points.add(new Point(2, -20));
- points.add(new Point(3, 0));
- points.add(new Point(4, -1));
- points.add(new Point(5, -1.5));
- points.add(new Point(6, 0));
-
- List<Point> points2 = new ArrayList<>();
- // points2.add(points.get(0));
- // points2.add(points.get(3));
- // points2.add(points.get(6));
- points2.add(new Point(1, -10));
- points2.add(new Point(3, 0));
-
- double area = total_areal_displacement(points, points2, true);
- System.out.println("Total area: " + area);
-
- // points = new ArrayList<>();
- // points.add(new Point(0, 0));
- // points.add(new Point(1, 2));
- // points.add(new Point(1.5, -10));
- // double area = calculatePolygonArea(points);
- // System.out.println(area);
- //
- // Point[] L1 = new Point[2];
- // L1[0] = new Point(1, 2);
- // L1[1] = new Point(2, -2);
- // Point[] L2 = new Point[2];
- // L2[0] = new Point(0, 0);
- // L2[1] = new Point(3, 0);
- // Object[] res = lineIntersection(L1, L2);
- // System.out.println(res[1]);
- }
+ return totalArea;
+ }
+
+ // 测试方法
+ public static void main(String[] args) {
+ // 示例数据
+ List<Point> points = new ArrayList<>();
+ points.add(new Point(0, 0));
+ points.add(new Point(1, 2));
+ points.add(new Point(2, -20));
+ points.add(new Point(3, 0));
+ points.add(new Point(4, -1));
+ points.add(new Point(5, -1.5));
+ points.add(new Point(6, 0));
+
+ List<Point> points2 = new ArrayList<>();
+ // points2.add(points.get(0));
+ // points2.add(points.get(3));
+ // points2.add(points.get(6));
+ points2.add(new Point(1, -10));
+ points2.add(new Point(3, 0));
+
+ double area = total_areal_displacement(points, points2, true);
+ System.out.println("Total area: " + area);
+
+ // points = new ArrayList<>();
+ // points.add(new Point(0, 0));
+ // points.add(new Point(1, 2));
+ // points.add(new Point(1.5, -10));
+ // double area = calculatePolygonArea(points);
+ // System.out.println(area);
+ //
+ // Point[] L1 = new Point[2];
+ // L1[0] = new Point(1, 2);
+ // L1[1] = new Point(2, -2);
+ // Point[] L2 = new Point[2];
+ // L2[0] = new Point(0, 0);
+ // L2[1] = new Point(3, 0);
+ // Object[] res = lineIntersection(L1, L2);
+ // System.out.println(res[1]);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index cc86a3fe8f9..8da7cca5175 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -28,363 +28,412 @@ import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
public class eBUG {
- public static List<Point> findEliminated(Polyline lineToSimplify, int pa_idx, int pb_idx) {
- // pa: left adjacent non-eliminated point of pi
- // pb: right adjacent non-eliminated point of pi
- // return a list of points, T'_max{0,k-e}[a:b] order by time in ascending order
-
- // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！至于其他全局更早淘汰的点则不一定位于a:b之间
- // pa,xxx,pi,xxx,pb
- // when e=0，遍历点数就是3
- // when e>0，遍历点数大于等于4、小于等于e+3
-
- List<Point> res = new ArrayList<>();
- Point start = lineToSimplify.get(pa_idx);
- Point end = lineToSimplify.get(pb_idx);
- // int cnt = 0;
- while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- res.add(start);
- start = start.next; // when e=0, only traversing three points pa pi pb
- // cnt += 1;
- }
- res.add(end);
- // cnt += 1;
- // System.out.println(cnt); // 3<=cnt<=e+3
-
- return res;
+ public static List<Point> findEliminated(List<Point> points, int pa_idx, int pb_idx) {
+ // pa: left adjacent non-eliminated point of pi
+ // pb: right adjacent non-eliminated point of pi
+ // return a list of points, T'_max{0,k-e}[a:b] order by time in ascending order
+
+ // 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！至于其他全局更早淘汰的点则不一定位于a:b之间
+ // pa,xxx,pi,xxx,pb
+ // when e=0，遍历点数就是3
+ // when e>0，遍历点数大于等于4、小于等于e+3
+
+ List<Point> res = new ArrayList<>();
+ Point start = points.get(pa_idx);
+ Point end = points.get(pb_idx);
+ // int cnt = 0;
+ while (start
+ != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ res.add(start);
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ // cnt += 1;
}
-
- public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
- // precomputation mode
- return buildEffectiveArea(lineToSimplify, e, debug, 0);
+ res.add(end);
+ // cnt += 1;
+ // System.out.println(cnt); // 3<=cnt<=e+3
+
+ return res;
+ }
+
+ public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug) {
+ // precomputation mode
+ return buildEffectiveArea(lineToSimplify.getVertices(), e, debug, 0);
+ }
+
+ public static List<Point> buildEffectiveArea(
+ Polyline lineToSimplify, int e, boolean debug, int m) {
+ // precomputation mode
+ return buildEffectiveArea(lineToSimplify.getVertices(), e, debug, m);
+ }
+
+ public static List<Point> buildEffectiveArea(List<Point> points, int e, boolean debug) {
+ // precomputation mode
+ return buildEffectiveArea(points, e, debug, 0);
+ }
+
+ public static List<Point> buildEffectiveArea(List<Point> points, int e, boolean debug, int m) {
+ if (e < 0) {
+ throw new IllegalArgumentException("Parameter e must be non-negative.");
}
- public static List<Point> buildEffectiveArea(List<Point> points, int e, boolean debug) {
- // precomputation mode
- Polyline polyline = new Polyline();
- polyline.setVertices(points);
- return buildEffectiveArea(polyline, e, debug, 0);
+ if (m > 2) {
+ System.out.println(
+ "online sampling mode, "
+ + "returning "
+ + m
+ + " sampled points sorted by time in ascending order");
+ } else {
+ System.out.println(
+ "offline precomputation mode, "
+ + "returning each point sorted by dominated significance (DS) in ascending order");
}
- public static List<Point> buildEffectiveArea(List<Point> points, int e, boolean debug, int m) {
- // precomputation mode
- Polyline polyline = new Polyline();
- polyline.setVertices(points);
- return buildEffectiveArea(polyline, e, debug, m);
- }
+ // List<Point> results = lineToSimplify.getVertices(); // 浅复制
+ // TODO 预计算结果改成按照bottom-up逐点淘汰顺序（DS递增）排列而不是按照时间戳，这样省去在线时对DS排序的过程
+ // add的是Point引用，所以没有多用一倍的空间
+ List<Point> resultsBottomUpEliminated = new ArrayList<>();
- public static List<Point> buildEffectiveArea(Polyline lineToSimplify, int e, boolean debug, int m) {
- if (e < 0) {
- throw new IllegalArgumentException("Parameter e must be non-negative.");
- }
+ // 存储的是点的引用，这样可以修改原来序列里点的淘汰状态
+ // 存储的是距离当前最新状态的滞后的尚未施加、待施加的e个淘汰点
+ LinkedList<Point> laggedEliminatedPoints = new LinkedList<>();
- if (m > 2) {
- System.out.println("online sampling mode, " + "returning " + m + " sampled points sorted by time in ascending order");
- } else {
- System.out.println("offline precomputation mode, " + "returning each point sorted by dominated significance (DS) in ascending order");
- }
+ int total = points.size();
+ if (total < 3) {
+ return points; // 不足 3 个点无法形成三角形
+ }
- // List<Point> results = lineToSimplify.getVertices(); // 浅复制
- // TODO 预计算结果改成按照bottom-up逐点淘汰顺序（DS递增）排列而不是按照时间戳，这样省去在线时对DS排序的过程
- // add的是Point引用，所以没有多用一倍的空间
- List<Point> resultsBottomUpEliminated = new ArrayList<>();
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ points.get(0).prev = null;
+ points.get(0).next = points.get(1);
+ points.get(total - 1).next = null;
+ points.get(total - 1).prev = points.get(total - 2);
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+ double area = triArea(points.get(index1), points.get(index2), points.get(index3));
+ triangles[i - 1] = new Triangle(index1, index2, index3, area);
+
+ // 初始化点的状态 for eBUG usage 用于快速按照按照时间顺序排列的滞后k个淘汰点状态下位于pa~pb的点；也用于最后在线采样结果顺序输出未淘汰点
+ points.get(i).prev = points.get(i - 1);
+ points.get(i).next = points.get(i + 1);
+ }
- // 存储的是点的引用，这样可以修改原来序列里点的淘汰状态
- // 存储的是距离当前最新状态的滞后的尚未施加、待施加的e个淘汰点
- LinkedList<Point> laggedEliminatedPoints = new LinkedList<>();
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ }
- int total = lineToSimplify.size();
- if (total < 3) {
- return lineToSimplify.getVertices(); // 不足 3 个点无法形成三角形
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+
+ double previousEA = -1;
+ // while (!triangleHeap.isEmpty()) { // TODO 注意triangleHeap里装的是non-terminal point对应的三角形
+ // TODO 在线采样m个点，也就是说最后留下m-2个non-terminal point
+ // 注意：triangleHeap里装的包括了标记删除的点！所以triangleHeap.size()不是真正的留下的未被淘汰点数！
+ // 因此需要用fakeCnt来统计heap里的非真实点数，这样triangleHeap.size()-fakeCnt就是真正的留下的未被淘汰点数
+ int remainNonTerminalPoint = Math.max(0, m - 2);
+ int fakeCnt = 0;
+ while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll(); // O(logn)
+
+ // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ if (tri.isDeleted) {
+ fakeCnt--; // 取出了一个被标记删除点
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
}
-
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- lineToSimplify.get(0).prev = null;
- lineToSimplify.get(0).next = lineToSimplify.get(1);
- lineToSimplify.get(total - 1).next = null;
- lineToSimplify.get(total - 1).prev = lineToSimplify.get(total - 2);
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
- triangles[i - 1] = new Triangle(index1, index2, index3, area);
-
- // 初始化点的状态 for eBUG usage 用于快速按照按照时间顺序排列的滞后k个淘汰点状态下位于pa~pb的点；也用于最后在线采样结果顺序输出未淘汰点
- lineToSimplify.get(i).prev = lineToSimplify.get(i - 1);
- lineToSimplify.get(i).next = lineToSimplify.get(i + 1);
+ continue;
+ }
+
+ // 真正的淘汰点
+ // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ if (debug) {
+ System.out.println("(1) eliminate " + points.get(tri.indices[1]));
+ }
+ if (e > 0) {
+ if (laggedEliminatedPoints.size() == e) {
+ // 已经有e个滞后淘汰点了，为了把最近淘汰点加入，得先把最老的淘汰点施加上去
+ Point removedPoint = laggedEliminatedPoints.removeFirst(); // 取出最早加入的淘汰点 复杂度1
+ removedPoint.markEliminated(); // 注意是引用，所以改的内容后面可以看到
}
-
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ laggedEliminatedPoints.addLast(points.get(tri.indices[1])); // 加入最新的淘汰点，滞后在这里先不施加
+ } else { // e=0 没有滞后 立即标记删除 T'_k
+ points.get(tri.indices[1]).markEliminated();
+ }
+ if (debug) {
+ System.out.println(
+ "the e most recently eliminated points (lagged):" + laggedEliminatedPoints);
+ }
+
+ // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
+ if (tri.area > previousEA) {
+ previousEA = tri.area;
+ }
+ // results.get(tri.indices[1]).z = previousEA; // dominated significance
+ points.get(tri.indices[1]).z = previousEA; // dominated significance
+ resultsBottomUpEliminated.add(points.get(tri.indices[1])); // TODO add的是Point引用，所以没有多用一倍的空间
+ if (debug) {
+ System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
+ }
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
+
+ Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+
+ // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // 前一个三角形连到后一个三角形
+ tri.prev.next =
+ tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ tri.prev.indices[2] = tri.indices[2];
+
+ // e parameter
+ double sig;
+ if (e > 0) {
+ List<Point> pointsForSig =
+ findEliminated(points, tri.prev.indices[0], tri.prev.indices[2]);
+ if (debug) {
+ System.out.println("(2) update point on the left " + points.get(tri.prev.indices[1]));
+ System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
+ for (Point point : pointsForSig) {
+ System.out.println("\t" + point);
+ }
+ }
+ List<Point> baseLine = new ArrayList<>();
+ baseLine.add(pointsForSig.get(0));
+ baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
+ sig = total_areal_displacement(pointsForSig, baseLine, false);
+ } else { // 直接用三角形面积
+ sig =
+ triArea(
+ points.get(tri.prev.indices[0]),
+ points.get(tri.prev.indices[1]),
+ points.get(tri.prev.indices[2]));
+ }
+ tri.prev.area = sig;
+
+ if (debug) {
+ System.out.println("sig=" + sig);
+ double tmpTri =
+ triArea(
+ points.get(tri.prev.indices[0]),
+ points.get(tri.prev.indices[1]),
+ points.get(tri.prev.indices[2]));
+ System.out.println(
+ "\t"
+ + "tri="
+ + tmpTri
+ + ", "
+ + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
}
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
-
- double previousEA = -1;
- // while (!triangleHeap.isEmpty()) { // TODO 注意triangleHeap里装的是non-terminal point对应的三角形
- // TODO 在线采样m个点，也就是说最后留下m-2个non-terminal point
- // 注意：triangleHeap里装的包括了标记删除的点！所以triangleHeap.size()不是真正的留下的未被淘汰点数！
- // 因此需要用fakeCnt来统计heap里的非真实点数，这样triangleHeap.size()-fakeCnt就是真正的留下的未被淘汰点数
- int remainNonTerminalPoint = Math.max(0, m - 2);
- int fakeCnt = 0;
- while (triangleHeap.size() - fakeCnt > remainNonTerminalPoint) {
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll(); // O(logn)
-
- // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
- // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
- if (tri.isDeleted) {
- fakeCnt--; // 取出了一个被标记删除点
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
- continue;
- }
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add来显式重新插入修改后的元素
+ triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
- // 真正的淘汰点
- // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
- if (debug) {
- System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
- }
- if (e > 0) {
- if (laggedEliminatedPoints.size() == e) {
- // 已经有e个滞后淘汰点了，为了把最近淘汰点加入，得先把最老的淘汰点施加上去
- Point removedPoint = laggedEliminatedPoints.removeFirst(); // 取出最早加入的淘汰点 复杂度1
- removedPoint.markEliminated(); // 注意是引用，所以改的内容后面可以看到
- }
- laggedEliminatedPoints.addLast(lineToSimplify.get(tri.indices[1])); // 加入最新的淘汰点，滞后在这里先不施加
- } else { // e=0 没有滞后 立即标记删除 T'_k
- lineToSimplify.get(tri.indices[1]).markEliminated();
- }
- if (debug) {
- System.out.println("the e most recently eliminated points (lagged):" + laggedEliminatedPoints);
- }
+ if (tri.next != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
- // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
- if (tri.area > previousEA) {
- previousEA = tri.area;
- }
- // results.get(tri.indices[1]).z = previousEA; // dominated significance
- lineToSimplify.get(tri.indices[1]).z = previousEA; // dominated significance
- resultsBottomUpEliminated.add(lineToSimplify.get(tri.indices[1])); // TODO add的是Point引用，所以没有多用一倍的空间
- if (debug) {
- System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
- }
+ // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+ tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
- // 更新相邻三角形
- if (tri.prev != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-
- Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
- // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
-
- // 2. 处理pi被淘汰引起tri.prev被更新的事情
- // 前一个三角形连到后一个三角形
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
- tri.prev.indices[2] = tri.indices[2];
-
- // e parameter
- double sig;
- if (e > 0) {
- List<Point> pointsForSig = findEliminated(lineToSimplify, tri.prev.indices[0], tri.prev.indices[2]);
- if (debug) {
- System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
- System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- sig = total_areal_displacement(pointsForSig, baseLine, false);
- } else { // 直接用三角形面积
- sig = triArea(lineToSimplify.get(tri.prev.indices[0]), lineToSimplify.get(tri.prev.indices[1]), lineToSimplify.get(tri.prev.indices[2]));
- }
- tri.prev.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri = triArea(lineToSimplify.get(tri.prev.indices[0]), lineToSimplify.get(tri.prev.indices[1]), lineToSimplify.get(tri.prev.indices[2]));
- System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add来显式重新插入修改后的元素
- triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
+ Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
- if (tri.next != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.next被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
- tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
-
- Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
- // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
-
- if (tri.prev != null) {
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
- }
-
- // 2. 处理pi被淘汰引起tri.next被更新的事情
- tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
- tri.next.indices[0] = tri.indices[0];
-
- // e parameter
- double sig;
- if (e > 0) {
- List<Point> pointsForSig = findEliminated(lineToSimplify, tri.next.indices[0], tri.next.indices[2]);
- if (debug) {
- System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
- System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
- for (Point point : pointsForSig) {
- System.out.println("\t" + point);
- }
- }
- List<Point> baseLine = new ArrayList<>();
- baseLine.add(pointsForSig.get(0));
- baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
- sig = total_areal_displacement(pointsForSig, baseLine, false);
- } else { // 直接用三角形面积
- sig = triArea(lineToSimplify.get(tri.next.indices[0]), lineToSimplify.get(tri.next.indices[1]), lineToSimplify.get(tri.next.indices[2]));
- }
- tri.next.area = sig;
-
- if (debug) {
- System.out.println("sig=" + sig);
- double tmpTri = triArea(lineToSimplify.get(tri.next.indices[0]), lineToSimplify.get(tri.next.indices[1]), lineToSimplify.get(tri.next.indices[2]));
- System.out.println("\t" + "tri=" + tmpTri + ", " + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
- triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
- if (debug) {
- System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
- }
+ if (tri.prev != null) {
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
}
- if (m > 2) { // online sampling mode
- // 把滞后的淘汰点施加上去，然后返回在线采样结果（也就是返回剩余未被淘汰的点）
- for (Point p : laggedEliminatedPoints) {
- p.markEliminated();
- if (debug) {
- System.out.println("apply lagged elimination of " + p);
- }
- }
- List<Point> onlineSampled = new ArrayList<>();
- Point start = lineToSimplify.get(0);
- Point end = lineToSimplify.get(lineToSimplify.size() - 1);
- while (start != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
- start = start.next; // when e=0, only traversing three points pa pi pb
+ // 2. 处理pi被淘汰引起tri.next被更新的事情
+ tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ tri.next.indices[0] = tri.indices[0];
+
+ // e parameter
+ double sig;
+ if (e > 0) {
+ List<Point> pointsForSig =
+ findEliminated(points, tri.next.indices[0], tri.next.indices[2]);
+ if (debug) {
+ System.out.println(
+ "(2) updating point on the right " + points.get(tri.next.indices[1]));
+ System.out.println("3<=cnt<=e+3. cnt=" + pointsForSig.size());
+ for (Point point : pointsForSig) {
+ System.out.println("\t" + point);
}
- onlineSampled.add(end);
- return onlineSampled;
- } else { // offline precomputation mode, for precomputing the dominated significance of each
- // point
- // return results; // 注意这就是lineToSimplify.getVertices()
- // TODO
- resultsBottomUpEliminated.add(lineToSimplify.get(0)); // 全局首点
- resultsBottomUpEliminated.add(lineToSimplify.get(lineToSimplify.size() - 1)); // 全局尾点
- return resultsBottomUpEliminated;
+ }
+ List<Point> baseLine = new ArrayList<>();
+ baseLine.add(pointsForSig.get(0));
+ baseLine.add(pointsForSig.get(pointsForSig.size() - 1)); // 直接连接左右两边最近的未被淘汰的点
+ sig = total_areal_displacement(pointsForSig, baseLine, false);
+ } else { // 直接用三角形面积
+ sig =
+ triArea(
+ points.get(tri.next.indices[0]),
+ points.get(tri.next.indices[1]),
+ points.get(tri.next.indices[2]));
}
+ tri.next.area = sig;
+
+ if (debug) {
+ System.out.println("sig=" + sig);
+ double tmpTri =
+ triArea(
+ points.get(tri.next.indices[0]),
+ points.get(tri.next.indices[1]),
+ points.get(tri.next.indices[2]));
+ System.out.println(
+ "\t"
+ + "tri="
+ + tmpTri
+ + ", "
+ + ((tmpTri > sig) ? "over-estimated" : "equal/less-estimated"));
+ }
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
}
- public static void main(String[] args) {
- // 实验：预计算耗时关于两个参数e、n的变化规律，是否符合复杂度理论建模
-
- // int eParam = 0;
- // int[] eParamList = {0, 0, 1, 2, 3, 4, 5, 10, 14, 15, 16, 20, 30, 40, 50, 100, 200,
- // 500, 1000, 2000, 5000,
- // 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000, 300_0000,
- // 500_0000, 800_0000, 1000_0000,
- // 1500_0000, 2000_0000, 2500_0000, 3000_0000};
- int[] eParamList = {100_0000};
- for (int eParam : eParamList) {
- try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
- for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
- // TODO 注意 要有一个点是n=300w
-
- Polyline polyline = new Polyline();
- int seed = 1;
- Random rand = new Random(seed); // TODO 注意seed在每轮里重设
- for (int i = 0; i < n; i++) {
- double v = rand.nextInt(1000000);
- polyline.addVertex(new Point(i, v));
- }
-
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false, 0);
- long endTime = System.currentTimeMillis();
-
- System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
- writer.println(n + "," + eParam + "," + (endTime - startTime));
- System.out.println(results.size());
-
- // // clear
- // polyline.clear();
- // results.clear();
- // polyline = null;
- // results = null;
- // System.gc();
- }
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
- }
+ if (m > 2) { // online sampling mode
+ // 把滞后的淘汰点施加上去，然后返回在线采样结果（也就是返回剩余未被淘汰的点）
+ for (Point p : laggedEliminatedPoints) {
+ p.markEliminated();
+ if (debug) {
+ System.out.println("apply lagged elimination of " + p);
}
-
- // int n = 300_0000;
- //
- // int seed = 1;
- // Random rand = new Random(seed); // TODO 注意seed在每轮里重设
- // Polyline polyline = new Polyline(); // 注意：如果是固定n变化e实验，这个数据要一次性生成，不然每次随机不一样
- // for (int i = 0; i < n; i++) {
- // double v = rand.nextInt(1000000);
- // polyline.addVertex(new Point(i, v)); //
- // point的状态会在buildEffectiveArea里重置的，所以在下面的e遍历里可以复用这一份数据
- // }
- //
- // try (PrintWriter writer = new PrintWriter(new File("exp_varyE_n" + n + ".csv"))) {
- // int[] eParamList = {0, 1, 2, 10, 1000, 10000, 10_0000, 30_0000, 50_0000, 60_0000,
- // 80_0000, 100_0000, 150_0000, 200_0000};
- // for (int eParam : eParamList) {
- // eParam *= 3; // TODO 注意 因为n=300w而不是100w
- //
- // long startTime = System.currentTimeMillis();
- // List<Point> results = buildEffectiveArea(polyline, eParam, false);
- // long endTime = System.currentTimeMillis();
- //
- // System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce
- // points: " + (endTime - startTime) + "ms");
- // writer.println(n + "," + eParam + "," + (endTime - startTime));
- // System.out.println(results.size());
- //
- // // clear 注意不能把原始数据清除，还要接着用
- //// System.gc();
- // }
- // } catch (FileNotFoundException e) {
- // throw new RuntimeException(e);
- // }
-
- System.out.println("finish");
+ }
+ List<Point> onlineSampled = new ArrayList<>();
+ Point start = points.get(0);
+ Point end = points.get(points.size() - 1);
+ while (start
+ != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ }
+ onlineSampled.add(end);
+ return onlineSampled;
+ } else { // offline precomputation mode, for precomputing the dominated significance of each
+ // point
+ // return results; // 注意这就是lineToSimplify.getVertices()
+ resultsBottomUpEliminated.add(points.get(0)); // 全局首点
+ resultsBottomUpEliminated.add(points.get(points.size() - 1)); // 全局尾点
+ return resultsBottomUpEliminated;
}
+ }
+
+ public static void main(String[] args) {
+ // 实验：预计算耗时关于两个参数e、n的变化规律，是否符合复杂度理论建模
+
+ // int eParam = 0;
+ // int[] eParamList = {0, 0, 1, 2, 3, 4, 5, 10, 14, 15, 16, 20, 30, 40, 50, 100, 200,
+ // 500, 1000, 2000, 5000,
+ // 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000, 300_0000,
+ // 500_0000, 800_0000, 1000_0000,
+ // 1500_0000, 2000_0000, 2500_0000, 3000_0000};
+ int[] eParamList = {100_0000};
+ for (int eParam : eParamList) {
+ try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
+ for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
+ // TODO 注意 要有一个点是n=300w
+
+ Polyline polyline = new Polyline();
+ int seed = 1;
+ Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+ for (int i = 0; i < n; i++) {
+ double v = rand.nextInt(1000000);
+ polyline.addVertex(new Point(i, v));
+ }
+
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false, 0);
+ long endTime = System.currentTimeMillis();
+
+ System.out.println(
+ "n="
+ + n
+ + ", e="
+ + eParam
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+ writer.println(n + "," + eParam + "," + (endTime - startTime));
+ System.out.println(results.size());
+
+ // // clear
+ // polyline.clear();
+ // results.clear();
+ // polyline = null;
+ // results = null;
+ // System.gc();
+ }
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // int n = 300_0000;
+ //
+ // int seed = 1;
+ // Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+ // Polyline polyline = new Polyline(); // 注意：如果是固定n变化e实验，这个数据要一次性生成，不然每次随机不一样
+ // for (int i = 0; i < n; i++) {
+ // double v = rand.nextInt(1000000);
+ // polyline.addVertex(new Point(i, v)); //
+ // point的状态会在buildEffectiveArea里重置的，所以在下面的e遍历里可以复用这一份数据
+ // }
+ //
+ // try (PrintWriter writer = new PrintWriter(new File("exp_varyE_n" + n + ".csv"))) {
+ // int[] eParamList = {0, 1, 2, 10, 1000, 10000, 10_0000, 30_0000, 50_0000, 60_0000,
+ // 80_0000, 100_0000, 150_0000, 200_0000};
+ // for (int eParam : eParamList) {
+ // eParam *= 3; // TODO 注意 因为n=300w而不是100w
+ //
+ // long startTime = System.currentTimeMillis();
+ // List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ // long endTime = System.currentTimeMillis();
+ //
+ // System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce
+ // points: " + (endTime - startTime) + "ms");
+ // writer.println(n + "," + eParam + "," + (endTime - startTime));
+ // System.out.println(results.size());
+ //
+ // // clear 注意不能把原始数据清除，还要接着用
+ //// System.gc();
+ // }
+ // } catch (FileNotFoundException e) {
+ // throw new RuntimeException(e);
+ // }
+
+ System.out.println("finish");
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_Build.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_Build.java
deleted file mode 100644
index fd8c6399c8f..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_Build.java
+++ /dev/null
@@ -1,139 +0,0 @@
-package org.apache.iotdb.db.query.eBUG;
-
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-
-import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
-
-public class eBUG_Build {
- // 输入一条时间序列 t,v
- // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
- // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
- public static void main(String[] args) {
- if (args.length < 6) {
- System.out.println(
- "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,eParam,(outDir)");
- }
- String input = args[0];
- boolean hasHeader = Boolean.parseBoolean(args[1]);
- int timeIdx = Integer.parseInt(args[2]);
- int valueIdx = Integer.parseInt(args[3]);
- int N = Integer.parseInt(args[4]); // N<=0表示读全部行，N>0表示读最多N行
- int eParam = Integer.parseInt(args[5]);
- String outDir;
- if (args.length > 6) {
- outDir = args[6]; // 表示输出文件保存到指定的文件夹
- } else {
- outDir = null; // 表示输出文件保存到input文件所在的文件夹
- }
- String outputFile = generateOutputFileName(input, eParam, outDir);
-
- // 打印信息供用户确认
- System.out.println("Input file: " + input);
- System.out.println("Has header: " + hasHeader);
- System.out.println("Time index: " + timeIdx);
- System.out.println("Value index: " + valueIdx);
- System.out.println("N: " + N);
- System.out.println("eParam: " + eParam);
- System.out.println("outDir: " + outDir);
- System.out.println("Output file: " + outputFile);
-
- // 开始运算
- Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
-// Polyline polyline = new Polyline();
-//
-// // Using Files.lines() for memory-efficient line-by-line reading
-// try (Stream<String> lines = Files.lines(Paths.get(input))) {
-// Iterator<String> iterator = lines.iterator();
-// int lineNumber = 0;
-//
-// // Skip the header line if necessary
-// if (hasHeader && iterator.hasNext()) {
-// iterator.next(); // Skip the header
-// }
-//
-// // Process each line
-// while (iterator.hasNext()) {
-// lineNumber++;
-// String line = iterator.next();
-// String[] columns = line.split(",");
-//
-// if (columns.length > Math.max(timeIdx, valueIdx)) {
-// double time;
-// if (timeIdx < 0) {
-// time = lineNumber;
-// } else {
-// time = Double.parseDouble(columns[timeIdx]);
-// }
-// double value = Double.parseDouble(columns[valueIdx]);
-// polyline.addVertex(new Point(time, value));
-// // System.out.println("Line " + lineNumber + " - Time: " + time + ",
-// // Value: " + value);
-// } else {
-// System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
-// }
-// }
-// } catch (IOException e) {
-// e.printStackTrace();
-// }
-
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false); // precomputation mode
- long endTime = System.currentTimeMillis();
-
- // System.out.println("result point number: " + results.size()); // precomputation
- // mode所以自然是等于n个点
- System.out.println(
- "n="
- + polyline.getVertices().size()
- + ", e="
- + eParam
- + ", Time taken to reduce points: "
- + (endTime - startTime)
- + "ms");
-
- // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
- // 写入表头
- writer.write("z,x,y");
- writer.newLine();
-
- // 写入数据行，按顺序 z, x, y
- for (Point point : results) {
- writer.write(point.z + "," + point.x + "," + point.y);
- writer.newLine();
- }
-
- System.out.println("CSV file written successfully to " + outputFile);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- // Method to generate the output file name based on input path
- private static String generateOutputFileName(String inputFilePath, int e, String outDir) {
- // Get the file name without extension
- Path path = Paths.get(inputFilePath);
- String fileNameWithoutExtension = path.getFileName().toString();
- String name = fileNameWithoutExtension.substring(0, fileNameWithoutExtension.lastIndexOf('.'));
-
- // Get the file extension
- String extension =
- path.getFileName().toString().substring(fileNameWithoutExtension.lastIndexOf('.'));
-
- // Create output file path by appending '-ds' before the extension
- String outputFile = name + "-ds-e" + e + extension;
-
- if (outDir == null) { // 表示使用input文件所在的文件夹
- // Handle path compatibility for different operating systems (Linux/Windows)
- return path.getParent().resolve(outputFile).toString();
- } else {
- Path out = Paths.get(outDir);
- return out.resolve(outputFile).toString();
- }
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_FSW.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_FSW.java
new file mode 100644
index 00000000000..426c99f4638
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_FSW.java
@@ -0,0 +1,82 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
+import static org.apache.iotdb.db.query.eBUG.Tool.getParam;
+
+public class sample_FSW {
+ // 输入一条时间序列 t,v
+ // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
+ // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
+ public static void main(String[] args) throws IOException {
+ if (args.length < 7) {
+ System.out.println(
+ "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,tolerantRatio,(outDir)");
+ }
+ String input = args[0];
+ boolean hasHeader = Boolean.parseBoolean(args[1]);
+ int timeIdx = Integer.parseInt(args[2]);
+ int valueIdx = Integer.parseInt(args[3]);
+ int N = Integer.parseInt(args[4]); // N<=0表示读全部行，N>0表示读最多N行
+ int m = Integer.parseInt(args[5]);
+ double tolerantRatio = Double.parseDouble(args[6]); // 查找参数epsilon以使得输出采样点数接近m的程度，比如0.01
+ String outDir;
+ if (args.length > 7) {
+ outDir = args[7]; // 表示输出文件保存到指定的文件夹
+ } else {
+ outDir = null; // 表示输出文件保存到input文件所在的文件夹
+ }
+
+ // 打印信息供用户确认
+ System.out.println("Input file: " + input);
+ System.out.println("Has header: " + hasHeader);
+ System.out.println("Time index: " + timeIdx);
+ System.out.println("Value index: " + valueIdx);
+ System.out.println("N: " + N);
+ System.out.println("m: " + m);
+ System.out.println("tolerantRatio: " + tolerantRatio);
+ System.out.println("outDir: " + outDir);
+
+ // 读取原始序列
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ String outputFile =
+ generateOutputFileName(input, outDir, "-FSW" + "-n" + points.size() + "-m" + m);
+ System.out.println("Output file: " + outputFile);
+
+ // 查找epsilon参数使得采样点数接近m
+ double epsilon = getParam(points, m, FSW::reducePoints, m * tolerantRatio);
+
+ // 采样
+ long startTime = System.currentTimeMillis();
+ List<Point> results = FSW.reducePoints(points, epsilon);
+ long endTime = System.currentTimeMillis();
+
+ System.out.println("result point number: " + results.size());
+ System.out.println(
+ "n="
+ + points.size()
+ + ", m="
+ + m
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+
+ // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
+ writer.write("x,y");
+ writer.newLine();
+ // 写入数据行，按顺序 x,y
+ for (Point point : results) {
+ writer.write(point.x + "," + point.y);
+ writer.newLine();
+ }
+ System.out.println("CSV file written successfully to " + outputFile);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java
new file mode 100644
index 00000000000..e97763633e7
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java
@@ -0,0 +1,83 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
+import static org.apache.iotdb.db.query.eBUG.Tool.getParam;
+
+public class sample_SWAB {
+ // 输入一条时间序列 t,v
+ // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
+ // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
+ public static void main(String[] args) throws IOException {
+ if (args.length < 7) {
+ System.out.println(
+ "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,tolerantRatio,(outDir)");
+ }
+ String input = args[0];
+ boolean hasHeader = Boolean.parseBoolean(args[1]);
+ int timeIdx = Integer.parseInt(args[2]);
+ int valueIdx = Integer.parseInt(args[3]);
+ int N = Integer.parseInt(args[4]); // N<=0表示读全部行，N>0表示读最多N行
+ int m = Integer.parseInt(args[5]);
+ double tolerantRatio = Double.parseDouble(args[6]); // 查找参数epsilon以使得输出采样点数接近m的程度，比如0.01
+ String outDir;
+ if (args.length > 7) {
+ outDir = args[7]; // 表示输出文件保存到指定的文件夹
+ } else {
+ outDir = null; // 表示输出文件保存到input文件所在的文件夹
+ }
+
+ // 打印信息供用户确认
+ System.out.println("Input file: " + input);
+ System.out.println("Has header: " + hasHeader);
+ System.out.println("Time index: " + timeIdx);
+ System.out.println("Value index: " + valueIdx);
+ System.out.println("N: " + N);
+ System.out.println("m: " + m);
+ System.out.println("tolerantRatio: " + tolerantRatio);
+ System.out.println("outDir: " + outDir);
+
+ // 读取原始序列
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+
+ String outputFile =
+ generateOutputFileName(input, outDir, "-SWAB" + "-n" + points.size() + "-m" + m);
+ System.out.println("Output file: " + outputFile);
+
+ // 查找epsilon参数使得采样点数接近m
+ double epsilon = getParam(points, m, SWAB::reducePoints, m * tolerantRatio, m);
+
+ // 采样
+ long startTime = System.currentTimeMillis();
+ List<Point> results = SWAB.reducePoints(points, epsilon, m);
+ long endTime = System.currentTimeMillis();
+
+ System.out.println("result point number: " + results.size());
+ System.out.println(
+ "n="
+ + points.size()
+ + ", m="
+ + m
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+
+ // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
+ writer.write("x,y");
+ writer.newLine();
+ // 写入数据行，按顺序 x,y
+ for (Point point : results) {
+ writer.write(point.x + "," + point.y);
+ writer.newLine();
+ }
+ System.out.println("CSV file written successfully to " + outputFile);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpL2.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpL2.java
new file mode 100644
index 00000000000..62c4b5ffe30
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpL2.java
@@ -0,0 +1,74 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
+
+public class sample_bottomUpL2 {
+ public static void main(String[] args) throws IOException {
+ if (args.length < 6) {
+ System.out.println(
+ "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,(outDir)");
+ }
+ String input = args[0];
+ boolean hasHeader = Boolean.parseBoolean(args[1]);
+ int timeIdx = Integer.parseInt(args[2]);
+ int valueIdx = Integer.parseInt(args[3]);
+ int N = Integer.parseInt(args[4]); // N<=0表示读全部行，N>0表示读最多N行
+ int m = Integer.parseInt(args[5]);
+ String outDir;
+ if (args.length > 6) {
+ outDir = args[6]; // 表示输出文件保存到指定的文件夹
+ } else {
+ outDir = null; // 表示输出文件保存到input文件所在的文件夹
+ }
+
+ // 打印信息供用户确认
+ System.out.println("Input file: " + input);
+ System.out.println("Has header: " + hasHeader);
+ System.out.println("Time index: " + timeIdx);
+ System.out.println("Value index: " + valueIdx);
+ System.out.println("N: " + N);
+ System.out.println("m: " + m);
+ System.out.println("outDir: " + outDir);
+
+ // 读取原始序列
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+
+ String outputFile =
+ generateOutputFileName(input, outDir, "-BUL2" + "-n" + points.size() + "-m" + m);
+ System.out.println("Output file: " + outputFile);
+
+ // 采样
+ long startTime = System.currentTimeMillis();
+ List<Point> results = SWAB.seg_bottomUp_m_withTimestamps(points, m, null, false);
+ long endTime = System.currentTimeMillis();
+
+ System.out.println("result point number: " + results.size());
+ System.out.println(
+ "n="
+ + points.size()
+ + ", m="
+ + m
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+
+ // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
+ writer.write("x,y");
+ writer.newLine();
+ // 写入数据行，按顺序 x,y
+ for (Point point : results) {
+ writer.write(point.x + "," + point.y);
+ writer.newLine();
+ }
+ System.out.println("CSV file written successfully to " + outputFile);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java
new file mode 100644
index 00000000000..ca44730456c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java
@@ -0,0 +1,100 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
+import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
+
+public class sample_eBUG {
+ // 输入一条时间序列 t,v
+ // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
+ // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
+ public static void main(String[] args) {
+ if (args.length < 7) {
+ System.out.println(
+ "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,eParam,(outDir)");
+ }
+ String input = args[0];
+ boolean hasHeader = Boolean.parseBoolean(args[1]);
+ int timeIdx = Integer.parseInt(args[2]);
+ int valueIdx = Integer.parseInt(args[3]);
+ int N = Integer.parseInt(args[4]); // N<=0表示读全部行，N>0表示读最多N行
+ int m = Integer.parseInt(args[5]); // m<=2代表预计算全部点，m>2代表在线采样m个点
+ int eParam = Integer.parseInt(args[6]);
+ String outDir;
+ if (args.length > 7) {
+ outDir = args[7]; // 表示输出文件保存到指定的文件夹
+ } else {
+ outDir = null; // 表示输出文件保存到input文件所在的文件夹
+ }
+
+ // 打印信息供用户确认
+ System.out.println("Input file: " + input);
+ System.out.println("Has header: " + hasHeader);
+ System.out.println("Time index: " + timeIdx);
+ System.out.println("Value index: " + valueIdx);
+ System.out.println("N: " + N);
+ System.out.println("m: " + m);
+ System.out.println("eParam: " + eParam);
+ System.out.println("outDir: " + outDir);
+
+ // 读取原始序列
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+
+ String outputFile =
+ generateOutputFileName(input, outDir, "-eBUG-e" + eParam + "-n" + points.size() + "-m" + m);
+ System.out.println("Output file: " + outputFile);
+
+ long startTime = System.currentTimeMillis();
+ // m<=2代表预计算全部点，m>2代表在线采样m个点
+ List<Point> results = buildEffectiveArea(points, eParam, false, m);
+ long endTime = System.currentTimeMillis();
+
+ System.out.println("result point number: " + results.size());
+ System.out.println(
+ "n="
+ + points.size()
+ + ", m="
+ + m
+ + ", e="
+ + eParam
+ + ", Time taken to reduce points: "
+ + (endTime - startTime)
+ + "ms");
+
+ // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
+ // 写入表头
+ if (m <= 2) {
+ System.out.println(
+ "precomputation mode, outputting (z,x,y) ordered by z in ascending order");
+ writer.write("z,x,y");
+ writer.newLine();
+
+ // 写入数据行，按顺序 z, x, y
+ for (Point point : results) {
+ writer.write(point.z + "," + point.x + "," + point.y);
+ writer.newLine();
+ }
+ } else {
+ System.out.println(
+ "online sampling mode, outputting (x,y) ordered by x in ascending order");
+ writer.write("x,y");
+ writer.newLine();
+
+ // 写入数据行，按顺序 x,y
+ for (Point point : results) {
+ writer.write(point.x + "," + point.y);
+ writer.newLine();
+ }
+ }
+
+ System.out.println("CSV file written successfully to " + outputFile);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
(iotdb) 36/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 224f11a774a15600ca10cbb4cbdf829238b80b99
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jan 29 00:36:15 2025 +0800
add
---
.../java/org/apache/iotdb/db/query/eBUG/Rdp.java | 36 --
.../org/apache/iotdb/db/query/eBUG/SWAB_AD.java | 524 +++++++++++++++++++++
2 files changed, 524 insertions(+), 36 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
index 45593369ebc..7111624e9f9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
@@ -27,42 +27,6 @@ public class Rdp {
return numerator / denominator;
}
- // public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
- // if (points == null || points.size() < 3) {
- // return points;
- // }
- //
- // // 找到距离最远的点
- // double maxDistance = 0;
- // int index = 0;
- // Point start = points.get(0);
- // Point end = points.get(points.size() - 1);
- //
- // for (int i = 1; i < points.size() - 1; i++) {
- // double distance = point_line_distance(points.get(i), start, end);
- // if (distance > maxDistance) {
- // maxDistance = distance;
- // index = i;
- // }
- // }
- //
- // List<Point> result = new ArrayList<>();
- // if (maxDistance > epsilon) {
- // List<Point> firstPart = reducePoints(points.subList(0, index + 1), epsilon);
- // List<Point> secondPart = reducePoints(points.subList(index, points.size()), epsilon);
- //
- // // 合并结果
- // result.addAll(firstPart.subList(0, firstPart.size() - 1)); // 避免重复添加中间点
- // result.addAll(secondPart);
- // } else {
- // // 如果最远距离小于阈值，则只保留起点和终点
- // result.add(start);
- // result.add(end);
- // }
- //
- // return result;
- // }
-
/**
* 使用迭代实现 RDP 算法。避免递归深度过大。
*
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java
new file mode 100644
index 00000000000..5da1b555258
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java
@@ -0,0 +1,524 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.*;
+
+public class SWAB_AD {
+ public static List<Point> seg_bottomUp_maxerror_withTimestamps(
+ List<Point> points, double maxError, Object[] prefixSum, boolean debug) {
+ int total = points.size();
+ if (total < 3) {
+ return points; // 不足 3 个点无法形成三角形
+ }
+
+ // 每个triangle对应两个相邻的待合并的分段
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ points.get(0).prev = null;
+ points.get(0).next = points.get(1);
+ // points.get(0).index = 0;
+ points.get(total - 1).next = null;
+ points.get(total - 1).prev = points.get(total - 2);
+ // points.get(total - 1).index = points.size() - 1;
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+ // 相当于e=infinity
+ double mc = DP.joint_segment_error(points, index1, index3, DP.ERRORtype.area);
+ triangles[i - 1] = new Triangle(index1, index2, index3, mc);
+
+ // 初始化点的状态 用于最后在线采样结果顺序输出未淘汰点
+ points.get(i).prev = points.get(i - 1);
+ points.get(i).next = points.get(i + 1);
+ // points.get(i).index = i; // for swab usage
+ }
+
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) {
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ }
+
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)
+
+ int fakeCnt = 0;
+ while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll(); // O(logn)
+
+ // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ if (tri.isDeleted) {
+ fakeCnt--; // 取出了一个被标记删除点
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ continue;
+ }
+
+ // 真正的淘汰点
+ // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ if (debug) {
+ System.out.println("(1) eliminate " + points.get(tri.indices[1]));
+ }
+ points.get(tri.indices[1]).markEliminated();
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
+
+ Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+
+ // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // 前一个三角形连到后一个三角形
+ tri.prev.next =
+ tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ tri.prev.indices[2] = tri.indices[2];
+
+ double mc = DP.joint_segment_error(points, tri.prev.indices[0], tri.prev.indices[2], DP.ERRORtype.area);
+ tri.prev.area = mc;
+ if (debug) {
+ System.out.println(
+ "(2) updating point on the left "
+ + points.get(tri.prev.indices[1])
+ + ", ranging ["
+ + tri.prev.indices[0]
+ + ","
+ + tri.prev.indices[2]
+ + "]");
+ }
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add来显式重新插入修改后的元素
+ triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+
+ if (tri.next != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+ tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+
+ Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
+
+ if (tri.prev != null) {
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
+ }
+
+ // 2. 处理pi被淘汰引起tri.next被更新的事情
+ tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ tri.next.indices[0] = tri.indices[0];
+
+ double mc = DP.joint_segment_error(points, tri.next.indices[0], tri.next.indices[2], DP.ERRORtype.area);
+ tri.next.area = mc;
+ if (debug) {
+ System.out.println(
+ "(3) updating point on the right "
+ + points.get(tri.next.indices[1])
+ + ", ranging ["
+ + tri.next.indices[0]
+ + ","
+ + tri.next.indices[2]
+ + "]");
+ }
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ }
+
+ List<Point> onlineSampled = new ArrayList<>();
+ Point start = points.get(0);
+ Point end = points.get(points.size() - 1);
+ while (start
+ != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ }
+ onlineSampled.add(end);
+ return onlineSampled;
+ }
+
+ public static int nextSlidingWindowWithTimestamps(
+ List<Point> points, double maxError, Object[] prefixSum) {
+ if (points.size() <= 1) {
+ return points.size() - 1;
+ }
+ int i = 2; // 从第二个点开始
+ while (i < points.size() && DP.joint_segment_error(points, 0, i, DP.ERRORtype.area) < maxError) {
+ i++; // 窗口扩大
+ }
+ return i - 1; // 从0开始，并且是从0到i-1的左闭右闭
+ }
+
+ public static List<Point> swab_framework(List<Point> points, double maxError, int m) {
+ return swab_framework(points, maxError, m, null, false);
+ }
+
+ public static List<Point> reducePoints(List<Point> points, double maxError, Object... kwargs)
+ throws IOException {
+ int m = -1; // 默认值
+ for (Object arg : kwargs) {
+ if (arg instanceof Integer) {
+ m = (Integer) arg; // 获取 m 参数
+ }
+ }
+ if (m <= 0) {
+ throw new IOException("please set m>0");
+ }
+ return swab_framework(points, maxError, m, null, false);
+ }
+
+ public static List<Point> swab_framework(
+ List<Point> points, double maxError, int m, Object[] prefixSum, boolean debug) {
+ if (debug) {
+ if (prefixSum != null) {
+ // deprecated
+ System.out.println("enable prefix sum for accelerating L2 error computation");
+ }
+ }
+
+ // 在这里给Point添加全局排序idx, 用于全局定位
+ for (int i = 0; i < points.size(); i++) {
+ points.get(i).index = i;
+ }
+
+ if (debug) {
+ System.out.println("data length=" + points.size() + ", max_error=" + maxError + ", m=" + m);
+ }
+ // m在这里的含义是分段数，虽然SWAB不能直接控制分段数，但是也需要这个参数来确定buffer大小
+ int bs = (int) Math.floor((double) points.size() / m * 6); // buffer size
+ int lb = bs / 2, ub = bs * 2;
+ if (lb == 0) {
+ lb += 2; // lb最小为2，否则陷入循环，因为joint buffer里至少会留一个点
+ ub += 2;
+ }
+ if (debug) {
+ System.out.println("bs=" + bs + ", lb=" + lb + ", ub=" + ub);
+ }
+
+ int buffer_startIdx = 0; // 左闭
+ int buffer_endIdx = bs; // 右开
+ // 注意虽然joint，但是buffer和remaining points是buffer_endIdx处是不重叠的
+ // buffer: [0:bs) 左闭右开
+ // remaining: [bs:] 左闭
+
+ List<Point> segTs = new ArrayList<>();
+ segTs.add(points.get(0)); // 全局首点
+
+ boolean needBottomUp = true; // false when only output, no input
+ int reuseIdx = 0; // 注意这里当needBottomUp=false的时候是直接取用上一轮的bottomup points但是base从reuseIdx开始
+ List<Point> sampled = Collections.emptyList();
+ while (true) {
+ if (debug) {
+ System.out.println("#####################################################");
+ System.out.println(
+ "buffer data: ["
+ + buffer_startIdx
+ + ","
+ + buffer_endIdx
+ + ")"
+ + ", remaining data: ["
+ + buffer_endIdx
+ + ","
+ + points.size()
+ + ")");
+ System.out.println("reBottomUp:" + needBottomUp);
+ }
+
+ if (needBottomUp) {
+ if (debug) {
+ System.out.println(">>>>bottom up on the buffer");
+ }
+ reuseIdx = 0; // 重置
+ if (prefixSum == null) {
+ sampled =
+ seg_bottomUp_maxerror_withTimestamps(
+ points.subList(buffer_startIdx, buffer_endIdx), maxError, null, false);
+ } else {
+ // deprecated
+ double[] prefixSumX =
+ Arrays.copyOfRange((double[]) prefixSum[0], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumY =
+ Arrays.copyOfRange((double[]) prefixSum[1], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumX2 =
+ Arrays.copyOfRange((double[]) prefixSum[2], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumY2 =
+ Arrays.copyOfRange((double[]) prefixSum[3], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumXY =
+ Arrays.copyOfRange((double[]) prefixSum[4], buffer_startIdx, buffer_endIdx);
+ if (buffer_startIdx > 0) {
+ for (int i = 0; i < prefixSumX.length; i++) {
+ prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY.length; i++) {
+ prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumX2.length; i++) {
+ prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY2.length; i++) {
+ prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumXY.length; i++) {
+ prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_startIdx - 1];
+ }
+ }
+ Object[] prefixSumPart =
+ new Object[]{
+ prefixSumX,
+ prefixSumY,
+ prefixSumX2,
+ prefixSumY2,
+ prefixSumXY,
+ prefixSum[5],
+ prefixSum[6]
+ };
+ sampled =
+ seg_bottomUp_maxerror_withTimestamps(
+ points.subList(buffer_startIdx, buffer_endIdx), maxError, prefixSumPart, false);
+ // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
+ }
+
+ // TODO eBUG also
+ // sampled = eBUG.buildEffectiveArea(points.subList(buffer_startIdx,
+ // buffer_endIdx), 0, false, m);
+ }
+
+ if (debug) {
+ System.out.println("number of bottomUp points=" + (sampled.size() - reuseIdx));
+ System.out.println("---------------------------------");
+ }
+
+ if (sampled.size() - reuseIdx >= 2) { // 左边输出一个segment
+ if (debug) {
+ System.out.println(
+ ">>>>left output a segment:["
+ + sampled.get(reuseIdx).index
+ + ","
+ + sampled.get(1 + reuseIdx).index
+ + "]");
+ }
+ segTs.add(sampled.get(1 + reuseIdx)); // 分段右端点
+ buffer_startIdx = sampled.get(1 + reuseIdx).index; // buffer起点右移
+ }
+
+ if (debug) {
+ System.out.println(
+ "buffer data: ["
+ + buffer_startIdx
+ + ","
+ + buffer_endIdx
+ + ")"
+ + ", remaining data: ["
+ + buffer_endIdx
+ + ","
+ + points.size()
+ + ")");
+ System.out.println("---------------------------------");
+ }
+
+ if (buffer_endIdx >= points.size()) { // no more point added into buffer w
+ if (debug) {
+ System.out.println(">>>>no more data remaining");
+ }
+ // Add remaining segments and break if no more input data
+ if (sampled.size() - reuseIdx
+ > 2) { // more than one segment, 其中第一段segment已经在上面被记录了，但是没有被移出去
+ if (debug) {
+ System.out.println(">>>>so just left output all remaining segments");
+ }
+ for (int k = 2 + reuseIdx; k < sampled.size(); k++) { // 全局尾点就在这里输出
+ segTs.add(sampled.get(k));
+ if (debug) {
+ System.out.println("output: " + sampled.get(k));
+ }
+ }
+ }
+ break;
+ }
+
+ if ((buffer_endIdx - buffer_startIdx) >= lb) {
+ if (debug) {
+ System.out.println(
+ ">>>>no need adding the sliding segment from right because len(w) >= lb");
+ }
+ needBottomUp = false;
+
+ reuseIdx++; // 这样简单，不需要sampled.remove(0)复杂度
+ if (debug) {
+ System.out.println(
+ "next iteration uses the remaining segment without re-bottomUp, reuseIdx="
+ + reuseIdx);
+ }
+ } else {
+ if (debug) {
+ System.out.println(">>>>adding the sliding segment from right because len(w) < lb");
+ }
+ needBottomUp = true; // 加了新的数据进来意味着下一轮里就要re-bottomUp
+
+ while ((buffer_endIdx - buffer_startIdx) < lb && (points.size() - buffer_endIdx) > 0) {
+ // 从0开始，并且rx是从0开始的右闭 [0,rx]一共rx+1个点
+ List<Point> remaining_points = points.subList(buffer_endIdx, points.size());
+ int rx;
+ if (prefixSum == null) {
+ rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, null);
+ } else {
+ // deprecated
+ double[] prefixSumX =
+ Arrays.copyOfRange((double[]) prefixSum[0], buffer_endIdx, points.size());
+ double[] prefixSumY =
+ Arrays.copyOfRange((double[]) prefixSum[1], buffer_endIdx, points.size());
+ double[] prefixSumX2 =
+ Arrays.copyOfRange((double[]) prefixSum[2], buffer_endIdx, points.size());
+ double[] prefixSumY2 =
+ Arrays.copyOfRange((double[]) prefixSum[3], buffer_endIdx, points.size());
+ double[] prefixSumXY =
+ Arrays.copyOfRange((double[]) prefixSum[4], buffer_endIdx, points.size());
+ if (buffer_endIdx > 0) {
+ for (int i = 0; i < prefixSumX.length; i++) {
+ prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY.length; i++) {
+ prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumX2.length; i++) {
+ prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY2.length; i++) {
+ prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumXY.length; i++) {
+ prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_endIdx - 1];
+ }
+ }
+ Object[] prefixSumPart =
+ new Object[]{
+ prefixSumX,
+ prefixSumY,
+ prefixSumX2,
+ prefixSumY2,
+ prefixSumXY,
+ prefixSum[5],
+ prefixSum[6]
+ };
+ rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, prefixSumPart);
+ // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
+ }
+ if ((buffer_endIdx - buffer_startIdx) + (rx + 1) > ub) {
+ // # avoid more than ub
+ rx = ub - (buffer_endIdx - buffer_startIdx) - 1;
+ }
+ buffer_endIdx += (rx + 1); // buffer终点右移
+ if (debug) {
+ System.out.println("input len=" + (rx + 1));
+ }
+ }
+
+ if (debug) {
+ System.out.println(
+ "buffer data: ["
+ + buffer_startIdx
+ + ","
+ + buffer_endIdx
+ + ")"
+ + ", remaining data: ["
+ + buffer_endIdx
+ + ","
+ + points.size()
+ + ")");
+ if ((buffer_endIdx - buffer_startIdx) < lb) {
+ System.out.println("warn less");
+ }
+ if ((buffer_endIdx - buffer_startIdx) > ub) {
+ System.out.println("warn more");
+ }
+ }
+ }
+ }
+ return segTs;
+ }
+
+ public static void main(String[] args) throws IOException {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 100_0000;
+ // Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ // Object[] prefixSum = prefixSum(polyline.getVertices());
+
+ double maxError = 20000000;
+ int m = 1000;
+
+ // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
+ long startTime = System.currentTimeMillis();
+ // List<Point> sampled = swab_framework(polyline.getVertices(), maxError, m, null,
+ // false);
+ List<Point> sampled = reducePoints(polyline.getVertices(), maxError, m);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ System.out.println(sampled.size());
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+
+}
(iotdb) 44/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c35524ad1a6f3f26a393b3df18e2a3ab1192d74a
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Feb 24 01:40:29 2025 +0800
add
---
.../apache/iotdb/db/query/eBUG/BottomUpYdiff.java | 4 +-
.../java/org/apache/iotdb/db/query/eBUG/DNSL1.java | 160 ++++++-------
.../iotdb/db/query/eBUG/LTD_slow_deprecated.java | 255 ---------------------
.../java/org/apache/iotdb/db/query/eBUG/Rdp.java | 2 +-
.../java/org/apache/iotdb/db/query/eBUG/SWAB.java | 6 +-
.../org/apache/iotdb/db/query/eBUG/SWAB_AD.java | 4 +-
.../java/org/apache/iotdb/db/query/eBUG/Tmp.java | 44 ----
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 6 +-
8 files changed, 91 insertions(+), 390 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/BottomUpYdiff.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/BottomUpYdiff.java
index 6090415241f..ee78ec309a2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/BottomUpYdiff.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/BottomUpYdiff.java
@@ -42,7 +42,7 @@ public class BottomUpYdiff {
}
// 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ for (int i = 0; i < nTriangles; i++) {
triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
}
@@ -50,7 +50,7 @@ public class BottomUpYdiff {
// 使用优先队列构建 minHeap
PriorityQueue<Triangle> triangleHeap =
new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+ Collections.addAll(triangleHeap, triangles);
int remainNonTerminalPoint = m - 2;
int fakeCnt = 0;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
index 28f26a3f619..e28f347a1d5 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
@@ -9,94 +9,94 @@ import java.util.Random;
import static org.apache.iotdb.db.query.eBUG.DP.dynamic_programming;
public class DNSL1 {
- // 默认使用linear interpolation连接分段首尾点来近似
- // 默认使用L1 error
- // 使用divide and conquer算法
- public static List<Point> reducePoints(List<Point> points, int m, boolean debug) {
- int n = points.size();
- int k = m - 1; // 分段数
- int intervalNum = (int) Math.pow((double) n / k, (double) 2 / 3); // divide batch点数
- int intervalPts = n / intervalNum;
+ // 默认使用linear interpolation连接分段首尾点来近似
+ // 默认使用L1 error
+ // 使用divide and conquer算法
+ public static List<Point> reducePoints(List<Point> points, int m, boolean debug) {
+ int n = points.size();
+ int k = m - 1; // 分段数
+ int intervalNum = (int) Math.pow((double) n / k, (double) 2 / 3); // divide batch点数
+ int intervalPts = n / intervalNum;
- if (debug) {
- System.out.println("interval point length=" + intervalPts);
- }
+ if (debug) {
+ System.out.println("interval point length=" + intervalPts);
+ }
- // divide into intervalPts equal-length intervals
- List<Point> allSampledPoints = new ArrayList<>();
- for (int start = 0; start < n; start += intervalPts) {
- int end = Math.min(start + intervalPts, n);
- List<Point> batch = points.subList(start, end); // 左闭右开
- if (debug) {
- System.out.println("Processing batch: [" + start + "," + end + ")");
- }
- List<Point> sampledForBatch = dynamic_programming(batch, k, DP.ERRORtype.L1, false);
- allSampledPoints.addAll(sampledForBatch); // 把上一步的结果加到大的容器里
- }
+ // divide into intervalPts equal-length intervals
+ List<Point> allSampledPoints = new ArrayList<>();
+ for (int start = 0; start < n; start += intervalPts) {
+ int end = Math.min(start + intervalPts, n);
+ List<Point> batch = points.subList(start, end); // 左闭右开
+ if (debug) {
+ System.out.println("Processing batch: [" + start + "," + end + ")");
+ }
+ List<Point> sampledForBatch = dynamic_programming(batch, k, DP.ERRORtype.L1, false);
+ allSampledPoints.addAll(sampledForBatch); // 把上一步的结果加到大的容器里
+ }
- // 在大的容器上最后执行DP.dynamic_programming得到最后的m个采样点
- if (debug) {
- System.out.println("number of points from batch sampling: " + allSampledPoints.size());
- }
+ // 在大的容器上最后执行DP.dynamic_programming得到最后的m个采样点
+ if (debug) {
+ System.out.println("number of points from batch sampling: " + allSampledPoints.size());
+ }
- // TODO 但是注意这里用的是linear interpolation近似，而不是DNS原文的constant
- List<Point> finalSampledPoints =
- dynamic_programming(allSampledPoints, k, DP.ERRORtype.L1, false);
- if (debug) {
- System.out.println("result point number: " + finalSampledPoints.size());
- }
- return finalSampledPoints;
+ // TODO 但是注意这里用的是linear interpolation近似，而不是DNS原文的constant
+ List<Point> finalSampledPoints =
+ dynamic_programming(allSampledPoints, k, DP.ERRORtype.L1, false);
+ if (debug) {
+ System.out.println("result point number: " + finalSampledPoints.size());
}
+ return finalSampledPoints;
+ }
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "raw.csv";
- boolean hasHeader = true;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 100_0000;
-// List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
- List<Point> points = polyline.getVertices();
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "raw.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 100_0000;
+ // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
- // 写入每个点的数据
- for (Point point : points) {
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(points.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
- // int m = (int) Math.pow(points.size(), 0.5);
- int m = 1_000;
+ // int m = (int) Math.pow(points.size(), 0.5);
+ int m = 1_000;
- // long startTime = System.currentTimeMillis();
- // List<Point> sampled = dynamic_programming(points, m - 1, DP.ERROR.area, false);
- // long endTime = System.currentTimeMillis();
- // System.out.println("Time taken: " + (endTime - startTime) + "ms");
- // for (Point p : sampled) {
- // System.out.println(p);
- // }
- // System.out.println(sampled.size());
+ // long startTime = System.currentTimeMillis();
+ // List<Point> sampled = dynamic_programming(points, m - 1, DP.ERROR.area, false);
+ // long endTime = System.currentTimeMillis();
+ // System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ // System.out.println(sampled.size());
- long startTime2 = System.currentTimeMillis();
- // System.out.println(points.size() * m / (Math.pow(points.size() * 1.0 / (m - 1), 2 /
- // 3.0)));
- // System.out.println(10000000 * (2 / (Math.pow(10000000 * 1.0 / (2 - 1), 2 / 3.0))));
- List<Point> sampled2 = reducePoints(points, m, true);
- long endTime2 = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime2 - startTime2) + "ms");
- // for (Point p : sampled2) {
- // System.out.println(p);
- // }
- // System.out.println(sampled2.size());
- }
+ long startTime2 = System.currentTimeMillis();
+ // System.out.println(points.size() * m / (Math.pow(points.size() * 1.0 / (m - 1), 2 /
+ // 3.0)));
+ // System.out.println(10000000 * (2 / (Math.pow(10000000 * 1.0 / (2 - 1), 2 / 3.0))));
+ List<Point> sampled2 = reducePoints(points, m, true);
+ long endTime2 = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime2 - startTime2) + "ms");
+ // for (Point p : sampled2) {
+ // System.out.println(p);
+ // }
+ // System.out.println(sampled2.size());
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD_slow_deprecated.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD_slow_deprecated.java
deleted file mode 100644
index 7746a5c8f8d..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD_slow_deprecated.java
+++ /dev/null
@@ -1,255 +0,0 @@
-package org.apache.iotdb.db.query.eBUG;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-
-public class LTD_slow_deprecated {
- public static Point calculateAveragePoint(List<Point> points, int startClosed, int endClosed) {
- double sumX = 0;
- double sumY = 0;
- for (int i = startClosed; i <= endClosed; i++) {
- sumX += points.get(i).x;
- sumY += points.get(i).y;
- }
- int len = endClosed - startClosed + 1;
- return new Point(sumX / len, sumY / len);
- }
-
- public static double[] calculateLinearRegressionCoefficients(
- List<Point> points, int startClosed, int endClosed) {
- Point averages = calculateAveragePoint(points, startClosed, endClosed);
- double avgX = averages.x;
- double avgY = averages.y;
-
- double aNumerator = 0.0;
- double aDenominator = 0.0;
- for (int i = startClosed; i <= endClosed; i++) {
- aNumerator += (points.get(i).x - avgX) * (points.get(i).y - avgY);
- aDenominator += (points.get(i).x - avgX) * (points.get(i).x - avgX);
- }
-
- double a = aNumerator / aDenominator;
- double b = avgY - a * avgX;
-
- return new double[] {a, b};
- }
-
- public static double calculateSSEForBucket(List<Point> points, int startClosed, int endClosed) {
- double[] coefficients = calculateLinearRegressionCoefficients(points, startClosed, endClosed);
- double a = coefficients[0];
- double b = coefficients[1];
-
- double sse = 0.0;
-
- for (int i = startClosed; i <= endClosed; i++) {
- double error = points.get(i).y - (a * points.get(i).x + b);
- sse += error * error;
- }
-
- return sse;
- }
-
- public static List<Integer> getLtdBinIdxs(List<Point> points, int m, int maxIter, boolean debug) {
- int numOfIterations = (int) (points.size() * 1.0 / m * 10);
- if (maxIter >= 0) {
- // numOfIterations = Math.min(numOfIterations, maxIter);
- numOfIterations = maxIter; // overwrite
- }
- double blockSize = (points.size() - 3) * 1.0 / (m - 2);
-
- List<Integer> offset = new LinkedList<>();
- for (double i = 1; i < points.size(); i += blockSize) {
- offset.add((int) i); // 1~n-2, 这样最后一个offset+1才不会超出边界
- }
- if (debug) {
- System.out.println("numOfIterations=" + numOfIterations);
- System.out.println(offset);
- }
-
- List<Double> sse = new LinkedList<>();
-
- // Initialization
- for (int i = 0; i < m - 2; i++) {
- // with one extra point overlapping for each adjacent bucket
- sse.add(calculateSSEForBucket(points, offset.get(i) - 1, offset.get(i + 1) + 1));
- }
-
- for (int c = 0; c < numOfIterations; c++) {
- if (debug) {
- System.out.println("--------------[" + c + "]----------------");
- }
- // Find the bucket to be split
- int maxSSEIndex = -1;
- double maxSSE = Double.NEGATIVE_INFINITY;
- for (int i = 0; i < m - 2; i++) {
- if (offset.get(i + 1) - offset.get(i) <= 1) {
- continue;
- }
- if (sse.get(i) > maxSSE) {
- maxSSE = sse.get(i);
- maxSSEIndex = i;
- }
- }
- if (maxSSEIndex < 0) {
- if (debug) {
- System.out.println(c);
- System.out.println(maxSSEIndex);
- System.out.println("break max");
- }
- break;
- }
-
- // Find the buckets to be merged
- int minSSEIndex = -1;
- double minSSE = Double.POSITIVE_INFINITY;
- for (int i = 0; i < m - 3; i++) {
- if (i == maxSSEIndex || i + 1 == maxSSEIndex) {
- continue;
- }
- if (sse.get(i) + sse.get(i + 1) < minSSE) {
- minSSE = sse.get(i) + sse.get(i + 1);
- minSSEIndex = i;
- }
- }
- if (minSSEIndex < 0) {
- if (debug) {
- System.out.println(c);
- System.out.println(minSSEIndex);
- System.out.println("break min");
- }
- break;
- }
-
- // Split
- int startIdx = offset.get(maxSSEIndex);
- int endIdx = offset.get(maxSSEIndex + 1);
- if (debug) {
- System.out.println("+++To split bucket: " + startIdx + "," + endIdx + ":" + maxSSE);
- }
- int middleIdx = (startIdx + endIdx) / 2;
- offset.add(maxSSEIndex + 1, middleIdx);
-
- // Update SSE affected by split
- sse.set(
- maxSSEIndex,
- calculateSSEForBucket(
- points, offset.get(maxSSEIndex) - 1, offset.get(maxSSEIndex + 1) + 1));
-
- double newSse =
- calculateSSEForBucket(
- points, offset.get(maxSSEIndex + 1) - 1, offset.get(maxSSEIndex + 2) + 1);
-
- sse.add(maxSSEIndex + 1, newSse);
-
- // Merge
- if (minSSEIndex > maxSSEIndex) {
- minSSEIndex += 1; // Adjust index
- }
- offset.remove(minSSEIndex + 1);
-
- if (debug) {
- System.out.println("---To merge bucket: " + offset.get(minSSEIndex) + ":" + minSSE);
- }
-
- sse.set(
- minSSEIndex,
- calculateSSEForBucket(
- points, offset.get(minSSEIndex) - 1, offset.get(minSSEIndex + 1) + 1));
-
- sse.remove(minSSEIndex + 1);
- }
-
- // Convert ArrayList to int[] and return
- if (debug) {
- System.out.println(offset);
- }
- return offset;
- }
-
- public static List<Point> LTTB(List<Point> points, List<Integer> bins) {
- List<Point> res = new ArrayList<>();
- res.add(points.get(0));
-
- for (int i = 0; i < bins.size() - 1; i++) {
- Point leftAnchor = res.get(res.size() - 1);
- Point rightFloater;
- if (i == bins.size() - 2) {
- rightFloater = points.get(points.size() - 1);
- } else {
- rightFloater = calculateAveragePoint(points, bins.get(i + 1), bins.get(i + 2));
- }
- int bestIdx = -1;
- double maxArea = -Double.MAX_VALUE;
- for (int j = bins.get(i); j < bins.get(i + 1); j++) {
- double area = Tool.triArea(points.get(j), leftAnchor, rightFloater);
- if (area > maxArea) {
- maxArea = area;
- bestIdx = j;
- }
- }
- if (bestIdx > 0) {
- res.add(points.get(bestIdx));
- }
- }
- res.add(points.get(points.size() - 1));
- return res;
- }
-
- public static List<Point> LTD(List<Point> points, int m, int maxIter, boolean debug) {
- List<Integer> bins = getLtdBinIdxs(points, m, maxIter, debug);
- return LTTB(points, bins);
- }
-
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = true;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 100_0000;
- // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
- List<Point> points = polyline.getVertices();
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (Point point : points) {
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(points.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
-
- int m = 1000;
- int maxIter = -1;
- long startTime = System.currentTimeMillis();
- // List<Integer> bins = getLtdBinIdxs(points, m, true);
- List<Point> sampled = LTD(points, m, maxIter, false);
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime - startTime) + "ms");
-
- for (Point p : sampled) {
- System.out.println(p);
- }
-
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < sampled.size(); i++) {
- writer.println(sampled.get(i).x + "," + sampled.get(i).y);
- }
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
index 7111624e9f9..7add7ab57cc 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
@@ -48,7 +48,7 @@ public class Rdp {
keep[0] = true;
keep[points.size() - 1] = true;
- int count = 0; // TODO debug
+ int count = 0;
while (!stack.isEmpty()) {
int[] range = stack.pop();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
index 5c29355fec1..871efecad78 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
@@ -147,7 +147,7 @@ public class SWAB {
}
// 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ for (int i = 0; i < nTriangles; i++) {
triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
}
@@ -155,10 +155,10 @@ public class SWAB {
// 使用优先队列构建 minHeap
PriorityQueue<Triangle> triangleHeap =
new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+ Collections.addAll(triangleHeap, triangles);
int fakeCnt = 0;
- while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
+ while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) {
// 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
// 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
Triangle tri = triangleHeap.poll(); // O(logn)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java
index 3fed7cc4ef0..0cc9a227314 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java
@@ -43,10 +43,10 @@ public class SWAB_AD {
// 使用优先队列构建 minHeap
PriorityQueue<Triangle> triangleHeap =
new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)
+ Collections.addAll(triangleHeap, triangles);
int fakeCnt = 0;
- while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
+ while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) {
// 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
// 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
Triangle tri = triangleHeap.poll(); // O(logn)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java
deleted file mode 100644
index 270aca1dee7..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package org.apache.iotdb.db.query.eBUG;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.PrintWriter;
-import java.util.Comparator;
-import java.util.PriorityQueue;
-import java.util.Random;
-
-public class Tmp {
- // 用于debug为什么n>2kw的耗时增大明显
- public static void main(String[] args) {
- int seed = 10;
- Random rand = new Random(seed);
- int eParam = 1;
- try (PrintWriter writer = new PrintWriter(new File("tmp.csv"))) {
- for (int n = 100_0000;
- n < 100000_0000;
- n += 5000_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
- // for (int n = 19000000; n < 3000_0000; n +=200_0000) {
- PriorityQueue<Point> heap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.y));
- for (int i = 0; i < n; i++) {
- double v = rand.nextInt(1000000);
- heap.add(new Point(i, v));
- }
-
- long startTime = System.currentTimeMillis();
- long sum = 0;
- while (!heap.isEmpty()) {
- Point p = heap.poll();
- sum += p.x;
- }
- long endTime = System.currentTimeMillis();
- System.out.println(n + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
- writer.println(n + "," + "0" + "," + (endTime - startTime));
- System.out.println(sum);
- }
- } catch (FileNotFoundException e) {
- throw new RuntimeException(e);
- }
-
- System.out.println("finish");
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 86d9abaccde..cb67d53a086 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -123,7 +123,7 @@ public class eBUG {
}
// 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ for (int i = 0; i < nTriangles; i++) {
triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
}
@@ -131,7 +131,7 @@ public class eBUG {
// 使用优先队列构建 minHeap
PriorityQueue<Triangle> triangleHeap =
new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
+ Collections.addAll(triangleHeap, triangles);
double previousEA = -1;
// while (!triangleHeap.isEmpty()) { // TODO 注意triangleHeap里装的是non-terminal point对应的三角形
@@ -374,7 +374,7 @@ public class eBUG {
int[] eParamList = {100_0000};
for (int eParam : eParamList) {
try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
- for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
+ for (int n = 100_0000; n < 2000_0000; n += 200_0000) {
// TODO 注意 要有一个点是n=300w
Polyline polyline = new Polyline();
(iotdb) 15/44: just array for e
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b8eb70f98c88baabeb3a4b82f2faa72422803c5f
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jan 22 02:02:07 2025 +0800
just array for e
---
.../db/query/eBUG/TimestampPriorityQueue.java | 38 +++++----
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 91 ++++++++++++----------
.../org/apache/iotdb/db/query/eBUG/eBUG_old.java | 3 +-
.../apache/iotdb/db/query/eBUG/eBUG_old_tmp.java | 4 +-
.../db/query/eBUG/{eBUG.java => eBUG_tmp.java} | 8 +-
5 files changed, 82 insertions(+), 62 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java
index 15c2dbc051c..8e8fb18ebfa 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java
@@ -5,42 +5,54 @@ import java.util.*;
public class TimestampPriorityQueue {
private final int e; // 队列的最大大小
private final LinkedList<Point> insertionOrder; // 保持插入顺序
- public final PriorityQueue<Point> queue; // 按时间戳排序的优先队列
+ // public final PriorityQueue<Point> queue; // 按时间戳排序的优先队列
+ private final List<Point> sortedList; // 维护排序后的列表
// 构造函数：初始化队列大小、插入顺序队列和优先队列
public TimestampPriorityQueue(int e) {
this.e = e;
this.insertionOrder = new LinkedList<>();
- this.queue = new PriorityQueue<>(Comparator.comparingDouble(p -> p.x));
+// this.queue = new PriorityQueue<>(Comparator.comparingDouble(p -> p.x));
+ this.sortedList = new ArrayList<>();
}
public int size() {
- return queue.size();
+// return queue.size();
+ return sortedList.size();
}
// 插入新的元素
public void insert(Point newPoint) {
// 如果队列已满，移除最早插入的点
- if (queue.size() == e) {
+ if (sortedList.size() == e) {
Point removedPoint = insertionOrder.removeFirst(); // 移除最早加入的点 复杂度1
- queue.remove(removedPoint); // 从优先队列中移除该点 TODO 复杂度O(e)!
+// queue.remove(removedPoint); // 从优先队列中移除该点 TODO 复杂度O(e)!
+ sortedList.remove(removedPoint); // 从排序后的列表中移除该点 复杂度O(e)
}
// 将新点添加到插入顺序队列中
insertionOrder.addLast(newPoint); // 复杂度1
// 插入到优先队列，保持时间戳顺序
- queue.offer(newPoint); // 复杂度log(e)
+// queue.add(newPoint); // 复杂度log(e)
+
+ // 插入到排序后的列表中，保持有序
+ int insertIndex = Collections.binarySearch(sortedList, newPoint, Comparator.comparingDouble(p -> p.x)); // 复杂度log(e)
+ if (insertIndex < 0) {
+ insertIndex = -(insertIndex + 1); // 如果未找到，binarySearch返回插入点的负值
+ }
+ sortedList.add(insertIndex, newPoint); // 复杂度O(e)
}
// 获取队列中的所有元素（按时间戳排序）
public List<Point> getQueue() {
- return new ArrayList<>(queue); // 浅复制
+// return new ArrayList<>(queue); // 这个不是排序的！
+ return sortedList;
}
public String toString() {
StringBuffer stringBuffer = new StringBuffer();
- for (Point point : queue) {
+ for (Point point : sortedList) {
stringBuffer.append(point.toString());
stringBuffer.append(",");
}
@@ -48,12 +60,12 @@ public class TimestampPriorityQueue {
}
public static void main(String[] args) {
- TimestampPriorityQueue tq = new TimestampPriorityQueue(3);
+ TimestampPriorityQueue tq = new TimestampPriorityQueue(10);
- tq.insert(new Point(10, 1000));
- tq.insert(new Point(20, 2000));
- tq.insert(new Point(15, 1500));
- tq.insert(new Point(25, 500));
+ tq.insert(new Point(0, 1000));
+ tq.insert(new Point(3, 2000));
+ tq.insert(new Point(11, 1500));
+ tq.insert(new Point(5, 500));
// 打印队列内容
for (Point point : tq.getQueue()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 61721a9e328..1596f32a726 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -19,8 +19,7 @@
package org.apache.iotdb.db.query.eBUG;
-import java.io.FileWriter;
-import java.io.IOException;
+import java.io.*;
import java.util.*;
import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
@@ -266,8 +265,7 @@ public class eBUG {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
-// int n = 1000_0000;
- int n = 20;
+ int n = 100_0000;
int p = 10;
for (int i = 0; i < n; i += p) {
@@ -282,52 +280,61 @@ public class eBUG {
polylineList.add(polylineBatch);
}
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+// try (FileWriter writer = new FileWriter("raw.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
+//
+// // 写入每个点的数据
+// for (int i = 0; i < polyline.size(); i++) {
+// Point point = polyline.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// }
+// System.out.println("Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
+// }
System.out.println("---------------------------------");
// List<Point> results = new ArrayList<>();
// 计算运行时间
- int eParam = 10;
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, true);
- // 输出结果
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(results.size());
-
- if (results.size() <= 100) {
- System.out.println("+++++++++++++++++++");
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+// int eParam = 10;
+ try (PrintWriter writer = new PrintWriter(new File("exp2.csv"))) {
+ int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000};
+// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
+ for (int eParam : eParamList) {
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ // 输出结果
+ long endTime = System.currentTimeMillis();
+ System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println(results.size());
+ writer.println(n + "," + eParam + "," + (endTime - startTime));
}
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
}
- try (FileWriter writer = new FileWriter("fast.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+// if (results.size() <= 100) {
+// System.out.println("+++++++++++++++++++");
+// for (int i = 0; i < results.size(); i++) {
+// Point point = results.get(i);
+// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+// }
+// }
- // 写入每个点的数据
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+// try (FileWriter writer = new FileWriter("fast.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
+//
+// // 写入每个点的数据
+// for (int i = 0; i < results.size(); i++) {
+// Point point = results.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// }
+// System.out.println("Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
+// }
// System.out.println("---------------------------------");
// List<List<Point>> resultsBatchList = new ArrayList<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old.java
index 81825ff1b79..180f277c44e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old.java
@@ -299,7 +299,8 @@ public class eBUG_old {
// 计算运行时间
// int eParam = 10;
try (PrintWriter writer = new PrintWriter(new File("exp.csv"))) {
- int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000};
+ int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 30_0000, 50_0000, 60_0000,
+ 70_0000, 80_0000, 90_0000, 100_0000, 150_0000, 200_0000, 250_0000, 300_0000};
// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
for (int eParam : eParamList) {
long startTime = System.currentTimeMillis();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old_tmp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old_tmp.java
index 2270b8253c2..5cfafb59110 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old_tmp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old_tmp.java
@@ -264,7 +264,7 @@ public class eBUG_old_tmp {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 20;
+ int n = 10000;
int p = 10;
for (int i = 0; i < n; i += p) {
@@ -298,7 +298,7 @@ public class eBUG_old_tmp {
// 计算运行时间
// int eParam = 10;
// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
- int eParam = 10;
+ int eParam = 22;
long startTime = System.currentTimeMillis();
List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_tmp.java
similarity index 99%
copy from server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
copy to server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_tmp.java
index 61721a9e328..a04c4c17920 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_tmp.java
@@ -27,7 +27,7 @@ import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-public class eBUG {
+public class eBUG_tmp {
public static List<Point> findEliminated(Polyline lineToSimplify, TimestampPriorityQueue recentEliminatedOrdered,
int pa_idx, int pi_idx, int pb_idx) {
// TODO 复杂度分析
@@ -267,7 +267,7 @@ public class eBUG {
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
// int n = 1000_0000;
- int n = 20;
+ int n = 10000;
int p = 10;
for (int i = 0; i < n; i += p) {
@@ -299,9 +299,9 @@ public class eBUG {
System.out.println("---------------------------------");
// List<Point> results = new ArrayList<>();
// 计算运行时间
- int eParam = 10;
+ int eParam = 0;
long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, true);
+ List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
long endTime = System.currentTimeMillis();
System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
(iotdb) 21/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c449be4b50651a669c2d8ba8427b7fd894703a68
Author: Lei Rui <10...@qq.com>
AuthorDate: Thu Jan 23 16:38:39 2025 +0800
add
---
.../src/main/java/org/apache/iotdb/db/query/eBUG/Point.java | 3 ---
.../src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java | 5 +++--
.../src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java | 1 +
.../src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java | 13 -------------
4 files changed, 4 insertions(+), 18 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
index 70aa19ea531..6e6fc3aa91e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Point.java
@@ -4,7 +4,6 @@ public class Point {
double x, y, z;
-// public boolean eliminated; // 滞后e个淘汰点 for eBUG usage
public Point prev; // 指向T'_max{0,k-e}里当前点的前一个点 for eBUG usage
public Point next; // 指向T'_max{0,k-e}里当前点的后一个点 for eBUG usage
// 注意Triangle里的prev&next用于最新状态下存在三角形之间的连接关系，
@@ -22,8 +21,6 @@ public class Point {
}
public void markEliminated() {
-// eliminated = true;
-
// to avoid traversing each point between pa to pb,
// instead only traversing at most e most recently eliminated points lagged
prev.next = next;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
index 28b0eebd53e..426cfdc538d 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test1.java
@@ -9,10 +9,11 @@ import java.util.Random;
import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
public class Test1 {
+ // 用于验证Java eBUG实现和python版本(e=0/1)结果的一致性
public static void main(String[] args) {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random(1);
+ Random rand = new Random(2);
int n = 10000;
int p = 10;
@@ -43,7 +44,7 @@ public class Test1 {
}
System.out.println("---------------------------------");
- int eParam = 1000000;
+ int eParam = 0;
long startTime = System.currentTimeMillis();
List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java
index ad95ee6ec51..9cfb8ac3d5f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java
@@ -8,6 +8,7 @@ import java.util.PriorityQueue;
import java.util.Random;
public class Tmp {
+ // 用于debug为什么n>2kw的耗时增大明显
public static void main(String[] args) {
int seed = 10;
Random rand = new Random(seed);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 408d440a46f..e283789239a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -52,12 +52,6 @@ public class eBUG {
// cnt += 1;
// System.out.println(cnt); // 3<=cnt<=e+3
-// for (int i = pa_idx; i <= pb_idx; i++) { // 左闭右闭
-// Point p = lineToSimplify.get(i);
-// if (!p.eliminated) {
-// res.add(p);
-// }
-// }
return res;
}
@@ -68,10 +62,6 @@ public class eBUG {
List<Point> results = lineToSimplify.getVertices(); // 浅复制
- // 用循环数组来记录最近淘汰的点
-// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
-// int recentEliminatedIdx = 0; // index in the array, circulate
-
// 存储的是点的引用，这样可以修改原来序列里点的淘汰状态
// 存储的是距离当前最新状态的滞后的尚未施加、待施加的e个淘汰点
LinkedList<Point> laggedEliminatedPoints = new LinkedList<>();
@@ -85,10 +75,8 @@ public class eBUG {
Triangle[] triangles = new Triangle[nTriangles];
// 创建所有三角形并计算初始面积
-// lineToSimplify.get(0).eliminated = false; // 初始化点的状态 for eBUG usage
lineToSimplify.get(0).prev = null;
lineToSimplify.get(0).next = lineToSimplify.get(1);
-// lineToSimplify.get(total - 1).eliminated = false; // 初始化点的状态 for eBUG usage
lineToSimplify.get(total - 1).next = null;
lineToSimplify.get(total - 1).prev = lineToSimplify.get(total - 2);
for (int i = 1; i < total - 1; i++) {
@@ -97,7 +85,6 @@ public class eBUG {
triangles[i - 1] = new Triangle(index1, index2, index3, area);
// 初始化点的状态 for eBUG usage
-// lineToSimplify.get(i).eliminated = false;
lineToSimplify.get(i).prev = lineToSimplify.get(i - 1);
lineToSimplify.get(i).next = lineToSimplify.get(i + 1);
}
(iotdb) 35/44: fix
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b74a25f14d2956775b3d830535c4ffffe0f3fede
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jan 28 21:17:36 2025 +0800
fix
---
server/pom.xml | 8 +-
server/sample_minmax-jar-with-dependencies.jar | Bin 40768753 -> 40769057 bytes
.../org/apache/iotdb/db/query/eBUG/MinMax.java | 28 +-
.../java/org/apache/iotdb/db/query/eBUG/Rdp.java | 526 ++++++++++-----------
4 files changed, 283 insertions(+), 279 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index f21af9a30cd..760ce5fe6dd 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -287,17 +287,17 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
- <finalName>sample_swab</finalName>
+ <!-- <finalName>sample_swab</finalName>-->
<!-- <finalName>sample_rdp</finalName>-->
- <!-- <finalName>sample_minmax</finalName>-->
+ <finalName>sample_minmax</finalName>
<!-- <finalName>sample_fsw</finalName>-->
<!-- <finalName>sample_eBUG</finalName>-->
<!-- <finalName>sample_BUYdiff</finalName>-->
<archive>
<manifest>
- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>-->
+ <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>-->
diff --git a/server/sample_minmax-jar-with-dependencies.jar b/server/sample_minmax-jar-with-dependencies.jar
index 1cf9f21e3f1..f827302b64a 100644
Binary files a/server/sample_minmax-jar-with-dependencies.jar and b/server/sample_minmax-jar-with-dependencies.jar differ
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/MinMax.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/MinMax.java
index 5ee516f0f9f..0afee480b29 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/MinMax.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/MinMax.java
@@ -38,12 +38,14 @@ public class MinMax {
Point p = points.get(i);
while (i >= currentBucketEndIdx) {
// record the results of the last bucket
- if (minIdx < maxIdx) {
- res.add(points.get(minIdx));
- res.add(points.get(maxIdx));
- } else {
- res.add(points.get(maxIdx));
- res.add(points.get(minIdx));
+ if (minIdx != -1) {
+ if (minIdx < maxIdx) {
+ res.add(points.get(minIdx));
+ res.add(points.get(maxIdx));
+ } else {
+ res.add(points.get(maxIdx));
+ res.add(points.get(minIdx));
+ }
}
if (debug) {
System.out.println(
@@ -129,12 +131,14 @@ public class MinMax {
Point p = points.get(i);
while (p.x >= currentBucketEndTime) {
// record the results of the last bucket
- if (minIdx < maxIdx) {
- res.add(points.get(minIdx));
- res.add(points.get(maxIdx));
- } else {
- res.add(points.get(maxIdx));
- res.add(points.get(minIdx));
+ if (minIdx != -1) {
+ if (minIdx < maxIdx) {
+ res.add(points.get(minIdx));
+ res.add(points.get(maxIdx));
+ } else {
+ res.add(points.get(maxIdx));
+ res.add(points.get(minIdx));
+ }
}
if (debug) {
System.out.println(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
index 77b6df5aa3c..45593369ebc 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
@@ -7,286 +7,286 @@ import java.util.Random;
import java.util.Stack;
public class Rdp {
- // Ramer–Douglas–Peucker algorithm (RDP)
- // top-down, for min perpendicular distance
- // worst case complexity n2
- // best case complexity nlogn
-
- public static double point_line_distance(Point point, Point lineStart, Point lineEnd) {
- // 计算点到直线的垂直正交距离
- // (x0,y0)到直线（经过(x1,y1)、(x2,y2)点的直线）的正交距离
- double x0 = point.x;
- double y0 = point.y;
- double x1 = lineStart.x;
- double y1 = lineStart.y;
- double x2 = lineEnd.x;
- double y2 = lineEnd.y;
-
- double numerator = Math.abs((x2 - x1) * (y1 - y0) - (x1 - x0) * (y2 - y1));
- double denominator = Math.sqrt(Math.pow(x2 - x1, 2) + Math.pow(y2 - y1, 2));
- return numerator / denominator;
+ // Ramer–Douglas–Peucker algorithm (RDP)
+ // top-down, for min perpendicular distance
+ // worst case complexity n2
+ // best case complexity nlogn
+
+ public static double point_line_distance(Point point, Point lineStart, Point lineEnd) {
+ // 计算点到直线的垂直正交距离
+ // (x0,y0)到直线（经过(x1,y1)、(x2,y2)点的直线）的正交距离
+ double x0 = point.x;
+ double y0 = point.y;
+ double x1 = lineStart.x;
+ double y1 = lineStart.y;
+ double x2 = lineEnd.x;
+ double y2 = lineEnd.y;
+
+ double numerator = Math.abs((x2 - x1) * (y1 - y0) - (x1 - x0) * (y2 - y1));
+ double denominator = Math.sqrt(Math.pow(x2 - x1, 2) + Math.pow(y2 - y1, 2));
+ return numerator / denominator;
+ }
+
+ // public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
+ // if (points == null || points.size() < 3) {
+ // return points;
+ // }
+ //
+ // // 找到距离最远的点
+ // double maxDistance = 0;
+ // int index = 0;
+ // Point start = points.get(0);
+ // Point end = points.get(points.size() - 1);
+ //
+ // for (int i = 1; i < points.size() - 1; i++) {
+ // double distance = point_line_distance(points.get(i), start, end);
+ // if (distance > maxDistance) {
+ // maxDistance = distance;
+ // index = i;
+ // }
+ // }
+ //
+ // List<Point> result = new ArrayList<>();
+ // if (maxDistance > epsilon) {
+ // List<Point> firstPart = reducePoints(points.subList(0, index + 1), epsilon);
+ // List<Point> secondPart = reducePoints(points.subList(index, points.size()), epsilon);
+ //
+ // // 合并结果
+ // result.addAll(firstPart.subList(0, firstPart.size() - 1)); // 避免重复添加中间点
+ // result.addAll(secondPart);
+ // } else {
+ // // 如果最远距离小于阈值，则只保留起点和终点
+ // result.add(start);
+ // result.add(end);
+ // }
+ //
+ // return result;
+ // }
+
+ /**
+ * 使用迭代实现 RDP 算法。避免递归深度过大。
+ *
+ * @param points 点集
+ * @param epsilon 简化阈值
+ * @return 简化后的点集
+ */
+ public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
+ if (points.size() <= 2) {
+ return points;
}
- // public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
- // if (points == null || points.size() < 3) {
- // return points;
- // }
- //
- // // 找到距离最远的点
- // double maxDistance = 0;
- // int index = 0;
- // Point start = points.get(0);
- // Point end = points.get(points.size() - 1);
- //
- // for (int i = 1; i < points.size() - 1; i++) {
- // double distance = point_line_distance(points.get(i), start, end);
- // if (distance > maxDistance) {
- // maxDistance = distance;
- // index = i;
- // }
- // }
- //
- // List<Point> result = new ArrayList<>();
- // if (maxDistance > epsilon) {
- // List<Point> firstPart = reducePoints(points.subList(0, index + 1), epsilon);
- // List<Point> secondPart = reducePoints(points.subList(index, points.size()), epsilon);
- //
- // // 合并结果
- // result.addAll(firstPart.subList(0, firstPart.size() - 1)); // 避免重复添加中间点
- // result.addAll(secondPart);
- // } else {
- // // 如果最远距离小于阈值，则只保留起点和终点
- // result.add(start);
- // result.add(end);
- // }
- //
- // return result;
- // }
-
- /**
- * 使用迭代实现 RDP 算法。避免递归深度过大。
- *
- * @param points 点集
- * @param epsilon 简化阈值
- * @return 简化后的点集
- */
- public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
- if (points.size() <= 2) {
- return points;
+ // 使用栈模拟递归
+ Stack<int[]> stack = new Stack<>();
+ stack.push(new int[] {0, points.size() - 1});
+
+ // 用于标记需要保留的点
+ boolean[] keep = new boolean[points.size()];
+ keep[0] = true;
+ keep[points.size() - 1] = true;
+
+ int count = 0; // TODO debug
+
+ while (!stack.isEmpty()) {
+ int[] range = stack.pop();
+ int start = range[0];
+ int end = range[1];
+
+ // 找到距离最远的点
+ double maxDistance = 0;
+ int index = start;
+ Point startPoint = points.get(start);
+ Point endPoint = points.get(end);
+
+ for (int i = start + 1; i < end; i++) {
+ count++;
+ double distance = point_line_distance(points.get(i), startPoint, endPoint);
+ if (distance > maxDistance) {
+ maxDistance = distance;
+ index = i;
}
-
- // 使用栈模拟递归
- Stack<int[]> stack = new Stack<>();
- stack.push(new int[]{0, points.size() - 1});
-
- // 用于标记需要保留的点
- boolean[] keep = new boolean[points.size()];
- keep[0] = true;
- keep[points.size() - 1] = true;
-
- int count = 0; // TODO debug
-
- while (!stack.isEmpty()) {
- int[] range = stack.pop();
- int start = range[0];
- int end = range[1];
-
- // 找到距离最远的点
- double maxDistance = 0;
- int index = start;
- Point startPoint = points.get(start);
- Point endPoint = points.get(end);
-
- for (int i = start + 1; i < end; i++) {
- count++;
- double distance = point_line_distance(points.get(i), startPoint, endPoint);
- if (distance > maxDistance) {
- maxDistance = distance;
- index = i;
- }
- }
-
- // 如果最远距离大于阈值，则继续分割
- if (maxDistance > epsilon) {
- keep[index] = true; // 保留当前点
- stack.push(new int[]{start, index});
- stack.push(new int[]{index, end});
- }
- }
-
- // 构建简化后的点集
- List<Point> result = new ArrayList<>();
- for (int i = 0; i < points.size(); i++) {
- if (keep[i]) {
- result.add(points.get(i));
- }
- }
-
- System.out.println("count=" + count);
- return result;
+ }
+
+ // 如果最远距离大于阈值，则继续分割
+ if (maxDistance > epsilon) {
+ keep[index] = true; // 保留当前点
+ stack.push(new int[] {start, index});
+ stack.push(new int[] {index, end});
+ }
}
- public static Object[] normalizePoints(List<Point> points) {
- // 提取 x 和 y 值
- double[] xValues = points.stream().mapToDouble(p -> p.x).toArray();
- double[] yValues = points.stream().mapToDouble(p -> p.y).toArray();
+ // 构建简化后的点集
+ List<Point> result = new ArrayList<>();
+ for (int i = 0; i < points.size(); i++) {
+ if (keep[i]) {
+ result.add(points.get(i));
+ }
+ }
- // 计算 x 和 y 的最小值和最大值
- double xMin = Double.MAX_VALUE, xMax = Double.MIN_VALUE;
- double yMin = Double.MAX_VALUE, yMax = Double.MIN_VALUE;
+ System.out.println("count=" + count);
+ return result;
+ }
- for (double x : xValues) {
- if (x < xMin) xMin = x;
- if (x > xMax) xMax = x;
- }
- for (double y : yValues) {
- if (y < yMin) yMin = y;
- if (y > yMax) yMax = y;
- }
+ public static Object[] normalizePoints(List<Point> points) {
+ // 提取 x 和 y 值
+ double[] xValues = points.stream().mapToDouble(p -> p.x).toArray();
+ double[] yValues = points.stream().mapToDouble(p -> p.y).toArray();
- // 标准化
- List<Point> normalizedPoints = new ArrayList<>();
- for (Point p : points) {
- double xNormalized = (xMax != xMin) ? (p.x - xMin) / (xMax - xMin) : 0;
- double yNormalized = (yMax != yMin) ? (p.y - yMin) / (yMax - yMin) : 0;
- normalizedPoints.add(new Point(xNormalized, yNormalized));
- }
+ // 计算 x 和 y 的最小值和最大值
+ double xMin = Double.MAX_VALUE, xMax = Double.MIN_VALUE;
+ double yMin = Double.MAX_VALUE, yMax = Double.MIN_VALUE;
- return new Object[]{normalizedPoints, xMin, xMax, yMin, yMax};
+ for (double x : xValues) {
+ if (x < xMin) xMin = x;
+ if (x > xMax) xMax = x;
+ }
+ for (double y : yValues) {
+ if (y < yMin) yMin = y;
+ if (y > yMax) yMax = y;
}
- public static List<Point> denormalizePoints(List<Point> points, double[] originalScale) {
- double xMin = originalScale[0];
- double xMax = originalScale[1];
- double yMin = originalScale[2];
- double yMax = originalScale[3];
-
- List<Point> denormalizedPoints = new ArrayList<>();
- for (Point p : points) {
- double xOriginal = p.x * (xMax - xMin) + xMin;
- double yOriginal = p.y * (yMax - yMin) + yMin;
- denormalizedPoints.add(new Point(xOriginal, yOriginal));
- }
-
- return denormalizedPoints;
+ // 标准化
+ List<Point> normalizedPoints = new ArrayList<>();
+ for (Point p : points) {
+ double xNormalized = (xMax != xMin) ? (p.x - xMin) / (xMax - xMin) : 0;
+ double yNormalized = (yMax != yMin) ? (p.y - yMin) / (yMax - yMin) : 0;
+ normalizedPoints.add(new Point(xNormalized, yNormalized));
}
- public static void main(String[] args) throws IOException {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = true;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 10;
- List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- // Polyline polyline = new Polyline();
- // for (int i = 0; i < N; i += 1) {
- // double v = rand.nextInt(1000);
- // polyline.addVertex(new Point(i, v));
- // }
- // List<Point> points = polyline.getVertices();
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (Point point : points) {
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(points.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ return new Object[] {normalizedPoints, xMin, xMax, yMin, yMax};
+ }
- // 标准化
- Object[] normalizedData = normalizePoints(points);
- List<Point> normalizedPoints = (List<Point>) normalizedData[0];
- double xMin = (double) normalizedData[1];
- double xMax = (double) normalizedData[2];
- double yMin = (double) normalizedData[3];
- double yMax = (double) normalizedData[4];
-
- int m = 7;
- double tolerantPts = 0;
- double epsilon = Tool.getParam(normalizedPoints, m, Rdp::reducePoints, tolerantPts);
-// double epsilon = 0.001;
-// double epsilon = 0.004547272052729534;
- System.out.println("epsilon=" + epsilon);
-
- long startTime = System.currentTimeMillis();
- List<Point> sampled_tmp = reducePoints(normalizedPoints, epsilon);
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime - startTime) + "ms");
-
- // 还原到原始尺度
- double[] originalScale = {xMin, xMax, yMin, yMax};
- List<Point> sampled = denormalizePoints(sampled_tmp, originalScale);
-
- // for (Point p : sampled) {
- // System.out.println(p);
- // }
- System.out.println(sampled.size());
-
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < sampled.size(); i++) {
- writer.println(sampled.get(i).x + "," + sampled.get(i).y);
- }
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
+ public static List<Point> denormalizePoints(List<Point> points, double[] originalScale) {
+ double xMin = originalScale[0];
+ double xMax = originalScale[1];
+ double yMin = originalScale[2];
+ double yMax = originalScale[3];
+
+ List<Point> denormalizedPoints = new ArrayList<>();
+ for (Point p : points) {
+ double xOriginal = p.x * (xMax - xMin) + xMin;
+ double yOriginal = p.y * (yMax - yMin) + yMin;
+ denormalizedPoints.add(new Point(xOriginal, yOriginal));
}
- // public static void reducePoints(List<Point> points, double epsilon, int s, int e,
- // List<Point> resultList) {
- // double dmax = 0;
- // int index = 0;
- //
- // final int start = s;
- // final int end = e - 1;
- // for (int i = start + 1; i < end; i++) {
- // // Point
- // final double px = list.get(i)[0];
- // final double py = list.get(i)[1];
- // // Start
- // final double vx = list.get(start)[0];
- // final double vy = list.get(start)[1];
- // // End
- // final double wx = list.get(end)[0];
- // final double wy = list.get(end)[1];
- // final double d = perpendicularDistance(px, py, vx, vy, wx, wy);
- // if (d > dmax) {
- // index = i;
- // dmax = d;
- // }
+ return denormalizedPoints;
+ }
+
+ public static void main(String[] args) throws IOException {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 10;
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ // Polyline polyline = new Polyline();
+ // for (int i = 0; i < N; i += 1) {
+ // double v = rand.nextInt(1000);
+ // polyline.addVertex(new Point(i, v));
// }
- // // If max distance is greater than epsilon, recursively simplify
- // if (dmax > epsilon) {
- // // Recursive call
- // douglasPeucker(list, s, index, epsilon, resultList);
- // douglasPeucker(list, index, e, epsilon, resultList);
- // } else {
- // if ((end - start) > 0) {
- // resultList.add(list.get(start));
- // resultList.add(list.get(end));
- // } else {
- // resultList.add(list.get(start));
- // }
+ // List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ // 标准化
+ Object[] normalizedData = normalizePoints(points);
+ List<Point> normalizedPoints = (List<Point>) normalizedData[0];
+ double xMin = (double) normalizedData[1];
+ double xMax = (double) normalizedData[2];
+ double yMin = (double) normalizedData[3];
+ double yMax = (double) normalizedData[4];
+
+ int m = 7;
+ double tolerantPts = 0;
+ double epsilon = Tool.getParam(normalizedPoints, m, Rdp::reducePoints, tolerantPts);
+ // double epsilon = 0.001;
+ // double epsilon = 0.004547272052729534;
+ System.out.println("epsilon=" + epsilon);
+
+ long startTime = System.currentTimeMillis();
+ List<Point> sampled_tmp = reducePoints(normalizedPoints, epsilon);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ // 还原到原始尺度
+ double[] originalScale = {xMin, xMax, yMin, yMax};
+ List<Point> sampled = denormalizePoints(sampled_tmp, originalScale);
+
+ // for (Point p : sampled) {
+ // System.out.println(p);
// }
- // }
-
- // /**
- // * Given a curve composed of line segments find a similar curve with fewer points.
- // *
- // * @param list List of Double[] points (x,y)
- // * @param epsilon Distance dimension
- // * @return Similar curve with fewer points
- // */
- // public static final List<Double[]> douglasPeucker(List<Double[]> list, double epsilon) {
- // final List<Double[]> resultList = new ArrayList<Double[]>();
- // douglasPeucker(list, 0, list.size(), epsilon, resultList);
- // return resultList;
- // }
+ System.out.println(sampled.size());
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+
+ // public static void reducePoints(List<Point> points, double epsilon, int s, int e,
+ // List<Point> resultList) {
+ // double dmax = 0;
+ // int index = 0;
+ //
+ // final int start = s;
+ // final int end = e - 1;
+ // for (int i = start + 1; i < end; i++) {
+ // // Point
+ // final double px = list.get(i)[0];
+ // final double py = list.get(i)[1];
+ // // Start
+ // final double vx = list.get(start)[0];
+ // final double vy = list.get(start)[1];
+ // // End
+ // final double wx = list.get(end)[0];
+ // final double wy = list.get(end)[1];
+ // final double d = perpendicularDistance(px, py, vx, vy, wx, wy);
+ // if (d > dmax) {
+ // index = i;
+ // dmax = d;
+ // }
+ // }
+ // // If max distance is greater than epsilon, recursively simplify
+ // if (dmax > epsilon) {
+ // // Recursive call
+ // douglasPeucker(list, s, index, epsilon, resultList);
+ // douglasPeucker(list, index, e, epsilon, resultList);
+ // } else {
+ // if ((end - start) > 0) {
+ // resultList.add(list.get(start));
+ // resultList.add(list.get(end));
+ // } else {
+ // resultList.add(list.get(start));
+ // }
+ // }
+ // }
+
+ // /**
+ // * Given a curve composed of line segments find a similar curve with fewer points.
+ // *
+ // * @param list List of Double[] points (x,y)
+ // * @param epsilon Distance dimension
+ // * @return Similar curve with fewer points
+ // */
+ // public static final List<Double[]> douglasPeucker(List<Double[]> list, double epsilon) {
+ // final List<Double[]> resultList = new ArrayList<Double[]>();
+ // douglasPeucker(list, 0, list.size(), epsilon, resultList);
+ // return resultList;
+ // }
}
(iotdb) 33/44: rdp
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e6bd68ed6a2419bf2b0059d38d31343ae55082cd
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jan 28 07:28:38 2025 +0800
rdp
---
server/pom.xml | 6 +-
server/sample_minmax-jar-with-dependencies.jar | Bin 40761381 -> 40768753 bytes
...es.jar => sample_rdp-jar-with-dependencies.jar} | Bin 40761381 -> 40768959 bytes
.../java/org/apache/iotdb/db/query/eBUG/Rdp.java | 286 +++++++++++++++++++++
.../apache/iotdb/db/query/eBUG/sample_MinMax.java | 3 +-
.../eBUG/{sample_MinMax.java => sampled_Rdp.java} | 48 ++--
6 files changed, 318 insertions(+), 25 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index 1808c5f6791..998fcc8f41f 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -287,13 +287,15 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
- <finalName>sample_minmax</finalName>
+ <finalName>sample_rdp</finalName>
+ <!-- <finalName>sample_minmax</finalName>-->
<!-- <finalName>sample_fsw</finalName>-->
<!-- <finalName>sample_eBUG</finalName>-->
<!-- <finalName>sample_BUYdiff</finalName>-->
<archive>
<manifest>
- <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>
+ <mainClass>org.apache.iotdb.db.query.eBUG.sampled_Rdp</mainClass>
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>-->
diff --git a/server/sample_minmax-jar-with-dependencies.jar b/server/sample_minmax-jar-with-dependencies.jar
index cd09c6c3bbf..1cf9f21e3f1 100644
Binary files a/server/sample_minmax-jar-with-dependencies.jar and b/server/sample_minmax-jar-with-dependencies.jar differ
diff --git a/server/sample_minmax-jar-with-dependencies.jar b/server/sample_rdp-jar-with-dependencies.jar
similarity index 99%
copy from server/sample_minmax-jar-with-dependencies.jar
copy to server/sample_rdp-jar-with-dependencies.jar
index cd09c6c3bbf..4f69e1781d5 100644
Binary files a/server/sample_minmax-jar-with-dependencies.jar and b/server/sample_rdp-jar-with-dependencies.jar differ
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
new file mode 100644
index 00000000000..64cfc12e889
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Rdp.java
@@ -0,0 +1,286 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.Stack;
+
+public class Rdp {
+ // Ramer–Douglas–Peucker algorithm (RDP)
+ // top-down, for min perpendicular distance
+ // worst case complexity n2
+ // best case complexity nlogn
+
+ public static double point_line_distance(Point point, Point lineStart, Point lineEnd) {
+ // 计算点到直线的垂直正交距离
+ // (x0,y0)到直线（经过(x1,y1)、(x2,y2)点的直线）的正交距离
+ double x0 = point.x;
+ double y0 = point.y;
+ double x1 = lineStart.x;
+ double y1 = lineStart.y;
+ double x2 = lineEnd.x;
+ double y2 = lineEnd.y;
+
+ double numerator = Math.abs((x2 - x1) * (y1 - y0) - (x1 - x0) * (y2 - y1));
+ double denominator = Math.sqrt(Math.pow(x2 - x1, 2) + Math.pow(y2 - y1, 2));
+ return numerator / denominator;
+ }
+
+ // public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
+ // if (points == null || points.size() < 3) {
+ // return points;
+ // }
+ //
+ // // 找到距离最远的点
+ // double maxDistance = 0;
+ // int index = 0;
+ // Point start = points.get(0);
+ // Point end = points.get(points.size() - 1);
+ //
+ // for (int i = 1; i < points.size() - 1; i++) {
+ // double distance = point_line_distance(points.get(i), start, end);
+ // if (distance > maxDistance) {
+ // maxDistance = distance;
+ // index = i;
+ // }
+ // }
+ //
+ // List<Point> result = new ArrayList<>();
+ // if (maxDistance > epsilon) {
+ // List<Point> firstPart = reducePoints(points.subList(0, index + 1), epsilon);
+ // List<Point> secondPart = reducePoints(points.subList(index, points.size()), epsilon);
+ //
+ // // 合并结果
+ // result.addAll(firstPart.subList(0, firstPart.size() - 1)); // 避免重复添加中间点
+ // result.addAll(secondPart);
+ // } else {
+ // // 如果最远距离小于阈值，则只保留起点和终点
+ // result.add(start);
+ // result.add(end);
+ // }
+ //
+ // return result;
+ // }
+
+ /**
+ * 使用迭代实现 RDP 算法。避免递归深度过大。
+ *
+ * @param points 点集
+ * @param epsilon 简化阈值
+ * @return 简化后的点集
+ */
+ public static List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs) {
+ if (points.size() <= 2) {
+ return points;
+ }
+
+ // 使用栈模拟递归
+ Stack<int[]> stack = new Stack<>();
+ stack.push(new int[] {0, points.size() - 1});
+
+ // 用于标记需要保留的点
+ boolean[] keep = new boolean[points.size()];
+ keep[0] = true;
+ keep[points.size() - 1] = true;
+
+ while (!stack.isEmpty()) {
+ int[] range = stack.pop();
+ int start = range[0];
+ int end = range[1];
+
+ // 找到距离最远的点
+ double maxDistance = 0;
+ int index = start;
+ Point startPoint = points.get(start);
+ Point endPoint = points.get(end);
+
+ for (int i = start + 1; i < end; i++) {
+ double distance = point_line_distance(points.get(i), startPoint, endPoint);
+ if (distance > maxDistance) {
+ maxDistance = distance;
+ index = i;
+ }
+ }
+
+ // 如果最远距离大于阈值，则继续分割
+ if (maxDistance > epsilon) {
+ keep[index] = true; // 保留当前点
+ stack.push(new int[] {start, index});
+ stack.push(new int[] {index, end});
+ }
+ }
+
+ // 构建简化后的点集
+ List<Point> result = new ArrayList<>();
+ for (int i = 0; i < points.size(); i++) {
+ if (keep[i]) {
+ result.add(points.get(i));
+ }
+ }
+
+ return result;
+ }
+
+ public static Object[] normalizePoints(List<Point> points) {
+ // 提取 x 和 y 值
+ double[] xValues = points.stream().mapToDouble(p -> p.x).toArray();
+ double[] yValues = points.stream().mapToDouble(p -> p.y).toArray();
+
+ // 计算 x 和 y 的最小值和最大值
+ double xMin = Double.MAX_VALUE, xMax = Double.MIN_VALUE;
+ double yMin = Double.MAX_VALUE, yMax = Double.MIN_VALUE;
+
+ for (double x : xValues) {
+ if (x < xMin) xMin = x;
+ if (x > xMax) xMax = x;
+ }
+ for (double y : yValues) {
+ if (y < yMin) yMin = y;
+ if (y > yMax) yMax = y;
+ }
+
+ // 标准化
+ List<Point> normalizedPoints = new ArrayList<>();
+ for (Point p : points) {
+ double xNormalized = (xMax != xMin) ? (p.x - xMin) / (xMax - xMin) : 0;
+ double yNormalized = (yMax != yMin) ? (p.y - yMin) / (yMax - yMin) : 0;
+ normalizedPoints.add(new Point(xNormalized, yNormalized));
+ }
+
+ return new Object[] {normalizedPoints, xMin, xMax, yMin, yMax};
+ }
+
+ public static List<Point> denormalizePoints(List<Point> points, double[] originalScale) {
+ double xMin = originalScale[0];
+ double xMax = originalScale[1];
+ double yMin = originalScale[2];
+ double yMax = originalScale[3];
+
+ List<Point> denormalizedPoints = new ArrayList<>();
+ for (Point p : points) {
+ double xOriginal = p.x * (xMax - xMin) + xMin;
+ double yOriginal = p.y * (yMax - yMin) + yMin;
+ denormalizedPoints.add(new Point(xOriginal, yOriginal));
+ }
+
+ return denormalizedPoints;
+ }
+
+ public static void main(String[] args) throws IOException {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 1000_0000;
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ // Polyline polyline = new Polyline();
+ // for (int i = 0; i < N; i += 1) {
+ // double v = rand.nextInt(1000);
+ // polyline.addVertex(new Point(i, v));
+ // }
+ // List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ // 标准化
+ Object[] normalizedData = normalizePoints(points);
+ List<Point> normalizedPoints = (List<Point>) normalizedData[0];
+ double xMin = (double) normalizedData[1];
+ double xMax = (double) normalizedData[2];
+ double yMin = (double) normalizedData[3];
+ double yMax = (double) normalizedData[4];
+
+ int m = 10;
+ double tolerantPts = 0;
+ double epsilon = Tool.getParam(normalizedPoints, m, Rdp::reducePoints, tolerantPts);
+ // double epsilon = 0.001;
+
+ long startTime = System.currentTimeMillis();
+ List<Point> sampled_tmp = reducePoints(normalizedPoints, epsilon);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ // 还原到原始尺度
+ double[] originalScale = {xMin, xMax, yMin, yMax};
+ List<Point> sampled = denormalizePoints(sampled_tmp, originalScale);
+
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ System.out.println(sampled.size());
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+
+ // public static void reducePoints(List<Point> points, double epsilon, int s, int e,
+ // List<Point> resultList) {
+ // double dmax = 0;
+ // int index = 0;
+ //
+ // final int start = s;
+ // final int end = e - 1;
+ // for (int i = start + 1; i < end; i++) {
+ // // Point
+ // final double px = list.get(i)[0];
+ // final double py = list.get(i)[1];
+ // // Start
+ // final double vx = list.get(start)[0];
+ // final double vy = list.get(start)[1];
+ // // End
+ // final double wx = list.get(end)[0];
+ // final double wy = list.get(end)[1];
+ // final double d = perpendicularDistance(px, py, vx, vy, wx, wy);
+ // if (d > dmax) {
+ // index = i;
+ // dmax = d;
+ // }
+ // }
+ // // If max distance is greater than epsilon, recursively simplify
+ // if (dmax > epsilon) {
+ // // Recursive call
+ // douglasPeucker(list, s, index, epsilon, resultList);
+ // douglasPeucker(list, index, e, epsilon, resultList);
+ // } else {
+ // if ((end - start) > 0) {
+ // resultList.add(list.get(start));
+ // resultList.add(list.get(end));
+ // } else {
+ // resultList.add(list.get(start));
+ // }
+ // }
+ // }
+
+ // /**
+ // * Given a curve composed of line segments find a similar curve with fewer points.
+ // *
+ // * @param list List of Double[] points (x,y)
+ // * @param epsilon Distance dimension
+ // * @return Similar curve with fewer points
+ // */
+ // public static final List<Double[]> douglasPeucker(List<Double[]> list, double epsilon) {
+ // final List<Double[]> resultList = new ArrayList<Double[]>();
+ // douglasPeucker(list, 0, list.size(), epsilon, resultList);
+ // return resultList;
+ // }
+
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_MinMax.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_MinMax.java
index 767b7f7151e..5d841a62e87 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_MinMax.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_MinMax.java
@@ -49,7 +49,8 @@ public class sample_MinMax {
List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
String outputFile =
- generateOutputFileName(input, outDir, "-" + bucketType + "-n" + points.size() + "-m" + m);
+ generateOutputFileName(
+ input, outDir, "-MinMax-" + bucketType + "-n" + points.size() + "-m" + m);
System.out.println("Output file: " + outputFile); // do not modify this hint string log
// 采样
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_MinMax.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sampled_Rdp.java
similarity index 60%
copy from server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_MinMax.java
copy to server/src/main/java/org/apache/iotdb/db/query/eBUG/sampled_Rdp.java
index 767b7f7151e..834f84d09b7 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_MinMax.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sampled_Rdp.java
@@ -5,13 +5,16 @@ import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
+import static org.apache.iotdb.db.query.eBUG.Rdp.denormalizePoints;
+import static org.apache.iotdb.db.query.eBUG.Rdp.normalizePoints;
import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
+import static org.apache.iotdb.db.query.eBUG.Tool.getParam;
-public class sample_MinMax {
+public class sampled_Rdp {
public static void main(String[] args) throws IOException {
if (args.length < 7) {
System.out.println(
- "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,bucketType,(outDir)");
+ "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,tolerantRatio,(outDir)");
}
String input = args[0];
boolean hasHeader = Boolean.parseBoolean(args[1]);
@@ -19,15 +22,7 @@ public class sample_MinMax {
int valueIdx = Integer.parseInt(args[3]);
int N = Integer.parseInt(args[4]); // N<=0表示读全部行，N>0表示读最多N行
int m = Integer.parseInt(args[5]);
- String bucketTypeStr = args[6];
- MinMax.fixedBUCKETtype bucketType;
- if (bucketTypeStr.equals("width")) {
- bucketType = MinMax.fixedBUCKETtype.width;
- } else if (bucketTypeStr.equals("frequency")) {
- bucketType = MinMax.fixedBUCKETtype.frequency;
- } else {
- throw new IOException("please input fixed bucket type as width/frequency");
- }
+ double tolerantRatio = Double.parseDouble(args[6]); // 查找参数epsilon以使得输出采样点数接近m的程度，比如0.01
String outDir;
if (args.length > 7) {
outDir = args[7]; // 表示输出文件保存到指定的文件夹
@@ -42,33 +37,42 @@ public class sample_MinMax {
System.out.println("Value index: " + valueIdx);
System.out.println("N: " + N);
System.out.println("m: " + m);
- System.out.println("bucketType: " + bucketType);
+ System.out.println("tolerantRatio: " + tolerantRatio);
System.out.println("outDir: " + outDir);
// 读取原始序列
List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
-
String outputFile =
- generateOutputFileName(input, outDir, "-" + bucketType + "-n" + points.size() + "-m" + m);
- System.out.println("Output file: " + outputFile); // do not modify this hint string log
+ generateOutputFileName(input, outDir, "-Rdp" + "-n" + points.size() + "-m" + m);
+ System.out.println("Output file: " + outputFile);
+
+ // 标准化，避免非常大的时间戳主导perpendicular distance
+ Object[] normalizedData = normalizePoints(points);
+ List<Point> normalizedPoints = (List<Point>) normalizedData[0];
+ double xMin = (double) normalizedData[1];
+ double xMax = (double) normalizedData[2];
+ double yMin = (double) normalizedData[3];
+ double yMax = (double) normalizedData[4];
+
+ // 查找epsilon参数使得采样点数接近m
+ double epsilon = getParam(normalizedPoints, m, Rdp::reducePoints, m * tolerantRatio);
// 采样
- List<Point> results;
long startTime = System.currentTimeMillis();
- if (bucketType == MinMax.fixedBUCKETtype.width) {
- results = MinMax.reducePoints_equalWidthBucket(points, m, false);
- } else {
- results = MinMax.reducePoints_equalFrequencyBucket(points, m, false);
- }
+ List<Point> results_tmp = Rdp.reducePoints(normalizedPoints, epsilon);
long endTime = System.currentTimeMillis();
+ // 还原到原始尺度
+ double[] originalScale = {xMin, xMax, yMin, yMax};
+ List<Point> results = denormalizePoints(results_tmp, originalScale);
+
System.out.println("result point number: " + results.size());
System.out.println(
"n="
+ points.size()
+ ", m="
+ m
- + ", Time taken to reduce points: " // do not modify this hint string log
+ + ", Time taken to reduce points: "
+ (endTime - startTime)
+ "ms");
(iotdb) 40/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dad9adc040abc97fd5016da155ba7d794e3b1b79
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Feb 3 21:33:07 2025 +0800
add
---
server/pom.xml | 74 +++++++++++-----------
.../apache/iotdb/db/query/eBUG/sample_eBUG.java | 17 +++--
2 files changed, 47 insertions(+), 44 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index 1688dc9e4c9..73af108a647 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -284,43 +284,43 @@
</execution>
</executions>
</plugin>
- <!-- <plugin>-->
- <!-- <artifactId>maven-assembly-plugin</artifactId>-->
- <!-- <configuration>-->
- <!-- <finalName>sample_ltd</finalName>-->
- <!-- <!– <finalName>sample_swab_ad</finalName>–>-->
- <!-- <!– <finalName>sample_swab</finalName>–>-->
- <!-- <!– <finalName>sample_rdp</finalName>–>-->
- <!-- <!– <finalName>sample_minmax</finalName>–>-->
- <!-- <!– <finalName>sample_fsw</finalName>–>-->
- <!-- <!– <finalName>sample_eBUG</finalName>–>-->
- <!-- <!– <finalName>sample_BUYdiff</finalName>–>-->
- <!-- <archive>-->
- <!-- <manifest>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_LTD</mainClass>-->
- <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWABAD</mainClass>–>-->
- <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>–>-->
- <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>–>-->
- <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>–>-->
- <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>–>-->
- <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>–>-->
- <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>–>-->
- <!-- </manifest>-->
- <!-- </archive>-->
- <!-- <descriptorRefs>-->
- <!-- <descriptorRef>jar-with-dependencies</descriptorRef>-->
- <!-- </descriptorRefs>-->
- <!-- </configuration>-->
- <!-- <executions>-->
- <!-- <execution>-->
- <!-- <id>make-assembly</id>-->
- <!-- <phase>package</phase>-->
- <!-- <goals>-->
- <!-- <goal>single</goal>-->
- <!-- </goals>-->
- <!-- </execution>-->
- <!-- </executions>-->
- <!-- </plugin>-->
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <!-- <finalName>sample_ltd</finalName>-->
+ <!-- <finalName>sample_swab_ad</finalName>-->
+ <!-- <finalName>sample_swab</finalName>-->
+ <!-- <finalName>sample_rdp</finalName>-->
+ <!-- <finalName>sample_minmax</finalName>-->
+ <!-- <finalName>sample_fsw</finalName>-->
+ <finalName>sample_eBUG</finalName>
+ <!-- <finalName>sample_BUYdiff</finalName>-->
+ <archive>
+ <manifest>
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_LTD</mainClass>-->
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWABAD</mainClass>-->
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>-->
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>-->
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>-->
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
+ <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>-->
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<profiles>
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java
index d8a4b5ea198..9e8033bded3 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java
@@ -10,8 +10,8 @@ import static org.apache.iotdb.db.query.eBUG.eBUG.buildEffectiveArea;
public class sample_eBUG {
// 输入一条时间序列 t,v
- // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
- // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
+ // 输出按照bottom-up淘汰顺序倒序排列的dominated significance,t,v。
+ // 用于后期在线采样时选取前m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
public static void main(String[] args) {
if (args.length < 7) {
System.out.println(
@@ -70,15 +70,18 @@ public class sample_eBUG {
// 写入表头
if (m <= 2) {
System.out.println(
- "precomputation mode, outputting (z,x,y) ordered by z in ascending order");
- writer.write("z,x,y");
+ "precomputation mode, outputting (id,x,y,z) ordered by z in descending order");
+ writer.write("id,x,y,z");
writer.newLine();
- // 写入数据行，按顺序 z, x, y
- // 按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
+ // 写入数据行，
+ // 按照idx,x,y,z四列，results结果已经按照z（即DS）递减排序，对应bottom-up的淘汰顺序的倒序，越大代表越晚被淘汰
+ // idx是从1开始的顺序编号
+ int idx = 1;
for (Point point : results) {
- writer.write(point.z + "," + point.x + "," + point.y);
+ writer.write(idx + "," + point.x + "," + point.y + "," + point.z);
writer.newLine();
+ idx++;
}
} else {
System.out.println(
(iotdb) 39/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 24d39c386f08dd27485461763df4b2c6f7881862
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Feb 3 21:15:24 2025 +0800
add
---
server/pom.xml | 74 ++++++------
.../resources/conf/iotdb-engine.properties | 16 +--
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 5 +
.../db/query/udf/builtin/BuiltinFunction.java | 3 +-
.../iotdb/db/query/udf/builtin/UDTFEBUG.java | 126 +++++++++++++++++++
.../apache/iotdb/db/integration/eBUG/MyTest.java | 133 +++++++++++++++++++++
6 files changed, 311 insertions(+), 46 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index efa727e5732..1688dc9e4c9 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -284,43 +284,43 @@
</execution>
</executions>
</plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <finalName>sample_ltd</finalName>
- <!-- <finalName>sample_swab_ad</finalName>-->
- <!-- <finalName>sample_swab</finalName>-->
- <!-- <finalName>sample_rdp</finalName>-->
- <!-- <finalName>sample_minmax</finalName>-->
- <!-- <finalName>sample_fsw</finalName>-->
- <!-- <finalName>sample_eBUG</finalName>-->
- <!-- <finalName>sample_BUYdiff</finalName>-->
- <archive>
- <manifest>
- <mainClass>org.apache.iotdb.db.query.eBUG.sample_LTD</mainClass>
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWABAD</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>-->
- </manifest>
- </archive>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
+ <!-- <plugin>-->
+ <!-- <artifactId>maven-assembly-plugin</artifactId>-->
+ <!-- <configuration>-->
+ <!-- <finalName>sample_ltd</finalName>-->
+ <!-- <!– <finalName>sample_swab_ad</finalName>–>-->
+ <!-- <!– <finalName>sample_swab</finalName>–>-->
+ <!-- <!– <finalName>sample_rdp</finalName>–>-->
+ <!-- <!– <finalName>sample_minmax</finalName>–>-->
+ <!-- <!– <finalName>sample_fsw</finalName>–>-->
+ <!-- <!– <finalName>sample_eBUG</finalName>–>-->
+ <!-- <!– <finalName>sample_BUYdiff</finalName>–>-->
+ <!-- <archive>-->
+ <!-- <manifest>-->
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_LTD</mainClass>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWABAD</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>–>-->
+ <!-- </manifest>-->
+ <!-- </archive>-->
+ <!-- <descriptorRefs>-->
+ <!-- <descriptorRef>jar-with-dependencies</descriptorRef>-->
+ <!-- </descriptorRefs>-->
+ <!-- </configuration>-->
+ <!-- <executions>-->
+ <!-- <execution>-->
+ <!-- <id>make-assembly</id>-->
+ <!-- <phase>package</phase>-->
+ <!-- <goals>-->
+ <!-- <goal>single</goal>-->
+ <!-- </goals>-->
+ <!-- </execution>-->
+ <!-- </executions>-->
+ <!-- </plugin>-->
</plugins>
</build>
<profiles>
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index d68392e0e9d..f89417c5430 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -38,10 +38,10 @@ pnv=100
#for ILTS
numIterations=4
-acc_avg=true
-acc_rectangle=true
-acc_convex=true
-acc_iterRepeat=true
+acc_avg=false
+acc_rectangle=false
+acc_convex=false
+acc_iterRepeat=false
####################
### enable CPV
@@ -275,7 +275,7 @@ seq_tsfile_size=1073741824
# close_tsfile_check_interval_in_ms=600000
# When the average point number of timeseries in memtable exceeds this, the memtable is flushed to disk. The default threshold is 10000.
-avg_series_point_number_threshold=100
+avg_series_point_number_threshold=10000
# How many threads can concurrently flush. When <= 0, use CPU core number.
# concurrent_flush_thread=0
@@ -632,17 +632,17 @@ enable_unseq_compaction=false
### Configurations for tsfile-format
####################
-write_convex_hull=true
+write_convex_hull=false
enableMinMaxLSM=false
use_Statistics=true
-use_TimeIndex=true
+use_TimeIndex=false
use_Mad=false
-use_ValueIndex=true
+use_ValueIndex=false
error_Param=1
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 44fc3298d96..86d9abaccde 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -353,6 +353,11 @@ public class eBUG {
// return results; // 注意这就是lineToSimplify.getVertices()
resultsBottomUpEliminated.add(points.get(0)); // 全局首点
resultsBottomUpEliminated.add(points.get(points.size() - 1)); // 全局尾点
+
+ // make them ordered by dominated sig in descending order (i.e., reverse of bottom-up
+ // elimination order)
+ Collections.reverse(resultsBottomUpEliminated); // O(n)
+
return resultsBottomUpEliminated;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java
index 5a9639faaf4..cc51a486e56 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java
@@ -50,7 +50,8 @@ public enum BuiltinFunction {
M4("M4", UDTFM4MAC.class),
M4_TW("M4_TW", UDTFM4.class),
Sample("SAMPLE", UDTFSample.class),
- MinMax("MINMAX", UDTFMinMax.class);
+ MinMax("MINMAX", UDTFMinMax.class),
+ EBUG("EBUG", UDTFEBUG.class),
;
private final String functionName;
diff --git a/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFEBUG.java b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFEBUG.java
new file mode 100644
index 00000000000..011ae9a7655
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/UDTFEBUG.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.query.udf.builtin;
+
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.query.eBUG.Point;
+import org.apache.iotdb.db.query.eBUG.eBUG;
+import org.apache.iotdb.db.query.udf.api.UDTF;
+import org.apache.iotdb.db.query.udf.api.access.Row;
+import org.apache.iotdb.db.query.udf.api.collector.PointCollector;
+import org.apache.iotdb.db.query.udf.api.customizer.config.UDTFConfigurations;
+import org.apache.iotdb.db.query.udf.api.customizer.parameter.UDFParameterValidator;
+import org.apache.iotdb.db.query.udf.api.customizer.parameter.UDFParameters;
+import org.apache.iotdb.db.query.udf.api.customizer.strategy.RowByRowAccessStrategy;
+import org.apache.iotdb.db.query.udf.api.exception.UDFException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class UDTFEBUG implements UDTF {
+
+ protected TSDataType dataType;
+
+ private int m; // target number of sampled points
+
+ private int e;
+
+ private List<Point> points;
+
+ @Override
+ public void validate(UDFParameterValidator validator) throws UDFException {
+ validator
+ .validateInputSeriesNumber(1)
+ .validateInputSeriesDataType(
+ 0, TSDataType.INT32, TSDataType.INT64, TSDataType.FLOAT, TSDataType.DOUBLE)
+ .validateRequiredAttribute("m");
+ }
+
+ @Override
+ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations)
+ throws MetadataException {
+ dataType = parameters.getDataType(0);
+ m = parameters.getInt("m"); // m>2 for online sampling mode, <=2 for precomputation mode
+ // if (m <= 2) {
+ // throw new MetadataException("m should be larger than 2 for online sampling mode");
+ // }
+ e = parameters.getInt("e");
+ if (e < 0) {
+ throw new MetadataException("e should not be smaller than 0");
+ }
+
+ if (m <= 2) { // precomputation mode
+ configurations
+ .setAccessStrategy(new RowByRowAccessStrategy())
+ .setOutputDataType(TSDataType.TEXT);
+ } else { // online sampling mode
+ configurations
+ .setAccessStrategy(new RowByRowAccessStrategy())
+ .setOutputDataType(TSDataType.DOUBLE);
+ }
+ points = new ArrayList<>();
+ }
+
+ @Override
+ public void transform(Row row, PointCollector collector)
+ throws QueryProcessException, IOException {
+ double t = row.getTime();
+ switch (dataType) {
+ case INT32:
+ points.add(new Point(t, row.getInt(0)));
+ break;
+ case INT64:
+ points.add(new Point(t, row.getLong(0)));
+ break;
+ case FLOAT:
+ points.add(new Point(t, row.getFloat(0)));
+ break;
+ case DOUBLE:
+ points.add(new Point(t, row.getDouble(0)));
+ break;
+ default:
+ break;
+ }
+ }
+
+ @Override
+ public void terminate(PointCollector collector) throws IOException, QueryProcessException {
+ List<Point> sampled = eBUG.buildEffectiveArea(points, e, false, m);
+
+ // collect result
+ if (m <= 2) {
+ // offline precomputation mode, for precomputing the dominated significance of each, ordered
+ // by dominated sig in descending order (i.e., reverse of bottom-up elimination order)
+ for (int i = 0; i < sampled.size(); i++) {
+ collector.putString(i + 1, sampled.get(i).toString());
+ // not using dominated significance z as timestamp,
+ // because dominated significance z can be double, if converted to long may lose some order
+ // information
+ }
+ } else {
+ // m>2, online sampling mode, order by time in ascending order
+ for (Point p : sampled) {
+ collector.putDouble((long) p.getTimestamp(), p.getValue());
+ }
+ }
+ }
+}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/eBUG/MyTest.java b/server/src/test/java/org/apache/iotdb/db/integration/eBUG/MyTest.java
new file mode 100644
index 00000000000..e276ce47143
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/integration/eBUG/MyTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.integration.eBUG;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.jdbc.Config;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Locale;
+
+import static org.junit.Assert.fail;
+
+public class MyTest {
+
+ private static final String TIMESTAMP_STR = "Time";
+
+ private static String[] creationSqls =
+ new String[] {
+ "SET STORAGE GROUP TO root.vehicle.d0",
+ "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=DOUBLE,ENCODING=PLAIN",
+ };
+
+ private final String d0s0 = "root.vehicle.d0.s0";
+
+ private static final String insertTemplate =
+ "INSERT INTO root.vehicle.d0(timestamp,s0)" + " VALUES(%d,%f)";
+
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private static boolean originalEnableCPV;
+ private static CompactionStrategy originalCompactionStrategy;
+ private static boolean originalUseChunkIndex;
+
+ @Before
+ public void setUp() throws Exception {
+ TSFileDescriptor.getInstance().getConfig().setTimeEncoder("PLAIN");
+ originalCompactionStrategy = config.getCompactionStrategy();
+ config.setCompactionStrategy(CompactionStrategy.NO_COMPACTION);
+
+ EnvironmentUtils.envSetUp();
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ // config.setTimestampPrecision("ms");
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ EnvironmentUtils.cleanEnv();
+ config.setCompactionStrategy(originalCompactionStrategy);
+ }
+
+ @Test
+ public void test1() {
+ prepareData1();
+
+ // String[] res =
+ // new String[]{
+ // "0,1,20,5.0,20.0,5.0[1],30.0[10]",
+ // "25,25,45,8.0,30.0,8.0[25],40.0[30]",
+ // "50,52,54,8.0,18.0,8.0[52],18.0[54]",
+ // "75,null,null,null,null,null,null"
+ // };
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ boolean hasResultSet =
+ statement.execute("SELECT EBUG(s0,'m'='10','e'='0') FROM root.vehicle.d0");
+
+ Assert.assertTrue(hasResultSet);
+ try (ResultSet resultSet = statement.getResultSet()) {
+ int i = 0;
+ while (resultSet.next()) {
+ String ans = resultSet.getString(1) + "," + resultSet.getString(2);
+ System.out.println(ans);
+ // Assert.assertEquals(res[i++], ans);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private static void prepareData1() {
+ try (Connection connection =
+ DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+
+ for (String sql : creationSqls) {
+ statement.execute(sql);
+ }
+
+ for (int i = 0; i < 1000; i++) {
+ double x = 2 * Math.PI * i / 1000.0;
+ double sineValue = Math.sin(x);
+ statement.execute(String.format(Locale.ENGLISH, insertTemplate, i, sineValue));
+ }
+
+ statement.execute("FLUSH");
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
(iotdb) 26/44: swab"
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ec0a3cebce1cd4b23f50066dd21bd73fc801f279
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jan 27 00:47:47 2025 +0800
swab"
---
.../java/org/apache/iotdb/db/query/eBUG/SWAB.java | 311 +++++++++------------
.../java/org/apache/iotdb/db/query/eBUG/Test7.java | 67 +++++
.../java/org/apache/iotdb/db/query/eBUG/Test8.java | 65 +++++
3 files changed, 260 insertions(+), 183 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
index 574f59e3bf4..31b2f02ea86 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB.java
@@ -1,6 +1,5 @@
package org.apache.iotdb.db.query.eBUG;
-import java.io.*;
import java.util.*;
@@ -82,12 +81,7 @@ public class SWAB {
}
// 计算L2误差，注意归一化还原
- return sumY2 * baseY * baseY
- + k * k * sumX2 * baseX * baseX
- + 2 * k * b * sumX * baseX
- - 2 * b * sumY * baseY
- - 2 * k * sumXY * baseX * baseY
- + b * b * (rx - lx + 1);
+ return sumY2 * baseY * baseY + k * k * sumX2 * baseX * baseX + 2 * k * b * sumX * baseX - 2 * b * sumY * baseY - 2 * k * sumXY * baseX * baseY + b * b * (rx - lx + 1);
}
public static double myLA_withTimestamps(List<Point> points, int lx, int rx) {
@@ -110,8 +104,7 @@ public class SWAB {
return tmp;
}
- public static List<Point> seg_bottomUp_maxerror_withTimestamps(
- List<Point> points, double maxError, Object[] prefixSum, boolean debug) {
+ public static List<Point> seg_bottomUp_maxerror_withTimestamps(List<Point> points, double maxError, Object[] prefixSum, boolean debug) {
int total = points.size();
if (total < 3) {
return points; // 不足 3 个点无法形成三角形
@@ -151,8 +144,7 @@ public class SWAB {
}
// 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap =
- new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)?
int fakeCnt = 0;
@@ -166,10 +158,7 @@ public class SWAB {
if (tri.isDeleted) {
fakeCnt--; // 取出了一个被标记删除点
if (debug) {
- System.out.println(
- ">>>bottom-up, remaining "
- + triangleHeap.size()
- + " triangles (including those marked for removal)");
+ System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
}
continue;
}
@@ -199,17 +188,13 @@ public class SWAB {
double mc;
if (prefixSum == null) {
- mc = myLA_withTimestamps(points,
- tri.prev.indices[0], tri.prev.indices[2]);
+ mc = myLA_withTimestamps(points, tri.prev.indices[0], tri.prev.indices[2]);
} else {
- mc = myLA_withTimestamps(points,
- prefixSum, tri.prev.indices[0], tri.prev.indices[2]);
+ mc = myLA_withTimestamps(points, prefixSum, tri.prev.indices[0], tri.prev.indices[2]);
}
tri.prev.area = mc;
if (debug) {
- System.out.println("(2) updating point on the left "
- + points.get(tri.prev.indices[1])
- + ", ranging [" + tri.prev.indices[0] + "," + tri.prev.indices[2] + "]");
+ System.out.println("(2) updating point on the left " + points.get(tri.prev.indices[1]) + ", ranging [" + tri.prev.indices[0] + "," + tri.prev.indices[2] + "]");
}
// 重新加入堆
@@ -239,17 +224,13 @@ public class SWAB {
double mc;
if (prefixSum == null) {
- mc = myLA_withTimestamps(points,
- tri.next.indices[0], tri.next.indices[2]);
+ mc = myLA_withTimestamps(points, tri.next.indices[0], tri.next.indices[2]);
} else {
- mc = myLA_withTimestamps(points,
- prefixSum, tri.next.indices[0], tri.next.indices[2]);
+ mc = myLA_withTimestamps(points, prefixSum, tri.next.indices[0], tri.next.indices[2]);
}
tri.next.area = mc;
if (debug) {
- System.out.println("(3) updating point on the right "
- + points.get(tri.next.indices[1])
- + ", ranging [" + tri.next.indices[0] + "," + tri.next.indices[2] + "]");
+ System.out.println("(3) updating point on the right " + points.get(tri.next.indices[1]) + ", ranging [" + tri.next.indices[0] + "," + tri.next.indices[2] + "]");
}
// 重新加入堆
@@ -259,10 +240,7 @@ public class SWAB {
fakeCnt++; // 表示heap里多了一个被标记删除的假点
}
if (debug) {
- System.out.println(
- ">>>bottom-up, remaining "
- + triangleHeap.size()
- + " triangles (including those marked for removal)");
+ System.out.println(">>>bottom-up, remaining " + triangleHeap.size() + " triangles (including those marked for removal)");
}
}
@@ -294,8 +272,18 @@ public class SWAB {
return i - 1; // 从0开始，并且是从0到i-1的左闭右闭
}
-
public static List<Point> swab_framework(List<Point> points, double maxError, int m, boolean debug) {
+ return swab_framework(points, maxError, m, null, debug);
+ }
+
+ public static List<Point> swab_framework(List<Point> points, double maxError, int m, Object[] prefixSum, boolean debug) {
+ if (debug) {
+ if (prefixSum != null) {
+ // deprecated
+ System.out.println("enable prefix sum for accelerating L2 error computation");
+ }
+ }
+
// 在这里给Point添加全局排序idx, 用于全局定位
for (int i = 0; i < points.size(); i++) {
points.get(i).index = i;
@@ -307,139 +295,125 @@ public class SWAB {
// m在这里的含义是分段数，虽然SWAB不能直接控制分段数，但是也需要这个参数来确定buffer大小
int bs = (int) Math.floor((double) points.size() / m * 6); // buffer size
int lb = bs / 2, ub = bs * 2;
+ if (lb == 0) {
+ lb += 2; // lb最小为2，否则陷入循环，因为joint buffer里至少会留一个点
+ ub += 2;
+ }
if (debug) {
System.out.println("bs=" + bs + ", lb=" + lb + ", ub=" + ub);
}
int buffer_startIdx = 0; // 左闭
- int buffer_endIdx = bs; // 右开。
- // 注意虽然joint，但是buffer和remaining points是断开的
- // buffer: [0:bs] 左闭右开
+ int buffer_endIdx = bs; // 右开
+ // 注意虽然joint，但是buffer和remaining points是buffer_endIdx处是不重叠的
+ // buffer: [0:bs) 左闭右开
// remaining: [bs:] 左闭
-// List<Point> w = points.subList(0, bs); // 浅复制 左闭右开
-// List<Point> remaining_points = points.subList(bs, points.size()); // TODO
-
- List<Point> segTs = new ArrayList<>(); // TODO
+ List<Point> segTs = new ArrayList<>();
segTs.add(points.get(0)); //全局首点
-// int base = 0; // global base
-// if (debug) {
-// System.out.println("base=" + base);
-// }
boolean needBottomUp = true; // false when only output, no input
- int reuseIdx = 0; // TODO
- List<Point> sampled = Collections.emptyList(); // TODO
+ int reuseIdx = 0; // 注意这里当needBottomUp=false的时候是直接取用上一轮的bottomup points但是base从reuseIdx开始
+ List<Point> sampled = Collections.emptyList();
while (true) {
if (debug) {
System.out.println("#####################################################");
- System.out.println("buffer data=" + (buffer_endIdx - buffer_startIdx)
- + ", remaining data=" + (points.size() - buffer_endIdx));
+ System.out.println("buffer data: [" + buffer_startIdx + "," + buffer_endIdx + ")" + ", remaining data: [" + buffer_endIdx + "," + points.size() + ")");
System.out.println("reBottomUp:" + needBottomUp);
}
if (needBottomUp) {
-
if (debug) {
System.out.println(">>>>bottom up on the buffer");
}
- reuseIdx = 0;
- sampled = seg_bottomUp_maxerror_withTimestamps(
- points.subList(buffer_startIdx, buffer_endIdx),
- maxError, null, false);
- // TODO prefix sum acc
+ reuseIdx = 0; // 重置
+ if (prefixSum == null) {
+ sampled = seg_bottomUp_maxerror_withTimestamps(points.subList(buffer_startIdx, buffer_endIdx),
+ maxError, null, false);
+ } else {
+ // deprecated
+ double[] prefixSumX = Arrays.copyOfRange((double[]) prefixSum[0], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumY = Arrays.copyOfRange((double[]) prefixSum[1], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumX2 = Arrays.copyOfRange((double[]) prefixSum[2], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumY2 = Arrays.copyOfRange((double[]) prefixSum[3], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumXY = Arrays.copyOfRange((double[]) prefixSum[4], buffer_startIdx, buffer_endIdx);
+ if (buffer_startIdx > 0) {
+ for (int i = 0; i < prefixSumX.length; i++) {
+ prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY.length; i++) {
+ prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumX2.length; i++) {
+ prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY2.length; i++) {
+ prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumXY.length; i++) {
+ prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_startIdx - 1];
+ }
+ }
+ Object[] prefixSumPart = new Object[]{prefixSumX, prefixSumY, prefixSumX2, prefixSumY2, prefixSumXY,
+ prefixSum[5], prefixSum[6]};
+ sampled = seg_bottomUp_maxerror_withTimestamps(points.subList(buffer_startIdx, buffer_endIdx),
+ maxError, prefixSumPart, false);
+ // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
+ }
// TODO eBUG also
// sampled = eBUG.buildEffectiveArea(points.subList(buffer_startIdx, buffer_endIdx), 0, false, m);
}
+ if (debug) {
+ System.out.println("number of bottomUp points=" + (sampled.size() - reuseIdx));
+ System.out.println("---------------------------------");
+ }
+
if (sampled.size() - reuseIdx >= 2) { // 左边输出一个segment
if (debug) {
System.out.println(">>>>left output a segment:["
- + sampled.get(0 + reuseIdx).index + "," + sampled.get(1 + reuseIdx) + "]");
+ + sampled.get(reuseIdx).index + "," + sampled.get(1 + reuseIdx).index + "]");
}
-// Map<String, Integer> firstSegment = segment.get(0);
-// firstSegment.put("lx", firstSegment.get("lx") + base);
-// firstSegment.put("rx", firstSegment.get("rx") + base);
-// sampled.get(1).index += base; // # note that no need to add base later for segment(1)
- segTs.add(sampled.get(1 + reuseIdx));
-
- buffer_startIdx = sampled.get(1 + reuseIdx).index;
-
-// if (sampled.size() == 2) {
-// // 更新buffer w
-// buffer_startIdx = buffer_endIdx - 1; // 注意左开右闭
-//// w = w.subList(w.size() - 1, w.size()); // last point in buffer for joint
-//
-// // 更新base
-// // NOTE python start from 0 not 1
-// // NOTE segment(0) rx already add base! i.e., segment(0) rx is already global position
-//// base = sampled.get(1).index;
-// } else {
-// // 更新buffer w
-// int out = sampled.get(1).index - sampled.get(0).index + 1;
-//
-// // 更新base
-// w = w.subList(out - 1, w.size()); // last point in buffer for joint
-// buffer_startIdx = sampled.get(1).index;
-//// base = base + sampled.get(1).index; // 对于joint来说，这个既是第一段的结尾又是第二段的开始位置
-// }
+ segTs.add(sampled.get(1 + reuseIdx)); // 分段右端点
+ buffer_startIdx = sampled.get(1 + reuseIdx).index; // buffer起点右移
}
if (debug) {
- System.out.println("buffer data: [" + buffer_startIdx + "," + buffer_endIdx + "]");
+ System.out.println("buffer data: [" + buffer_startIdx + "," + buffer_endIdx + ")" + ", remaining data: [" + buffer_endIdx + "," + points.size() + ")");
System.out.println("---------------------------------");
}
- if (buffer_endIdx >= points.size()) { // TODO no more point added into buffer w
+
+ if (buffer_endIdx >= points.size()) { // no more point added into buffer w
if (debug) {
System.out.println(">>>>no more data remaining");
}
// Add remaining segments and break if no more input data
- if (sampled.size() - reuseIdx > 2) {
- // 注意此时segment并没有把上面“左边输出一个segment”的第一个segment pop出去
- // 更新base
- // 这次不需要re-bottomUp了，所以回退到本次分段起点，这样后面的相对位置加上base才是绝对位置
- // NOTE segments does NOT have their index recalculated by bottom-up to start from 0,
- // therefore minus the previously added segment[1]["lx"]
-// base = base - sampled.get(1).index;
- for (int k = 2 + reuseIdx; k < sampled.size(); k++) {
-// Map<String, Integer> seg = segment.get(k);
-// seg.put("lx", seg.get("lx") + base);
-// seg.put("rx", seg.get("rx") + base);
- // 注意此时segment并没有把上面“左边输出一个segment”的第一个segment pop出去！
- // 所以要从第二个segment开始继续输出，同时注意python从0开始计数
-// sampled.get(k).index += base;
+ if (sampled.size() - reuseIdx > 2) { // more than one segment, 其中第一段segment已经在上面被记录了，但是没有被移出去
+ if (debug) {
+ System.out.println(">>>>so just left output all remaining segments");
+ }
+ for (int k = 2 + reuseIdx; k < sampled.size(); k++) { // 全局尾点就在这里输出
segTs.add(sampled.get(k));
+ if (debug) {
+ System.out.println("output: " + sampled.get(k));
+ }
}
}
break;
}
if ((buffer_endIdx - buffer_startIdx) >= lb) {
- // buffer w上面已经更新了，已经是去除了左边输出的第一个segment之后的剩余的点
if (debug) {
System.out.println(">>>>no need adding the sliding segment from right because len(w) >= lb");
}
needBottomUp = false;
- // 更新base
- // 这次不需要re-bottomUp了，所以回退到本次分段起点，这样后面的相对位置加上base才是绝对位置
- // NOTE segments does NOT have their index recalculated by bottom-up to start from 0,
- // therefore minus the previously added segment[1]["lx"]
-// base = base - sampled.get(1).index;
-
- // 注意此步骤不要在更新base之前执行！！！因为上面更新base时假设“左边输出一个segment”的第一个segment还没有pop
- // 注意这里真的把上面“左边输出一个segment”的第一个segment pop出去了，
- // 因为下一轮迭代要直接用剩余的segment
-// segment.remove(0); // Pop the first segment
-// sampled.remove(0); // TODO 这个复杂度？
- reuseIdx++; // TODO
+ reuseIdx++; // 这样简单，不需要sampled.remove(0)复杂度
if (debug) {
System.out.println("next iteration uses the remaining segment without re-bottomUp, reuseIdx=" + reuseIdx);
}
-
- // # assume lb>1, at least two segments
} else {
if (debug) {
System.out.println(">>>>adding the sliding segment from right because len(w) < lb");
@@ -449,88 +423,59 @@ public class SWAB {
while ((buffer_endIdx - buffer_startIdx) < lb && (points.size() - buffer_endIdx) > 0) {
// 从0开始，并且rx是从0开始的右闭 [0,rx]一共rx+1个点
List<Point> remaining_points = points.subList(buffer_endIdx, points.size());
- int rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, null); // TODO prefixsum加速
+ int rx;
+ if (prefixSum == null) {
+ rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, null);
+ } else {
+ // deprecated
+ double[] prefixSumX = Arrays.copyOfRange((double[]) prefixSum[0], buffer_endIdx, points.size());
+ double[] prefixSumY = Arrays.copyOfRange((double[]) prefixSum[1], buffer_endIdx, points.size());
+ double[] prefixSumX2 = Arrays.copyOfRange((double[]) prefixSum[2], buffer_endIdx, points.size());
+ double[] prefixSumY2 = Arrays.copyOfRange((double[]) prefixSum[3], buffer_endIdx, points.size());
+ double[] prefixSumXY = Arrays.copyOfRange((double[]) prefixSum[4], buffer_endIdx, points.size());
+ if (buffer_endIdx > 0) {
+ for (int i = 0; i < prefixSumX.length; i++) {
+ prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY.length; i++) {
+ prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumX2.length; i++) {
+ prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY2.length; i++) {
+ prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumXY.length; i++) {
+ prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_endIdx - 1];
+ }
+ }
+ Object[] prefixSumPart = new Object[]{prefixSumX, prefixSumY, prefixSumX2, prefixSumY2, prefixSumXY,
+ prefixSum[5], prefixSum[6]};
+ rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, prefixSumPart);
+ // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
+ }
if ((buffer_endIdx - buffer_startIdx) + (rx + 1) > ub) {
// # avoid more than ub
rx = ub - (buffer_endIdx - buffer_startIdx) - 1;
}
- buffer_endIdx += (rx + 1); // TODO
-// w.addAll(remaining_points.subList(0, rx + 1)); // TODO
-// remaining_points = remaining_points.subList(rx + 1, remaining_points.size());
+ buffer_endIdx += (rx + 1); // buffer终点右移
if (debug) {
System.out.println("input len=" + (rx + 1));
}
}
- }
- if (debug) {
- System.out.println("buffer data: [" + buffer_startIdx + "," + buffer_endIdx + "), "
- + "remaining data: [" + buffer_endIdx + "," + points.size() + ")");
- if ((buffer_endIdx - buffer_startIdx) < lb) {
- System.out.println("warn less");
- }
- if ((buffer_endIdx - buffer_startIdx) > ub) {
- System.out.println("warn more");
+ if (debug) {
+ System.out.println("buffer data: [" + buffer_startIdx + "," + buffer_endIdx + ")" + ", remaining data: [" + buffer_endIdx + "," + points.size() + ")");
+ if ((buffer_endIdx - buffer_startIdx) < lb) {
+ System.out.println("warn less");
+ }
+ if ((buffer_endIdx - buffer_startIdx) > ub) {
+ System.out.println("warn more");
+ }
}
}
}
return segTs;
}
-
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = false;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 100;
-// Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
- }
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(polyline.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
-
- long startTime = System.currentTimeMillis();
- Object[] prefixSum = prefixSum(polyline.getVertices());
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
-
- double maxError = 1000000000;
- int m = 10;
-
- // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
- startTime = System.currentTimeMillis();
- List<Point> sampled = swab_framework(polyline.getVertices(), maxError, m, true);
- endTime = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime - startTime) + "ms");
-
- for (Point p : sampled) {
- System.out.println(p);
- }
- System.out.println(sampled.size());
-
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < sampled.size(); i++) {
- writer.println(sampled.get(i).x + "," + sampled.get(i).y);
- }
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test7.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test7.java
new file mode 100644
index 00000000000..7590d40914d
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test7.java
@@ -0,0 +1,67 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.SWAB.swab_framework;
+
+public class Test7 {
+ // 用于验证java swab实现正确性
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 10000;
+ Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+// Polyline polyline = new Polyline();
+// for (int i = 0; i < N; i += 1) {
+// double v = rand.nextInt(1000);
+// polyline.addVertex(new Point(i, v));
+// }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+// long startTime = System.currentTimeMillis();
+// Object[] prefixSum = prefixSum(polyline.getVertices());
+// long endTime = System.currentTimeMillis();
+// System.out.println("Time taken to precompute prefix sum: " + (endTime - startTime) + "ms");
+
+ double maxError = 0.2;
+ int m = 1000;
+
+ // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
+ long startTime = System.currentTimeMillis();
+ List<Point> sampled = swab_framework(polyline.getVertices(), maxError, m, null, true);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+// for (Point p : sampled) {
+// System.out.println(p);
+// }
+ System.out.println(sampled.size());
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test8.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test8.java
new file mode 100644
index 00000000000..93b2ddf531c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test8.java
@@ -0,0 +1,65 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.SWAB.prefixSum;
+import static org.apache.iotdb.db.query.eBUG.SWAB.swab_framework;
+
+public class Test8 {
+ // 用于测试SWAB速度
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 1000_0000;
+// Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+// Object[] prefixSum = prefixSum(polyline.getVertices());
+
+ double maxError = 20000000;
+ int m = 10000;
+
+ // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
+ long startTime = System.currentTimeMillis();
+ List<Point> sampled = swab_framework(polyline.getVertices(), maxError, m, null, false);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+// for (Point p : sampled) {
+// System.out.println(p);
+// }
+ System.out.println(sampled.size());
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+}
(iotdb) 32/44: minmax
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 30c63392fcff1659ac8979a7858472911598f3e4
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jan 28 04:25:35 2025 +0800
minmax
---
server/pom.xml | 12 +-
server/sample_minmax-jar-with-dependencies.jar | Bin 0 -> 40761381 bytes
.../org/apache/iotdb/db/query/eBUG/MinMax.java | 241 +++++++++++++++++++++
.../org/apache/iotdb/db/query/eBUG/sample_FSW.java | 4 -
...ample_bottomUpYdiff.java => sample_MinMax.java} | 33 +--
.../iotdb/db/query/eBUG/sample_bottomUpYdiff.java | 1 -
.../apache/iotdb/db/query/eBUG/sample_eBUG.java | 3 +-
7 files changed, 270 insertions(+), 24 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index f918a874eb2..1808c5f6791 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -206,6 +206,12 @@
<artifactId>caffeine</artifactId>
<version>2.9.1</version>
</dependency>
+ <dependency>
+ <groupId>org.datanucleus</groupId>
+ <artifactId>datanucleus-core</artifactId>
+ <version>4.1.17</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
@@ -281,12 +287,14 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
- <finalName>sample_fsw</finalName>
+ <finalName>sample_minmax</finalName>
+ <!-- <finalName>sample_fsw</finalName>-->
<!-- <finalName>sample_eBUG</finalName>-->
<!-- <finalName>sample_BUYdiff</finalName>-->
<archive>
<manifest>
- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>
+ <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>-->
</manifest>
diff --git a/server/sample_minmax-jar-with-dependencies.jar b/server/sample_minmax-jar-with-dependencies.jar
new file mode 100644
index 00000000000..cd09c6c3bbf
Binary files /dev/null and b/server/sample_minmax-jar-with-dependencies.jar differ
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/MinMax.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/MinMax.java
new file mode 100644
index 00000000000..5ee516f0f9f
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/MinMax.java
@@ -0,0 +1,241 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+public class MinMax {
+ public enum fixedBUCKETtype {
+ width,
+ frequency,
+ }
+
+ public static List<Point> reducePoints_equalFrequencyBucket(
+ List<Point> points, int m, boolean debug) {
+ // 不算首尾点，还要(m-2)/2个分桶，每个分桶里采集MinMax两个点
+ int numOfBuckets = (int) ((m - 2) / 2.0);
+
+ // 等宽分桶
+ int frequency = (int) Math.ceil(points.size() * 1.0 / numOfBuckets);
+
+ if (debug) {
+ System.out.println("numOfBuckets=" + numOfBuckets);
+ System.out.println("frequency=" + frequency);
+ }
+
+ List<Point> res = new ArrayList<>();
+ res.add(points.get(0));
+
+ int currentBucketIdx = 0;
+ double currentBucketStartIdx = currentBucketIdx * frequency;
+ double currentBucketEndIdx = currentBucketStartIdx + frequency;
+ double minValue = Double.MAX_VALUE;
+ int minIdx = -1;
+ double maxValue = -Double.MAX_VALUE; // Double.MIN_VALUE is positive so do not use it!!!
+ int maxIdx = -1;
+ for (int i = 0; i < points.size() - 1; i++) { // 注意到倒数第二个点为止，因为如果到全局尾点，由于精度出入会有bug
+ Point p = points.get(i);
+ while (i >= currentBucketEndIdx) {
+ // record the results of the last bucket
+ if (minIdx < maxIdx) {
+ res.add(points.get(minIdx));
+ res.add(points.get(maxIdx));
+ } else {
+ res.add(points.get(maxIdx));
+ res.add(points.get(minIdx));
+ }
+ if (debug) {
+ System.out.println(
+ currentBucketStartIdx
+ + ","
+ + currentBucketEndIdx
+ + ","
+ + currentBucketIdx
+ + ","
+ + res.size());
+ }
+ // find the bucket that holds p, and reset for this new bucket
+ currentBucketIdx++;
+ currentBucketStartIdx = currentBucketIdx * frequency;
+ currentBucketEndIdx = currentBucketStartIdx + frequency;
+ minValue = Double.MAX_VALUE;
+ maxValue = -Double.MAX_VALUE;
+ minIdx = -1;
+ maxIdx = -1;
+ }
+ if (p.y < minValue) {
+ minValue = p.y;
+ minIdx = i;
+ }
+ if (p.y > maxValue) {
+ maxValue = p.y;
+ maxIdx = i;
+ }
+ }
+
+ // record the results of the last bucket
+ if (minIdx < maxIdx) {
+ res.add(points.get(minIdx));
+ res.add(points.get(maxIdx));
+ } else {
+ res.add(points.get(maxIdx));
+ res.add(points.get(minIdx));
+ }
+ if (debug) {
+ System.out.println(
+ currentBucketStartIdx
+ + ","
+ + currentBucketEndIdx
+ + ","
+ + currentBucketIdx
+ + ","
+ + res.size());
+ }
+
+ // record the last point
+ res.add(points.get(points.size() - 1));
+
+ return res;
+ }
+
+ public static List<Point> reducePoints_equalWidthBucket(
+ List<Point> points, int m, boolean debug) {
+ double startTime = points.get(0).x;
+ double endTime = points.get(points.size() - 1).x;
+
+ // 不算首尾点，还要(m-2)/2个分桶，每个分桶里采集MinMax两个点
+ int numOfBuckets = (int) ((m - 2) / 2.0);
+
+ // 等宽分桶
+ double interval = (endTime - startTime) / numOfBuckets;
+
+ if (debug) {
+ System.out.println("numOfBuckets=" + numOfBuckets);
+ System.out.println("width=" + interval);
+ }
+
+ List<Point> res = new ArrayList<>();
+ res.add(points.get(0));
+
+ int currentBucketIdx = 0;
+ double currentBucketStartTime = currentBucketIdx * interval;
+ double currentBucketEndTime = currentBucketStartTime + interval;
+ double minValue = Double.MAX_VALUE;
+ int minIdx = -1;
+ double maxValue = -Double.MAX_VALUE; // Double.MIN_VALUE is positive so do not use it!!!
+ int maxIdx = -1;
+ for (int i = 0; i < points.size() - 1; i++) { // 注意到倒数第二个点为止，因为如果到全局尾点，由于精度出入会有bug
+ Point p = points.get(i);
+ while (p.x >= currentBucketEndTime) {
+ // record the results of the last bucket
+ if (minIdx < maxIdx) {
+ res.add(points.get(minIdx));
+ res.add(points.get(maxIdx));
+ } else {
+ res.add(points.get(maxIdx));
+ res.add(points.get(minIdx));
+ }
+ if (debug) {
+ System.out.println(
+ currentBucketStartTime
+ + ","
+ + currentBucketEndTime
+ + ","
+ + currentBucketIdx
+ + ","
+ + res.size());
+ }
+ // find the bucket that holds p, and reset for this new bucket
+ currentBucketIdx++;
+ currentBucketStartTime = currentBucketIdx * interval;
+ currentBucketEndTime = currentBucketStartTime + interval;
+ minValue = Double.MAX_VALUE;
+ maxValue = -Double.MAX_VALUE;
+ minIdx = -1;
+ maxIdx = -1;
+ }
+ if (p.y < minValue) {
+ minValue = p.y;
+ minIdx = i;
+ }
+ if (p.y > maxValue) {
+ maxValue = p.y;
+ maxIdx = i;
+ }
+ }
+
+ // record the results of the last bucket
+ if (minIdx < maxIdx) {
+ res.add(points.get(minIdx));
+ res.add(points.get(maxIdx));
+ } else {
+ res.add(points.get(maxIdx));
+ res.add(points.get(minIdx));
+ }
+ if (debug) {
+ System.out.println(
+ currentBucketStartTime
+ + ","
+ + currentBucketEndTime
+ + ","
+ + currentBucketIdx
+ + ","
+ + res.size());
+ }
+
+ // record the last point
+ res.add(points.get(points.size() - 1));
+
+ return res;
+ }
+
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = -1;
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ // Polyline polyline = new Polyline();
+ // for (int i = 0; i < N; i += 1) {
+ // double v = rand.nextInt(1000);
+ // polyline.addVertex(new Point(i, v));
+ // }
+ // List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+ int m = 10;
+
+ long startTime = System.currentTimeMillis();
+ List<Point> sampled = reducePoints_equalFrequencyBucket(points, m, false);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ System.out.println(sampled.size());
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_FSW.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_FSW.java
index 426c99f4638..098cf1e36ff 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_FSW.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_FSW.java
@@ -9,9 +9,6 @@ import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
import static org.apache.iotdb.db.query.eBUG.Tool.getParam;
public class sample_FSW {
- // 输入一条时间序列 t,v
- // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
- // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
public static void main(String[] args) throws IOException {
if (args.length < 7) {
System.out.println(
@@ -65,7 +62,6 @@ public class sample_FSW {
+ (endTime - startTime)
+ "ms");
- // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
writer.write("x,y");
writer.newLine();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpYdiff.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_MinMax.java
similarity index 71%
copy from server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpYdiff.java
copy to server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_MinMax.java
index ec96d1d45e8..767b7f7151e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpYdiff.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_MinMax.java
@@ -7,11 +7,11 @@ import java.util.List;
import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
-public class sample_bottomUpYdiff {
+public class sample_MinMax {
public static void main(String[] args) throws IOException {
if (args.length < 7) {
System.out.println(
- "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,errorType,(outDir)");
+ "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,bucketType,(outDir)");
}
String input = args[0];
boolean hasHeader = Boolean.parseBoolean(args[1]);
@@ -19,16 +19,14 @@ public class sample_bottomUpYdiff {
int valueIdx = Integer.parseInt(args[3]);
int N = Integer.parseInt(args[4]); // N<=0表示读全部行，N>0表示读最多N行
int m = Integer.parseInt(args[5]);
- String errorTypeStr = args[6];
- DP.ERRORtype errorType;
- if (errorTypeStr.equals("L1")) {
- errorType = DP.ERRORtype.L1;
- } else if (errorTypeStr.equals("L2")) {
- errorType = DP.ERRORtype.L2;
- } else if (errorTypeStr.equals("L_infy")) {
- errorType = DP.ERRORtype.L_infy;
+ String bucketTypeStr = args[6];
+ MinMax.fixedBUCKETtype bucketType;
+ if (bucketTypeStr.equals("width")) {
+ bucketType = MinMax.fixedBUCKETtype.width;
+ } else if (bucketTypeStr.equals("frequency")) {
+ bucketType = MinMax.fixedBUCKETtype.frequency;
} else {
- throw new IOException("please input errorType as L1/L2/L_infy");
+ throw new IOException("please input fixed bucket type as width/frequency");
}
String outDir;
if (args.length > 7) {
@@ -44,20 +42,24 @@ public class sample_bottomUpYdiff {
System.out.println("Value index: " + valueIdx);
System.out.println("N: " + N);
System.out.println("m: " + m);
- System.out.println("errorType: " + errorType);
+ System.out.println("bucketType: " + bucketType);
System.out.println("outDir: " + outDir);
// 读取原始序列
List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
String outputFile =
- generateOutputFileName(input, outDir, "-BU" + errorType + "-n" + points.size() + "-m" + m);
+ generateOutputFileName(input, outDir, "-" + bucketType + "-n" + points.size() + "-m" + m);
System.out.println("Output file: " + outputFile); // do not modify this hint string log
// 采样
+ List<Point> results;
long startTime = System.currentTimeMillis();
- // List<Point> results = SWAB.seg_bottomUp_m_withTimestamps(points, m, null, false);
- List<Point> results = BottomUpYdiff.reducePoints(points, m, errorType, false);
+ if (bucketType == MinMax.fixedBUCKETtype.width) {
+ results = MinMax.reducePoints_equalWidthBucket(points, m, false);
+ } else {
+ results = MinMax.reducePoints_equalFrequencyBucket(points, m, false);
+ }
long endTime = System.currentTimeMillis();
System.out.println("result point number: " + results.size());
@@ -70,7 +72,6 @@ public class sample_bottomUpYdiff {
+ (endTime - startTime)
+ "ms");
- // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
writer.write("x,y");
writer.newLine();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpYdiff.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpYdiff.java
index ec96d1d45e8..47c4f1a21e1 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpYdiff.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_bottomUpYdiff.java
@@ -70,7 +70,6 @@ public class sample_bottomUpYdiff {
+ (endTime - startTime)
+ "ms");
- // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
writer.write("x,y");
writer.newLine();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java
index ca44730456c..d8a4b5ea198 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_eBUG.java
@@ -65,7 +65,7 @@ public class sample_eBUG {
+ (endTime - startTime)
+ "ms");
- // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
+ // 输出结果到csv
try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
// 写入表头
if (m <= 2) {
@@ -75,6 +75,7 @@ public class sample_eBUG {
writer.newLine();
// 写入数据行，按顺序 z, x, y
+ // 按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
for (Point point : results) {
writer.write(point.z + "," + point.x + "," + point.y);
writer.newLine();
(iotdb) 09/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 776f3845619179d72955f03e358f68c7170d8468
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jan 17 21:21:01 2025 +0800
add
---
.../main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
index 369c0fec918..4edf126b531 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
@@ -216,7 +216,7 @@ public class Visval_standard {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 1000_000;
+ int n = 100_0000;
int p = 1000;
for (int i = 0; i < n; i += p) {
(iotdb) 37/44: ltd
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 84b85948d46003114d3d831948ab80f166ea0a02
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jan 29 03:42:18 2025 +0800
ltd
---
server/pom.xml | 8 +-
...es.jar => sample_ltd-jar-with-dependencies.jar} | Bin 40769057 -> 40785602 bytes
server/sample_minmax-jar-with-dependencies.jar | Bin 40769057 -> 40785644 bytes
...ar => sample_swab_ad-jar-with-dependencies.jar} | Bin 40769057 -> 40785651 bytes
.../java/org/apache/iotdb/db/query/eBUG/LTD.java | 242 ++++++
.../org/apache/iotdb/db/query/eBUG/SWAB_AD.java | 956 +++++++++++----------
.../eBUG/{sample_SWAB.java => sample_LTD.java} | 25 +-
.../apache/iotdb/db/query/eBUG/sample_SWAB.java | 3 -
.../eBUG/{sample_SWAB.java => sample_SWABAD.java} | 11 +-
9 files changed, 741 insertions(+), 504 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index 760ce5fe6dd..efa727e5732 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -287,17 +287,21 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
+ <finalName>sample_ltd</finalName>
+ <!-- <finalName>sample_swab_ad</finalName>-->
<!-- <finalName>sample_swab</finalName>-->
<!-- <finalName>sample_rdp</finalName>-->
- <finalName>sample_minmax</finalName>
+ <!-- <finalName>sample_minmax</finalName>-->
<!-- <finalName>sample_fsw</finalName>-->
<!-- <finalName>sample_eBUG</finalName>-->
<!-- <finalName>sample_BUYdiff</finalName>-->
<archive>
<manifest>
+ <mainClass>org.apache.iotdb.db.query.eBUG.sample_LTD</mainClass>
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWABAD</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>-->
- <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>-->
diff --git a/server/sample_minmax-jar-with-dependencies.jar b/server/sample_ltd-jar-with-dependencies.jar
similarity index 96%
copy from server/sample_minmax-jar-with-dependencies.jar
copy to server/sample_ltd-jar-with-dependencies.jar
index f827302b64a..f4b6c621844 100644
Binary files a/server/sample_minmax-jar-with-dependencies.jar and b/server/sample_ltd-jar-with-dependencies.jar differ
diff --git a/server/sample_minmax-jar-with-dependencies.jar b/server/sample_minmax-jar-with-dependencies.jar
index f827302b64a..36a793f0ce3 100644
Binary files a/server/sample_minmax-jar-with-dependencies.jar and b/server/sample_minmax-jar-with-dependencies.jar differ
diff --git a/server/sample_minmax-jar-with-dependencies.jar b/server/sample_swab_ad-jar-with-dependencies.jar
similarity index 96%
copy from server/sample_minmax-jar-with-dependencies.jar
copy to server/sample_swab_ad-jar-with-dependencies.jar
index f827302b64a..f3565d6849b 100644
Binary files a/server/sample_minmax-jar-with-dependencies.jar and b/server/sample_swab_ad-jar-with-dependencies.jar differ
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD.java
new file mode 100644
index 00000000000..6eb2051411c
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD.java
@@ -0,0 +1,242 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
+public class LTD {
+ public static Point calculateAveragePoint(List<Point> points, int startClosed, int endClosed) {
+ double sumX = 0;
+ double sumY = 0;
+ for (int i = startClosed; i <= endClosed; i++) {
+ sumX += points.get(i).x;
+ sumY += points.get(i).y;
+ }
+ int len = endClosed - startClosed + 1;
+ return new Point(sumX / len, sumY / len);
+ }
+
+ public static double[] calculateLinearRegressionCoefficients(
+ List<Point> points, int startClosed, int endClosed) {
+ Point averages = calculateAveragePoint(points, startClosed, endClosed);
+ double avgX = averages.x;
+ double avgY = averages.y;
+
+ double aNumerator = 0.0;
+ double aDenominator = 0.0;
+ for (int i = startClosed; i <= endClosed; i++) {
+ aNumerator += (points.get(i).x - avgX) * (points.get(i).y - avgY);
+ aDenominator += (points.get(i).x - avgX) * (points.get(i).x - avgX);
+ }
+
+ double a = aNumerator / aDenominator;
+ double b = avgY - a * avgX;
+
+ return new double[] {a, b};
+ }
+
+ public static double calculateSSEForBucket(List<Point> points, int startClosed, int endClosed) {
+ double[] coefficients = calculateLinearRegressionCoefficients(points, startClosed, endClosed);
+ double a = coefficients[0];
+ double b = coefficients[1];
+
+ double sse = 0.0;
+
+ for (int i = startClosed; i <= endClosed; i++) {
+ double error = points.get(i).y - (a * points.get(i).x + b);
+ sse += error * error;
+ }
+
+ return sse;
+ }
+
+ public static List<Integer> getLtdBinIdxs(List<Point> points, int m, int maxIter, boolean debug) {
+ int numOfIterations = (int) (points.size() * 1.0 / m * 10);
+ if (maxIter >= 0) {
+ // numOfIterations = Math.min(numOfIterations, maxIter);
+ numOfIterations = maxIter; // overwrite
+ }
+ double blockSize = (points.size() - 3) * 1.0 / (m - 2);
+
+ List<Integer> offset = new LinkedList<>();
+ for (double i = 1; i < points.size(); i += blockSize) {
+ offset.add((int) i); // 1~n-2, 这样最后一个offset+1才不会超出边界
+ }
+ if (debug) {
+ System.out.println("numOfIterations=" + numOfIterations);
+ System.out.println(offset);
+ }
+
+ List<Double> sse = new LinkedList<>();
+
+ // Initialization
+ for (int i = 0; i < m - 2; i++) {
+ // with one extra point overlapping for each adjacent bucket
+ sse.add(calculateSSEForBucket(points, offset.get(i) - 1, offset.get(i + 1) + 1));
+ }
+
+ for (int c = 0; c < numOfIterations; c++) {
+ // Find the bucket to be split
+ int maxSSEIndex = -1;
+ double maxSSE = Double.NEGATIVE_INFINITY;
+ for (int i = 0; i < m - 2; i++) {
+ if (offset.get(i + 1) - offset.get(i) <= 1) {
+ continue;
+ }
+ if (sse.get(i) > maxSSE) {
+ maxSSE = sse.get(i);
+ maxSSEIndex = i;
+ }
+ }
+ if (maxSSEIndex < 0) {
+ if (debug) {
+ System.out.println(c);
+ System.out.println(maxSSEIndex);
+ System.out.println("break max");
+ }
+ break;
+ }
+
+ // Find the buckets to be merged
+ int minSSEIndex = -1;
+ double minSSE = Double.POSITIVE_INFINITY;
+ for (int i = 0; i < m - 3; i++) {
+ if (i == maxSSEIndex || i + 1 == maxSSEIndex) {
+ continue;
+ }
+ if (sse.get(i) + sse.get(i + 1) < minSSE) {
+ minSSE = sse.get(i) + sse.get(i + 1);
+ minSSEIndex = i;
+ }
+ }
+ if (minSSEIndex < 0) {
+ if (debug) {
+ System.out.println(c);
+ System.out.println(minSSEIndex);
+ System.out.println("break min");
+ }
+ break;
+ }
+
+ // Split
+ int startIdx = offset.get(maxSSEIndex);
+ int endIdx = offset.get(maxSSEIndex + 1);
+ int middleIdx = (startIdx + endIdx) / 2;
+ offset.add(maxSSEIndex + 1, middleIdx);
+
+ // Update SSE affected by split
+ sse.set(
+ maxSSEIndex,
+ calculateSSEForBucket(
+ points, offset.get(maxSSEIndex) - 1, offset.get(maxSSEIndex + 1) + 1));
+
+ double newSse =
+ calculateSSEForBucket(
+ points, offset.get(maxSSEIndex + 1) - 1, offset.get(maxSSEIndex + 2) + 1);
+
+ sse.add(maxSSEIndex + 1, newSse);
+
+ // Merge
+ if (minSSEIndex > maxSSEIndex) {
+ minSSEIndex += 1; // Adjust index
+ }
+ offset.remove(minSSEIndex + 1);
+
+ sse.set(
+ minSSEIndex,
+ calculateSSEForBucket(
+ points, offset.get(minSSEIndex) - 1, offset.get(minSSEIndex + 1) + 1));
+
+ sse.remove(minSSEIndex + 1);
+ }
+
+ // Convert ArrayList to int[] and return
+ return offset;
+ }
+
+ public static List<Point> LTTB(List<Point> points, List<Integer> bins) {
+ List<Point> res = new ArrayList<>();
+ res.add(points.get(0));
+
+ for (int i = 0; i < bins.size() - 1; i++) {
+ Point leftAnchor = res.get(res.size() - 1);
+ Point rightFloater;
+ if (i == bins.size() - 2) {
+ rightFloater = points.get(points.size() - 1);
+ } else {
+ rightFloater = calculateAveragePoint(points, bins.get(i + 1), bins.get(i + 2));
+ }
+ int bestIdx = -1;
+ double maxArea = -Double.MAX_VALUE;
+ for (int j = bins.get(i); j < bins.get(i + 1); j++) {
+ double area = Tool.triArea(points.get(j), leftAnchor, rightFloater);
+ if (area > maxArea) {
+ maxArea = area;
+ bestIdx = j;
+ }
+ }
+ if (bestIdx > 0) {
+ res.add(points.get(bestIdx));
+ }
+ }
+ res.add(points.get(points.size() - 1));
+ return res;
+ }
+
+ public static List<Point> LTD(List<Point> points, int m, int maxIter, boolean debug) {
+ List<Integer> bins = getLtdBinIdxs(points, m, maxIter, debug);
+ return LTTB(points, bins);
+ }
+
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 10000;
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ // Polyline polyline = new Polyline();
+ // for (int i = 0; i < N; i += 1) {
+ // double v = rand.nextInt(1000);
+ // polyline.addVertex(new Point(i, v));
+ // }
+ // List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ int m = 10;
+ int maxIter = 10;
+ long startTime = System.currentTimeMillis();
+ // List<Integer> bins = getLtdBinIdxs(points, m, true);
+ List<Point> sampled = LTD(points, m, maxIter, true);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ for (Point p : sampled) {
+ System.out.println(p);
+ }
+
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java
index 5da1b555258..3fed7cc4ef0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/SWAB_AD.java
@@ -4,521 +4,525 @@ import java.io.*;
import java.util.*;
public class SWAB_AD {
- public static List<Point> seg_bottomUp_maxerror_withTimestamps(
- List<Point> points, double maxError, Object[] prefixSum, boolean debug) {
- int total = points.size();
- if (total < 3) {
- return points; // 不足 3 个点无法形成三角形
- }
+ public static List<Point> seg_bottomUp_maxerror_withTimestamps(
+ List<Point> points, double maxError, Object[] prefixSum, boolean debug) {
+ int total = points.size();
+ if (total < 3) {
+ return points; // 不足 3 个点无法形成三角形
+ }
- // 每个triangle对应两个相邻的待合并的分段
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- points.get(0).prev = null;
- points.get(0).next = points.get(1);
- // points.get(0).index = 0;
- points.get(total - 1).next = null;
- points.get(total - 1).prev = points.get(total - 2);
- // points.get(total - 1).index = points.size() - 1;
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- // 相当于e=infinity
- double mc = DP.joint_segment_error(points, index1, index3, DP.ERRORtype.area);
- triangles[i - 1] = new Triangle(index1, index2, index3, mc);
-
- // 初始化点的状态 用于最后在线采样结果顺序输出未淘汰点
- points.get(i).prev = points.get(i - 1);
- points.get(i).next = points.get(i + 1);
- // points.get(i).index = i; // for swab usage
- }
+ // 每个triangle对应两个相邻的待合并的分段
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ points.get(0).prev = null;
+ points.get(0).next = points.get(1);
+ // points.get(0).index = 0;
+ points.get(total - 1).next = null;
+ points.get(total - 1).prev = points.get(total - 2);
+ // points.get(total - 1).index = points.size() - 1;
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+ // 相当于e=infinity
+ double mc = DP.joint_segment_error(points, index1, index3, DP.ERRORtype.area);
+ triangles[i - 1] = new Triangle(index1, index2, index3, mc);
+
+ // 初始化点的状态 用于最后在线采样结果顺序输出未淘汰点
+ points.get(i).prev = points.get(i - 1);
+ points.get(i).next = points.get(i + 1);
+ // points.get(i).index = i; // for swab usage
+ }
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) {
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) {
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ }
+
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)
+
+ int fakeCnt = 0;
+ while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll(); // O(logn)
+
+ // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ if (tri.isDeleted) {
+ fakeCnt--; // 取出了一个被标记删除点
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
+ continue;
+ }
+
+ // 真正的淘汰点
+ // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
+ if (debug) {
+ System.out.println("(1) eliminate " + points.get(tri.indices[1]));
+ }
+ points.get(tri.indices[1]).markEliminated();
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
+ // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
+
+ Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+
+ // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // 前一个三角形连到后一个三角形
+ tri.prev.next =
+ tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ tri.prev.indices[2] = tri.indices[2];
+
+ double mc =
+ DP.joint_segment_error(
+ points, tri.prev.indices[0], tri.prev.indices[2], DP.ERRORtype.area);
+ tri.prev.area = mc;
+ if (debug) {
+ System.out.println(
+ "(2) updating point on the left "
+ + points.get(tri.prev.indices[1])
+ + ", ranging ["
+ + tri.prev.indices[0]
+ + ","
+ + tri.prev.indices[2]
+ + "]");
}
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap =
- new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles); // complexity TODO O(n) or O(nlogn)
-
- int fakeCnt = 0;
- while (!triangleHeap.isEmpty() && triangleHeap.peek().area < maxError) { // TODO
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll(); // O(logn)
-
- // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
- // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
- if (tri.isDeleted) {
- fakeCnt--; // 取出了一个被标记删除点
- if (debug) {
- System.out.println(
- ">>>bottom-up, remaining "
- + triangleHeap.size()
- + " triangles (including those marked for removal)");
- }
- continue;
- }
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add来显式重新插入修改后的元素
+ triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
- // 真正的淘汰点
- // 记录最近淘汰的点，注意不要重复记录也就是在上面执行之后再确认淘汰
- if (debug) {
- System.out.println("(1) eliminate " + points.get(tri.indices[1]));
- }
- points.get(tri.indices[1]).markEliminated();
-
- // 更新相邻三角形
- if (tri.prev != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
-
- Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
- // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
-
- // 2. 处理pi被淘汰引起tri.prev被更新的事情
- // 前一个三角形连到后一个三角形
- tri.prev.next =
- tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
- tri.prev.indices[2] = tri.indices[2];
-
- double mc = DP.joint_segment_error(points, tri.prev.indices[0], tri.prev.indices[2], DP.ERRORtype.area);
- tri.prev.area = mc;
- if (debug) {
- System.out.println(
- "(2) updating point on the left "
- + points.get(tri.prev.indices[1])
- + ", ranging ["
- + tri.prev.indices[0]
- + ","
- + tri.prev.indices[2]
- + "]");
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add来显式重新插入修改后的元素
- triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
+ if (tri.next != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
- if (tri.next != null) {
- // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
-
- // 1. 处理旧的tri.next被标记删除的事情（角色替代）
- // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
- tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
-
- Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
- // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
-
- if (tri.prev != null) {
- tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
- }
-
- // 2. 处理pi被淘汰引起tri.next被更新的事情
- tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
- tri.next.indices[0] = tri.indices[0];
-
- double mc = DP.joint_segment_error(points, tri.next.indices[0], tri.next.indices[2], DP.ERRORtype.area);
- tri.next.area = mc;
- if (debug) {
- System.out.println(
- "(3) updating point on the right "
- + points.get(tri.next.indices[1])
- + ", ranging ["
- + tri.next.indices[0]
- + ","
- + tri.next.indices[2]
- + "]");
- }
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
- triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
- fakeCnt++; // 表示heap里多了一个被标记删除的假点
- }
- if (debug) {
- System.out.println(
- ">>>bottom-up, remaining "
- + triangleHeap.size()
- + " triangles (including those marked for removal)");
- }
+ // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+ tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+
+ Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
+
+ if (tri.prev != null) {
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
}
- List<Point> onlineSampled = new ArrayList<>();
- Point start = points.get(0);
- Point end = points.get(points.size() - 1);
- while (start
- != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
- start = start.next; // when e=0, only traversing three points pa pi pb
+ // 2. 处理pi被淘汰引起tri.next被更新的事情
+ tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ tri.next.indices[0] = tri.indices[0];
+
+ double mc =
+ DP.joint_segment_error(
+ points, tri.next.indices[0], tri.next.indices[2], DP.ERRORtype.area);
+ tri.next.area = mc;
+ if (debug) {
+ System.out.println(
+ "(3) updating point on the right "
+ + points.get(tri.next.indices[1])
+ + ", ranging ["
+ + tri.next.indices[0]
+ + ","
+ + tri.next.indices[2]
+ + "]");
}
- onlineSampled.add(end);
- return onlineSampled;
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ fakeCnt++; // 表示heap里多了一个被标记删除的假点
+ }
+ if (debug) {
+ System.out.println(
+ ">>>bottom-up, remaining "
+ + triangleHeap.size()
+ + " triangles (including those marked for removal)");
+ }
}
- public static int nextSlidingWindowWithTimestamps(
- List<Point> points, double maxError, Object[] prefixSum) {
- if (points.size() <= 1) {
- return points.size() - 1;
- }
- int i = 2; // 从第二个点开始
- while (i < points.size() && DP.joint_segment_error(points, 0, i, DP.ERRORtype.area) < maxError) {
- i++; // 窗口扩大
- }
- return i - 1; // 从0开始，并且是从0到i-1的左闭右闭
+ List<Point> onlineSampled = new ArrayList<>();
+ Point start = points.get(0);
+ Point end = points.get(points.size() - 1);
+ while (start
+ != end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
+ onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ start = start.next; // when e=0, only traversing three points pa pi pb
+ }
+ onlineSampled.add(end);
+ return onlineSampled;
+ }
+
+ public static int nextSlidingWindowWithTimestamps(
+ List<Point> points, double maxError, Object[] prefixSum) {
+ if (points.size() <= 1) {
+ return points.size() - 1;
+ }
+ int i = 2; // 从第二个点开始
+ while (i < points.size()
+ && DP.joint_segment_error(points, 0, i, DP.ERRORtype.area) < maxError) {
+ i++; // 窗口扩大
+ }
+ return i - 1; // 从0开始，并且是从0到i-1的左闭右闭
+ }
+
+ public static List<Point> swab_framework(List<Point> points, double maxError, int m) {
+ return swab_framework(points, maxError, m, null, false);
+ }
+
+ public static List<Point> reducePoints(List<Point> points, double maxError, Object... kwargs)
+ throws IOException {
+ int m = -1; // 默认值
+ for (Object arg : kwargs) {
+ if (arg instanceof Integer) {
+ m = (Integer) arg; // 获取 m 参数
+ }
+ }
+ if (m <= 0) {
+ throw new IOException("please set m>0");
+ }
+ return swab_framework(points, maxError, m, null, false);
+ }
+
+ public static List<Point> swab_framework(
+ List<Point> points, double maxError, int m, Object[] prefixSum, boolean debug) {
+ if (debug) {
+ if (prefixSum != null) {
+ // deprecated
+ System.out.println("enable prefix sum for accelerating L2 error computation");
+ }
}
- public static List<Point> swab_framework(List<Point> points, double maxError, int m) {
- return swab_framework(points, maxError, m, null, false);
+ // 在这里给Point添加全局排序idx, 用于全局定位
+ for (int i = 0; i < points.size(); i++) {
+ points.get(i).index = i;
}
- public static List<Point> reducePoints(List<Point> points, double maxError, Object... kwargs)
- throws IOException {
- int m = -1; // 默认值
- for (Object arg : kwargs) {
- if (arg instanceof Integer) {
- m = (Integer) arg; // 获取 m 参数
- }
- }
- if (m <= 0) {
- throw new IOException("please set m>0");
- }
- return swab_framework(points, maxError, m, null, false);
+ if (debug) {
+ System.out.println("data length=" + points.size() + ", max_error=" + maxError + ", m=" + m);
+ }
+ // m在这里的含义是分段数，虽然SWAB不能直接控制分段数，但是也需要这个参数来确定buffer大小
+ int bs = (int) Math.floor((double) points.size() / m * 6); // buffer size
+ int lb = bs / 2, ub = bs * 2;
+ if (lb == 0) {
+ lb += 2; // lb最小为2，否则陷入循环，因为joint buffer里至少会留一个点
+ ub += 2;
+ }
+ if (debug) {
+ System.out.println("bs=" + bs + ", lb=" + lb + ", ub=" + ub);
}
- public static List<Point> swab_framework(
- List<Point> points, double maxError, int m, Object[] prefixSum, boolean debug) {
+ int buffer_startIdx = 0; // 左闭
+ int buffer_endIdx = bs; // 右开
+ // 注意虽然joint，但是buffer和remaining points是buffer_endIdx处是不重叠的
+ // buffer: [0:bs) 左闭右开
+ // remaining: [bs:] 左闭
+
+ List<Point> segTs = new ArrayList<>();
+ segTs.add(points.get(0)); // 全局首点
+
+ boolean needBottomUp = true; // false when only output, no input
+ int reuseIdx = 0; // 注意这里当needBottomUp=false的时候是直接取用上一轮的bottomup points但是base从reuseIdx开始
+ List<Point> sampled = Collections.emptyList();
+ while (true) {
+ if (debug) {
+ System.out.println("#####################################################");
+ System.out.println(
+ "buffer data: ["
+ + buffer_startIdx
+ + ","
+ + buffer_endIdx
+ + ")"
+ + ", remaining data: ["
+ + buffer_endIdx
+ + ","
+ + points.size()
+ + ")");
+ System.out.println("reBottomUp:" + needBottomUp);
+ }
+
+ if (needBottomUp) {
if (debug) {
- if (prefixSum != null) {
- // deprecated
- System.out.println("enable prefix sum for accelerating L2 error computation");
+ System.out.println(">>>>bottom up on the buffer");
+ }
+ reuseIdx = 0; // 重置
+ if (prefixSum == null) {
+ sampled =
+ seg_bottomUp_maxerror_withTimestamps(
+ points.subList(buffer_startIdx, buffer_endIdx), maxError, null, false);
+ } else {
+ // deprecated
+ double[] prefixSumX =
+ Arrays.copyOfRange((double[]) prefixSum[0], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumY =
+ Arrays.copyOfRange((double[]) prefixSum[1], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumX2 =
+ Arrays.copyOfRange((double[]) prefixSum[2], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumY2 =
+ Arrays.copyOfRange((double[]) prefixSum[3], buffer_startIdx, buffer_endIdx);
+ double[] prefixSumXY =
+ Arrays.copyOfRange((double[]) prefixSum[4], buffer_startIdx, buffer_endIdx);
+ if (buffer_startIdx > 0) {
+ for (int i = 0; i < prefixSumX.length; i++) {
+ prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY.length; i++) {
+ prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumX2.length; i++) {
+ prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_startIdx - 1];
}
+ for (int i = 0; i < prefixSumY2.length; i++) {
+ prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_startIdx - 1];
+ }
+ for (int i = 0; i < prefixSumXY.length; i++) {
+ prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_startIdx - 1];
+ }
+ }
+ Object[] prefixSumPart =
+ new Object[] {
+ prefixSumX,
+ prefixSumY,
+ prefixSumX2,
+ prefixSumY2,
+ prefixSumXY,
+ prefixSum[5],
+ prefixSum[6]
+ };
+ sampled =
+ seg_bottomUp_maxerror_withTimestamps(
+ points.subList(buffer_startIdx, buffer_endIdx), maxError, prefixSumPart, false);
+ // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
}
- // 在这里给Point添加全局排序idx, 用于全局定位
- for (int i = 0; i < points.size(); i++) {
- points.get(i).index = i;
- }
+ // TODO eBUG also
+ // sampled = eBUG.buildEffectiveArea(points.subList(buffer_startIdx,
+ // buffer_endIdx), 0, false, m);
+ }
+ if (debug) {
+ System.out.println("number of bottomUp points=" + (sampled.size() - reuseIdx));
+ System.out.println("---------------------------------");
+ }
+
+ if (sampled.size() - reuseIdx >= 2) { // 左边输出一个segment
if (debug) {
- System.out.println("data length=" + points.size() + ", max_error=" + maxError + ", m=" + m);
- }
- // m在这里的含义是分段数，虽然SWAB不能直接控制分段数，但是也需要这个参数来确定buffer大小
- int bs = (int) Math.floor((double) points.size() / m * 6); // buffer size
- int lb = bs / 2, ub = bs * 2;
- if (lb == 0) {
- lb += 2; // lb最小为2，否则陷入循环，因为joint buffer里至少会留一个点
- ub += 2;
+ System.out.println(
+ ">>>>left output a segment:["
+ + sampled.get(reuseIdx).index
+ + ","
+ + sampled.get(1 + reuseIdx).index
+ + "]");
}
+ segTs.add(sampled.get(1 + reuseIdx)); // 分段右端点
+ buffer_startIdx = sampled.get(1 + reuseIdx).index; // buffer起点右移
+ }
+
+ if (debug) {
+ System.out.println(
+ "buffer data: ["
+ + buffer_startIdx
+ + ","
+ + buffer_endIdx
+ + ")"
+ + ", remaining data: ["
+ + buffer_endIdx
+ + ","
+ + points.size()
+ + ")");
+ System.out.println("---------------------------------");
+ }
+
+ if (buffer_endIdx >= points.size()) { // no more point added into buffer w
if (debug) {
- System.out.println("bs=" + bs + ", lb=" + lb + ", ub=" + ub);
+ System.out.println(">>>>no more data remaining");
}
-
- int buffer_startIdx = 0; // 左闭
- int buffer_endIdx = bs; // 右开
- // 注意虽然joint，但是buffer和remaining points是buffer_endIdx处是不重叠的
- // buffer: [0:bs) 左闭右开
- // remaining: [bs:] 左闭
-
- List<Point> segTs = new ArrayList<>();
- segTs.add(points.get(0)); // 全局首点
-
- boolean needBottomUp = true; // false when only output, no input
- int reuseIdx = 0; // 注意这里当needBottomUp=false的时候是直接取用上一轮的bottomup points但是base从reuseIdx开始
- List<Point> sampled = Collections.emptyList();
- while (true) {
- if (debug) {
- System.out.println("#####################################################");
- System.out.println(
- "buffer data: ["
- + buffer_startIdx
- + ","
- + buffer_endIdx
- + ")"
- + ", remaining data: ["
- + buffer_endIdx
- + ","
- + points.size()
- + ")");
- System.out.println("reBottomUp:" + needBottomUp);
- }
-
- if (needBottomUp) {
- if (debug) {
- System.out.println(">>>>bottom up on the buffer");
- }
- reuseIdx = 0; // 重置
- if (prefixSum == null) {
- sampled =
- seg_bottomUp_maxerror_withTimestamps(
- points.subList(buffer_startIdx, buffer_endIdx), maxError, null, false);
- } else {
- // deprecated
- double[] prefixSumX =
- Arrays.copyOfRange((double[]) prefixSum[0], buffer_startIdx, buffer_endIdx);
- double[] prefixSumY =
- Arrays.copyOfRange((double[]) prefixSum[1], buffer_startIdx, buffer_endIdx);
- double[] prefixSumX2 =
- Arrays.copyOfRange((double[]) prefixSum[2], buffer_startIdx, buffer_endIdx);
- double[] prefixSumY2 =
- Arrays.copyOfRange((double[]) prefixSum[3], buffer_startIdx, buffer_endIdx);
- double[] prefixSumXY =
- Arrays.copyOfRange((double[]) prefixSum[4], buffer_startIdx, buffer_endIdx);
- if (buffer_startIdx > 0) {
- for (int i = 0; i < prefixSumX.length; i++) {
- prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_startIdx - 1];
- }
- for (int i = 0; i < prefixSumY.length; i++) {
- prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_startIdx - 1];
- }
- for (int i = 0; i < prefixSumX2.length; i++) {
- prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_startIdx - 1];
- }
- for (int i = 0; i < prefixSumY2.length; i++) {
- prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_startIdx - 1];
- }
- for (int i = 0; i < prefixSumXY.length; i++) {
- prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_startIdx - 1];
- }
- }
- Object[] prefixSumPart =
- new Object[]{
- prefixSumX,
- prefixSumY,
- prefixSumX2,
- prefixSumY2,
- prefixSumXY,
- prefixSum[5],
- prefixSum[6]
- };
- sampled =
- seg_bottomUp_maxerror_withTimestamps(
- points.subList(buffer_startIdx, buffer_endIdx), maxError, prefixSumPart, false);
- // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
- }
-
- // TODO eBUG also
- // sampled = eBUG.buildEffectiveArea(points.subList(buffer_startIdx,
- // buffer_endIdx), 0, false, m);
- }
-
- if (debug) {
- System.out.println("number of bottomUp points=" + (sampled.size() - reuseIdx));
- System.out.println("---------------------------------");
- }
-
- if (sampled.size() - reuseIdx >= 2) { // 左边输出一个segment
- if (debug) {
- System.out.println(
- ">>>>left output a segment:["
- + sampled.get(reuseIdx).index
- + ","
- + sampled.get(1 + reuseIdx).index
- + "]");
- }
- segTs.add(sampled.get(1 + reuseIdx)); // 分段右端点
- buffer_startIdx = sampled.get(1 + reuseIdx).index; // buffer起点右移
- }
-
+ // Add remaining segments and break if no more input data
+ if (sampled.size() - reuseIdx
+ > 2) { // more than one segment, 其中第一段segment已经在上面被记录了，但是没有被移出去
+ if (debug) {
+ System.out.println(">>>>so just left output all remaining segments");
+ }
+ for (int k = 2 + reuseIdx; k < sampled.size(); k++) { // 全局尾点就在这里输出
+ segTs.add(sampled.get(k));
if (debug) {
- System.out.println(
- "buffer data: ["
- + buffer_startIdx
- + ","
- + buffer_endIdx
- + ")"
- + ", remaining data: ["
- + buffer_endIdx
- + ","
- + points.size()
- + ")");
- System.out.println("---------------------------------");
- }
-
- if (buffer_endIdx >= points.size()) { // no more point added into buffer w
- if (debug) {
- System.out.println(">>>>no more data remaining");
- }
- // Add remaining segments and break if no more input data
- if (sampled.size() - reuseIdx
- > 2) { // more than one segment, 其中第一段segment已经在上面被记录了，但是没有被移出去
- if (debug) {
- System.out.println(">>>>so just left output all remaining segments");
- }
- for (int k = 2 + reuseIdx; k < sampled.size(); k++) { // 全局尾点就在这里输出
- segTs.add(sampled.get(k));
- if (debug) {
- System.out.println("output: " + sampled.get(k));
- }
- }
- }
- break;
+ System.out.println("output: " + sampled.get(k));
}
+ }
+ }
+ break;
+ }
- if ((buffer_endIdx - buffer_startIdx) >= lb) {
- if (debug) {
- System.out.println(
- ">>>>no need adding the sliding segment from right because len(w) >= lb");
- }
- needBottomUp = false;
-
- reuseIdx++; // 这样简单，不需要sampled.remove(0)复杂度
- if (debug) {
- System.out.println(
- "next iteration uses the remaining segment without re-bottomUp, reuseIdx="
- + reuseIdx);
- }
- } else {
- if (debug) {
- System.out.println(">>>>adding the sliding segment from right because len(w) < lb");
- }
- needBottomUp = true; // 加了新的数据进来意味着下一轮里就要re-bottomUp
-
- while ((buffer_endIdx - buffer_startIdx) < lb && (points.size() - buffer_endIdx) > 0) {
- // 从0开始，并且rx是从0开始的右闭 [0,rx]一共rx+1个点
- List<Point> remaining_points = points.subList(buffer_endIdx, points.size());
- int rx;
- if (prefixSum == null) {
- rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, null);
- } else {
- // deprecated
- double[] prefixSumX =
- Arrays.copyOfRange((double[]) prefixSum[0], buffer_endIdx, points.size());
- double[] prefixSumY =
- Arrays.copyOfRange((double[]) prefixSum[1], buffer_endIdx, points.size());
- double[] prefixSumX2 =
- Arrays.copyOfRange((double[]) prefixSum[2], buffer_endIdx, points.size());
- double[] prefixSumY2 =
- Arrays.copyOfRange((double[]) prefixSum[3], buffer_endIdx, points.size());
- double[] prefixSumXY =
- Arrays.copyOfRange((double[]) prefixSum[4], buffer_endIdx, points.size());
- if (buffer_endIdx > 0) {
- for (int i = 0; i < prefixSumX.length; i++) {
- prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_endIdx - 1];
- }
- for (int i = 0; i < prefixSumY.length; i++) {
- prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_endIdx - 1];
- }
- for (int i = 0; i < prefixSumX2.length; i++) {
- prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_endIdx - 1];
- }
- for (int i = 0; i < prefixSumY2.length; i++) {
- prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_endIdx - 1];
- }
- for (int i = 0; i < prefixSumXY.length; i++) {
- prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_endIdx - 1];
- }
- }
- Object[] prefixSumPart =
- new Object[]{
- prefixSumX,
- prefixSumY,
- prefixSumX2,
- prefixSumY2,
- prefixSumXY,
- prefixSum[5],
- prefixSum[6]
- };
- rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, prefixSumPart);
- // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
- }
- if ((buffer_endIdx - buffer_startIdx) + (rx + 1) > ub) {
- // # avoid more than ub
- rx = ub - (buffer_endIdx - buffer_startIdx) - 1;
- }
- buffer_endIdx += (rx + 1); // buffer终点右移
- if (debug) {
- System.out.println("input len=" + (rx + 1));
- }
- }
-
- if (debug) {
- System.out.println(
- "buffer data: ["
- + buffer_startIdx
- + ","
- + buffer_endIdx
- + ")"
- + ", remaining data: ["
- + buffer_endIdx
- + ","
- + points.size()
- + ")");
- if ((buffer_endIdx - buffer_startIdx) < lb) {
- System.out.println("warn less");
- }
- if ((buffer_endIdx - buffer_startIdx) > ub) {
- System.out.println("warn more");
- }
- }
- }
+ if ((buffer_endIdx - buffer_startIdx) >= lb) {
+ if (debug) {
+ System.out.println(
+ ">>>>no need adding the sliding segment from right because len(w) >= lb");
}
- return segTs;
- }
+ needBottomUp = false;
- public static void main(String[] args) throws IOException {
- Random rand = new Random(10);
- String input = "D:\\datasets\\regular\\tmp2.csv";
- boolean hasHeader = false;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 100_0000;
- // Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
+ reuseIdx++; // 这样简单，不需要sampled.remove(0)复杂度
+ if (debug) {
+ System.out.println(
+ "next iteration uses the remaining segment without re-bottomUp, reuseIdx="
+ + reuseIdx);
+ }
+ } else {
+ if (debug) {
+ System.out.println(">>>>adding the sliding segment from right because len(w) < lb");
}
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ needBottomUp = true; // 加了新的数据进来意味着下一轮里就要re-bottomUp
+
+ while ((buffer_endIdx - buffer_startIdx) < lb && (points.size() - buffer_endIdx) > 0) {
+ // 从0开始，并且rx是从0开始的右闭 [0,rx]一共rx+1个点
+ List<Point> remaining_points = points.subList(buffer_endIdx, points.size());
+ int rx;
+ if (prefixSum == null) {
+ rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, null);
+ } else {
+ // deprecated
+ double[] prefixSumX =
+ Arrays.copyOfRange((double[]) prefixSum[0], buffer_endIdx, points.size());
+ double[] prefixSumY =
+ Arrays.copyOfRange((double[]) prefixSum[1], buffer_endIdx, points.size());
+ double[] prefixSumX2 =
+ Arrays.copyOfRange((double[]) prefixSum[2], buffer_endIdx, points.size());
+ double[] prefixSumY2 =
+ Arrays.copyOfRange((double[]) prefixSum[3], buffer_endIdx, points.size());
+ double[] prefixSumXY =
+ Arrays.copyOfRange((double[]) prefixSum[4], buffer_endIdx, points.size());
+ if (buffer_endIdx > 0) {
+ for (int i = 0; i < prefixSumX.length; i++) {
+ prefixSumX[i] -= ((double[]) prefixSum[0])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY.length; i++) {
+ prefixSumY[i] -= ((double[]) prefixSum[1])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumX2.length; i++) {
+ prefixSumX2[i] -= ((double[]) prefixSum[2])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumY2.length; i++) {
+ prefixSumY2[i] -= ((double[]) prefixSum[3])[buffer_endIdx - 1];
+ }
+ for (int i = 0; i < prefixSumXY.length; i++) {
+ prefixSumXY[i] -= ((double[]) prefixSum[4])[buffer_endIdx - 1];
+ }
}
- System.out.println(polyline.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
+ Object[] prefixSumPart =
+ new Object[] {
+ prefixSumX,
+ prefixSumY,
+ prefixSumX2,
+ prefixSumY2,
+ prefixSumXY,
+ prefixSum[5],
+ prefixSum[6]
+ };
+ rx = nextSlidingWindowWithTimestamps(remaining_points, maxError, prefixSumPart);
+ // TODO prefixsum加速 但是由于精度问题，再加上maxError控制，结果有时候不会完全相同
+ }
+ if ((buffer_endIdx - buffer_startIdx) + (rx + 1) > ub) {
+ // # avoid more than ub
+ rx = ub - (buffer_endIdx - buffer_startIdx) - 1;
+ }
+ buffer_endIdx += (rx + 1); // buffer终点右移
+ if (debug) {
+ System.out.println("input len=" + (rx + 1));
+ }
}
- // Object[] prefixSum = prefixSum(polyline.getVertices());
+ if (debug) {
+ System.out.println(
+ "buffer data: ["
+ + buffer_startIdx
+ + ","
+ + buffer_endIdx
+ + ")"
+ + ", remaining data: ["
+ + buffer_endIdx
+ + ","
+ + points.size()
+ + ")");
+ if ((buffer_endIdx - buffer_startIdx) < lb) {
+ System.out.println("warn less");
+ }
+ if ((buffer_endIdx - buffer_startIdx) > ub) {
+ System.out.println("warn more");
+ }
+ }
+ }
+ }
+ return segTs;
+ }
+
+ public static void main(String[] args) throws IOException {
+ Random rand = new Random(10);
+ String input = "D:\\datasets\\regular\\tmp2.csv";
+ boolean hasHeader = false;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 100_0000;
+ // Polyline polyline = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(polyline.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
- double maxError = 20000000;
- int m = 1000;
+ // Object[] prefixSum = prefixSum(polyline.getVertices());
- // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
- long startTime = System.currentTimeMillis();
- // List<Point> sampled = swab_framework(polyline.getVertices(), maxError, m, null,
- // false);
- List<Point> sampled = reducePoints(polyline.getVertices(), maxError, m);
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ double maxError = 20000000;
+ int m = 1000;
- // for (Point p : sampled) {
- // System.out.println(p);
- // }
- System.out.println(sampled.size());
+ // 33s worst-case O(n2)因为没有prefix sum加速计算L2误差，但是实际达不到worst-case所以实际运行不会那么大耗时
+ long startTime = System.currentTimeMillis();
+ // List<Point> sampled = swab_framework(polyline.getVertices(), maxError, m, null,
+ // false);
+ List<Point> sampled = reducePoints(polyline.getVertices(), maxError, m);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
- try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
- // 写入字符串
- for (int i = 0; i < sampled.size(); i++) {
- writer.println(sampled.get(i).x + "," + sampled.get(i).y);
- }
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ System.out.println(sampled.size());
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- }
+ try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+ // 写入字符串
+ for (int i = 0; i < sampled.size(); i++) {
+ writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+ }
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_LTD.java
similarity index 64%
copy from server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java
copy to server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_LTD.java
index e97763633e7..8f0901d0e0a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_LTD.java
@@ -6,16 +6,12 @@ import java.io.IOException;
import java.util.List;
import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
-import static org.apache.iotdb.db.query.eBUG.Tool.getParam;
-public class sample_SWAB {
- // 输入一条时间序列 t,v
- // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
- // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
+public class sample_LTD {
public static void main(String[] args) throws IOException {
if (args.length < 7) {
System.out.println(
- "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,tolerantRatio,(outDir)");
+ "Usage: Please provide arguments: inputFilePath,hasHeader,timeIdx,valueIdx,N,m,maxIter,(outDir)");
}
String input = args[0];
boolean hasHeader = Boolean.parseBoolean(args[1]);
@@ -23,7 +19,7 @@ public class sample_SWAB {
int valueIdx = Integer.parseInt(args[3]);
int N = Integer.parseInt(args[4]); // N<=0表示读全部行，N>0表示读最多N行
int m = Integer.parseInt(args[5]);
- double tolerantRatio = Double.parseDouble(args[6]); // 查找参数epsilon以使得输出采样点数接近m的程度，比如0.01
+ int maxIter = Integer.parseInt(args[6]); // >=0代表迭代次数直接由maxIter指定
String outDir;
if (args.length > 7) {
outDir = args[7]; // 表示输出文件保存到指定的文件夹
@@ -38,22 +34,20 @@ public class sample_SWAB {
System.out.println("Value index: " + valueIdx);
System.out.println("N: " + N);
System.out.println("m: " + m);
- System.out.println("tolerantRatio: " + tolerantRatio);
+ System.out.println("maxIter: " + maxIter);
System.out.println("outDir: " + outDir);
// 读取原始序列
List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
String outputFile =
- generateOutputFileName(input, outDir, "-SWAB" + "-n" + points.size() + "-m" + m);
- System.out.println("Output file: " + outputFile);
-
- // 查找epsilon参数使得采样点数接近m
- double epsilon = getParam(points, m, SWAB::reducePoints, m * tolerantRatio, m);
+ generateOutputFileName(input, outDir, "-LTD-" + maxIter + "-n" + points.size() + "-m" + m);
+ System.out.println("Output file: " + outputFile); // do not modify this hint string log
// 采样
+ List<Point> results;
long startTime = System.currentTimeMillis();
- List<Point> results = SWAB.reducePoints(points, epsilon, m);
+ results = LTD.LTD(points, m, maxIter, false);
long endTime = System.currentTimeMillis();
System.out.println("result point number: " + results.size());
@@ -62,11 +56,10 @@ public class sample_SWAB {
+ points.size()
+ ", m="
+ m
- + ", Time taken to reduce points: "
+ + ", Time taken to reduce points: " // do not modify this hint string log
+ (endTime - startTime)
+ "ms");
- // 输出结果到csv，按照z,x,y三列，因为results结果已经按照z（即DS）递增排序，对应bottom-up的淘汰顺序，越小代表越早被淘汰
try (BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile))) {
writer.write("x,y");
writer.newLine();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java
index e97763633e7..19801ec3f38 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java
@@ -9,9 +9,6 @@ import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
import static org.apache.iotdb.db.query.eBUG.Tool.getParam;
public class sample_SWAB {
- // 输入一条时间序列 t,v
- // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
- // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
public static void main(String[] args) throws IOException {
if (args.length < 7) {
System.out.println(
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWABAD.java
similarity index 82%
copy from server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java
copy to server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWABAD.java
index e97763633e7..32aba1d166c 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWAB.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/sample_SWABAD.java
@@ -8,10 +8,7 @@ import java.util.List;
import static org.apache.iotdb.db.query.eBUG.Tool.generateOutputFileName;
import static org.apache.iotdb.db.query.eBUG.Tool.getParam;
-public class sample_SWAB {
- // 输入一条时间序列 t,v
- // 输出按照bottom-up淘汰顺序排列的dominated significance,t,v。
- // 用于后期在线采样时选取倒数m个点（也就是DS最大的m个点，或者最晚淘汰的m个点）作为采样结果（选出之后要自行把这m个点重新按照时间戳x递增排列）
+public class sample_SWABAD {
public static void main(String[] args) throws IOException {
if (args.length < 7) {
System.out.println(
@@ -45,15 +42,15 @@ public class sample_SWAB {
List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
String outputFile =
- generateOutputFileName(input, outDir, "-SWAB" + "-n" + points.size() + "-m" + m);
+ generateOutputFileName(input, outDir, "-SWAB_AD" + "-n" + points.size() + "-m" + m);
System.out.println("Output file: " + outputFile);
// 查找epsilon参数使得采样点数接近m
- double epsilon = getParam(points, m, SWAB::reducePoints, m * tolerantRatio, m);
+ double epsilon = getParam(points, m, SWAB_AD::reducePoints, m * tolerantRatio, m);
// 采样
long startTime = System.currentTimeMillis();
- List<Point> results = SWAB.reducePoints(points, epsilon, m);
+ List<Point> results = SWAB_AD.reducePoints(points, epsilon, m);
long endTime = System.currentTimeMillis();
System.out.println("result point number: " + results.size());
(iotdb) 42/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8174cf4381e99576526bd237262d3b27860e8928
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Feb 4 00:04:36 2025 +0800
add
---
.../main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 3e90214d277..071f495bef9 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -27,15 +27,15 @@ import java.nio.charset.Charset;
/** TSFileConfig is a configure class. Every variables is public and has default value. */
public class TSFileConfig implements Serializable {
- private boolean writeConvexHull = true;
+ private boolean writeConvexHull = false;
private boolean enableMinMaxLSM = false;
private boolean useStatistics = true;
- private boolean useTimeIndex = true;
+ private boolean useTimeIndex = false;
- private boolean useValueIndex = true;
+ private boolean useValueIndex = false;
private boolean useMad = false;
(iotdb) 07/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 54b9ed1181a013cc5d9c13a9bb6754fd173b1619
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jan 17 20:40:51 2025 +0800
add
---
.../iotdb/db/query/simpiece/Visval_standard.java | 100 +++++++++++----------
.../db/query/simpiece/Visval_standard_slow.java | 62 +++++++------
2 files changed, 85 insertions(+), 77 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
index 80aa0995809..28f0f26618a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.query.simpiece;
import java.io.FileWriter;
import java.io.IOException;
import java.util.*;
-import java.util.stream.Collectors;
// adapted from the open source C++ code
// https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
@@ -127,9 +126,7 @@ public class Visval_standard {
// 创建所有三角形并计算初始面积
for (int i = 1; i < total - 1; i++) {
int index1 = i - 1, index2 = i, index3 = i + 1;
- double area =
- triArea(
- lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
+ double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
triangles[i - 1] = new Triangle(index1, index2, index3, area);
}
@@ -140,8 +137,7 @@ public class Visval_standard {
}
// 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap =
- new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
Collections.addAll(triangleHeap, triangles); // complexity TODO
double previousEA = -1;
@@ -163,6 +159,8 @@ public class Visval_standard {
results.get(tri.indices[1]).z = previousEA;
// System.out.println(tri.indices[1] + "," + previousEA);
+ System.out.println(Arrays.toString(tri.indices) + "," + previousEA);
+
// 更新相邻三角形
if (tri.prev != null) {
// 1. 处理旧的tri.prev被标记删除的事情（角色替代）
@@ -170,21 +168,17 @@ public class Visval_standard {
tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
- tri.prev = newPre; // replace
+// tri.prev = newPre; // replace TODO can omit
// 2. 处理pi被淘汰引起tri.prev被更新的事情
// 前一个三角形连到后一个三角形
tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
tri.prev.indices[2] = tri.indices[2];
- tri.prev.area =
- triArea(
- lineToSimplify.get(tri.prev.indices[0]),
- lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2]));
+ tri.prev.area = triArea(lineToSimplify.get(tri.prev.indices[0]), lineToSimplify.get(tri.prev.indices[1]), lineToSimplify.get(tri.prev.indices[2]));
// 重新加入堆
// 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过add 来显式重新插入修改后的元素
+ // 所以必须通过add来显式重新插入修改后的元素
triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
}
@@ -194,7 +188,7 @@ public class Visval_standard {
tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
- tri.next = newNext; // replace
+// tri.next = newNext; // replace TODO can omit
if (tri.prev != null) {
tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
@@ -203,11 +197,7 @@ public class Visval_standard {
// 2. 处理pi被淘汰引起tri.next被更新的事情
tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
tri.next.indices[0] = tri.indices[0];
- tri.next.area =
- triArea(
- lineToSimplify.get(tri.next.indices[0]),
- lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2]));
+ tri.next.area = triArea(lineToSimplify.get(tri.next.indices[0]), lineToSimplify.get(tri.next.indices[1]), lineToSimplify.get(tri.next.indices[2]));
// 重新加入堆
// 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
@@ -221,13 +211,13 @@ public class Visval_standard {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 10000; //1000_000;
+ int n = 100000; //1000_000;
int p = 100;
for (int i = 0; i < n; i += p) {
Polyline polylineBatch = new Polyline();
for (int j = i; j < Math.min(i + p, n); j++) {
- double v = rand.nextInt(1000);
+ double v = rand.nextInt(1000000);
polyline.addVertex(new vPoint(j, v));
@@ -236,6 +226,20 @@ public class Visval_standard {
polylineList.add(polylineBatch);
}
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ vPoint point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
System.out.println("---------------------------------");
List<vPoint> results = new ArrayList<>();
// 计算运行时间
@@ -268,33 +272,33 @@ public class Visval_standard {
System.out.println("Error writing to CSV file: " + e.getMessage());
}
- System.out.println("---------------------------------");
- List<List<vPoint>> resultsBatchList = new ArrayList<>();
- // 计算运行时间
- int cnt = 0;
- startTime = System.currentTimeMillis();
- for (Polyline polylineBatch : polylineList) {
- List<vPoint> resultsBatch = new ArrayList<>();
- buildEffectiveArea(polylineBatch, resultsBatch);
- cnt += resultsBatch.size();
- resultsBatchList.add(resultsBatch);
- }
- // 输出结果
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(cnt);
-
- System.out.println("---------------------------------");
- // 使用 Stream API 合并所有列表
- List<vPoint> mergedList =
- resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
- int sameCnt = 0;
- for (int i = 0; i < mergedList.size(); i++) {
- if (mergedList.get(i).z == results.get(i).z) {
- sameCnt += 1;
- }
- }
- System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
+// System.out.println("---------------------------------");
+// List<List<vPoint>> resultsBatchList = new ArrayList<>();
+// // 计算运行时间
+// int cnt = 0;
+// startTime = System.currentTimeMillis();
+// for (Polyline polylineBatch : polylineList) {
+// List<vPoint> resultsBatch = new ArrayList<>();
+// buildEffectiveArea(polylineBatch, resultsBatch);
+// cnt += resultsBatch.size();
+// resultsBatchList.add(resultsBatch);
+// }
+// // 输出结果
+// endTime = System.currentTimeMillis();
+// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+// System.out.println(cnt);
+//
+// System.out.println("---------------------------------");
+// // 使用 Stream API 合并所有列表
+// List<vPoint> mergedList =
+// resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
+// int sameCnt = 0;
+// for (int i = 0; i < mergedList.size(); i++) {
+// if (mergedList.get(i).z == results.get(i).z) {
+// sameCnt += 1;
+// }
+// }
+// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
}
// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard_slow.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard_slow.java
index 929fa304568..33259cc4cff 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard_slow.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard_slow.java
@@ -119,6 +119,8 @@
// // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
// Triangle tri = triangleHeap.poll(); // O(logn)
//
+//// System.out.println(Arrays.toString(tri.indices));
+//
// // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
// if (tri.area > previousEA) {
// previousEA = tri.area;
@@ -126,6 +128,8 @@
// results.get(tri.indices[1]).z = previousEA;
// // System.out.println(tri.indices[1] + "," + previousEA);
//
+// System.out.println(Arrays.toString(tri.indices) + "," + previousEA);
+//
// // 更新相邻三角形
// if (tri.prev != null) {
// // 前一个三角形连到后一个三角形
@@ -166,13 +170,13 @@
// List<Polyline> polylineList = new ArrayList<>();
// Random rand = new Random(1);
//// int n = 1000_000;
-// int n = 10000;
+// int n = 100000;
//
// int p = 100;
// for (int i = 0; i < n; i += p) {
// Polyline polylineBatch = new Polyline();
// for (int j = i; j < Math.min(i + p, n); j++) {
-// double v = rand.nextInt(1000);
+// double v = rand.nextInt(1000000);
//
// polyline.addVertex(new vPoint(j, v));
//
@@ -213,33 +217,33 @@
// System.out.println("Error writing to CSV file: " + e.getMessage());
// }
//
-// System.out.println("---------------------------------");
-// List<List<vPoint>> resultsBatchList = new ArrayList<>();
-// // 计算运行时间
-// int cnt = 0;
-// startTime = System.currentTimeMillis();
-// for (Polyline polylineBatch : polylineList) {
-// List<vPoint> resultsBatch = new ArrayList<>();
-// buildEffectiveArea(polylineBatch, resultsBatch);
-// cnt += resultsBatch.size();
-// resultsBatchList.add(resultsBatch);
-// }
-// // 输出结果
-// endTime = System.currentTimeMillis();
-// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
-// System.out.println(cnt);
-//
-// System.out.println("---------------------------------");
-// // 使用 Stream API 合并所有列表
-// List<vPoint> mergedList =
-// resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
-// int sameCnt = 0;
-// for (int i = 0; i < mergedList.size(); i++) {
-// if (mergedList.get(i).z == results.get(i).z) {
-// sameCnt += 1;
-// }
-// }
-// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
+//// System.out.println("---------------------------------");
+//// List<List<vPoint>> resultsBatchList = new ArrayList<>();
+//// // 计算运行时间
+//// int cnt = 0;
+//// startTime = System.currentTimeMillis();
+//// for (Polyline polylineBatch : polylineList) {
+//// List<vPoint> resultsBatch = new ArrayList<>();
+//// buildEffectiveArea(polylineBatch, resultsBatch);
+//// cnt += resultsBatch.size();
+//// resultsBatchList.add(resultsBatch);
+//// }
+//// // 输出结果
+//// endTime = System.currentTimeMillis();
+//// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+//// System.out.println(cnt);
+////
+//// System.out.println("---------------------------------");
+//// // 使用 Stream API 合并所有列表
+//// List<vPoint> mergedList =
+//// resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
+//// int sameCnt = 0;
+//// for (int i = 0; i < mergedList.size(); i++) {
+//// if (mergedList.get(i).z == results.get(i).z) {
+//// sameCnt += 1;
+//// }
+//// }
+//// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
//
//
//
(iotdb) 05/44: format code
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 545bff4a975f9da11f15d4228222eadd469880b2
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jan 3 14:01:45 2025 +0800
format code
---
.../iotdb/db/query/simpiece/Visval_standard.java | 116 ++++++++++++---------
1 file changed, 68 insertions(+), 48 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
index b5033b86097..5471f62f65f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.iotdb.db.query.simpiece;
import java.util.ArrayList;
@@ -8,7 +27,8 @@ import java.util.PriorityQueue;
import java.util.Random;
import java.util.stream.Collectors;
-// adapted from the open source C++ code https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
+// adapted from the open source C++ code
+// https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
class Triangle {
int[] indices = new int[3];
@@ -31,7 +51,7 @@ class vPoint {
public vPoint(double x, double y) {
this.x = x;
this.y = y;
- this.z = Double.POSITIVE_INFINITY;// effective area
+ this.z = Double.POSITIVE_INFINITY; // effective area
}
}
@@ -79,8 +99,9 @@ public class Visval_standard {
// 创建所有三角形并计算初始面积
for (int i = 1; i < total - 1; i++) {
int index1 = i - 1, index2 = i, index3 = i + 1;
- double area = triArea(lineToSimplify.get(index1), lineToSimplify.get(index2),
- lineToSimplify.get(index3));
+ double area =
+ triArea(
+ lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
triangles[i - 1] = new Triangle(index1, index2, index3, area);
}
@@ -91,8 +112,8 @@ public class Visval_standard {
}
// 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap = new PriorityQueue<>(
- Comparator.comparingDouble(t -> t.area));
+ PriorityQueue<Triangle> triangleHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
Collections.addAll(triangleHeap, triangles);
double previousEA = -1;
@@ -106,18 +127,18 @@ public class Visval_standard {
previousEA = tri.area;
}
results.get(tri.indices[1]).z = previousEA;
-// System.out.println(tri.indices[1] + "," + previousEA);
+ // System.out.println(tri.indices[1] + "," + previousEA);
// 更新相邻三角形
if (tri.prev != null) {
// 前一个三角形连到后一个三角形
tri.prev.next = tri.next;
tri.prev.indices[2] = tri.indices[2];
- tri.prev.area = triArea(
- lineToSimplify.get(tri.prev.indices[0]),
- lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2])
- );
+ tri.prev.area =
+ triArea(
+ lineToSimplify.get(tri.prev.indices[0]),
+ lineToSimplify.get(tri.prev.indices[1]),
+ lineToSimplify.get(tri.prev.indices[2]));
// 重新加入堆
// 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
@@ -130,11 +151,11 @@ public class Visval_standard {
// 后一个三角形连到前一个三角形
tri.next.prev = tri.prev;
tri.next.indices[0] = tri.indices[0];
- tri.next.area = triArea(
- lineToSimplify.get(tri.next.indices[0]),
- lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2])
- );
+ tri.next.area =
+ triArea(
+ lineToSimplify.get(tri.next.indices[0]),
+ lineToSimplify.get(tri.next.indices[1]),
+ lineToSimplify.get(tri.next.indices[2]));
// 重新加入堆
triangleHeap.remove(tri.next);
@@ -169,10 +190,10 @@ public class Visval_standard {
System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
System.out.println(results.size());
-// for (int i = 0; i < results.size(); i++) {
-// vPoint point = results.get(i);
-// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
-// }
+ // for (int i = 0; i < results.size(); i++) {
+ // vPoint point = results.get(i);
+ // System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ // }
System.out.println("---------------------------------");
List<List<vPoint>> resultsBatchList = new ArrayList<>();
@@ -192,8 +213,8 @@ public class Visval_standard {
System.out.println("---------------------------------");
// 使用 Stream API 合并所有列表
- List<vPoint> mergedList = resultsBatchList.stream().flatMap(List::stream)
- .collect(Collectors.toList());
+ List<vPoint> mergedList =
+ resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
int sameCnt = 0;
for (int i = 0; i < mergedList.size(); i++) {
if (mergedList.get(i).z == results.get(i).z) {
@@ -201,31 +222,30 @@ public class Visval_standard {
}
}
System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
-
}
-// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842, 23089};
-// for (int i = 0; i < vlist.length; i++) {
-// polyline.addVertex(new vPoint(i, vlist[i]));
-// }
-
-// ArrayList<Double> v = new ArrayList<>();
-// String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
-// try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
-// String line;
-// int count = 0;
-// while ((line = br.readLine()) != null && count < 1000) {
-// String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
-// v.add(Double.parseDouble(columns[0])); // 读取第一列数据
-// count++;
-// }
-// } catch (Exception e) {
-// e.printStackTrace();
-// }
-// // 打印数据长度
-// System.out.println("Data length: " + v.size());
-// for (int i = 0; i < v.size(); i++) {
-// polyline.addVertex(new vPoint(i, v.get(i)));
-// }
+ // float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
+ // 23089};
+ // for (int i = 0; i < vlist.length; i++) {
+ // polyline.addVertex(new vPoint(i, vlist[i]));
+ // }
+
+ // ArrayList<Double> v = new ArrayList<>();
+ // String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
+ // try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+ // String line;
+ // int count = 0;
+ // while ((line = br.readLine()) != null && count < 1000) {
+ // String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
+ // v.add(Double.parseDouble(columns[0])); // 读取第一列数据
+ // count++;
+ // }
+ // } catch (Exception e) {
+ // e.printStackTrace();
+ // }
+ // // 打印数据长度
+ // System.out.println("Data length: " + v.size());
+ // for (int i = 0; i < v.size(); i++) {
+ // polyline.addVertex(new vPoint(i, v.get(i)));
+ // }
}
-
(iotdb) 41/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a815c1137dfb7896306cc056a1b4373fbe451e11
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Feb 3 22:17:42 2025 +0800
add
---
server/pom.xml | 74 +++++++++++++++++++++++++++++-----------------------------
1 file changed, 37 insertions(+), 37 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index 73af108a647..a37788e9d72 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -284,43 +284,43 @@
</execution>
</executions>
</plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <!-- <finalName>sample_ltd</finalName>-->
- <!-- <finalName>sample_swab_ad</finalName>-->
- <!-- <finalName>sample_swab</finalName>-->
- <!-- <finalName>sample_rdp</finalName>-->
- <!-- <finalName>sample_minmax</finalName>-->
- <!-- <finalName>sample_fsw</finalName>-->
- <finalName>sample_eBUG</finalName>
- <!-- <finalName>sample_BUYdiff</finalName>-->
- <archive>
- <manifest>
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_LTD</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWABAD</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>-->
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>-->
- </manifest>
- </archive>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
+ <!-- <plugin>-->
+ <!-- <artifactId>maven-assembly-plugin</artifactId>-->
+ <!-- <configuration>-->
+ <!-- <!– <finalName>sample_ltd</finalName>–>-->
+ <!-- <!– <finalName>sample_swab_ad</finalName>–>-->
+ <!-- <!– <finalName>sample_swab</finalName>–>-->
+ <!-- <!– <finalName>sample_rdp</finalName>–>-->
+ <!-- <!– <finalName>sample_minmax</finalName>–>-->
+ <!-- <!– <finalName>sample_fsw</finalName>–>-->
+ <!-- <finalName>sample_eBUG</finalName>-->
+ <!-- <!– <finalName>sample_BUYdiff</finalName>–>-->
+ <!-- <archive>-->
+ <!-- <manifest>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_LTD</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWABAD</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_SWAB</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_Rdp</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_MinMax</mainClass>–>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>–>-->
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>-->
+ <!-- <!– <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>–>-->
+ <!-- </manifest>-->
+ <!-- </archive>-->
+ <!-- <descriptorRefs>-->
+ <!-- <descriptorRef>jar-with-dependencies</descriptorRef>-->
+ <!-- </descriptorRefs>-->
+ <!-- </configuration>-->
+ <!-- <executions>-->
+ <!-- <execution>-->
+ <!-- <id>make-assembly</id>-->
+ <!-- <phase>package</phase>-->
+ <!-- <goals>-->
+ <!-- <goal>single</goal>-->
+ <!-- </goals>-->
+ <!-- </execution>-->
+ <!-- </executions>-->
+ <!-- </plugin>-->
</plugins>
</build>
<profiles>
(iotdb) 31/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit eda41e9a27f5b602076b1bb84355e0e71292843b
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jan 28 02:21:29 2025 +0800
add
---
server/pom.xml | 8 ++++----
server/sample_fsw-jar-with-dependencies.jar | Bin 38729071 -> 38729071 bytes
.../java/org/apache/iotdb/db/query/eBUG/Tool.java | 2 +-
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index 8ea227c6cd3..f918a874eb2 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -281,13 +281,13 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
- <!-- <finalName>sample_fsw</finalName>-->
- <finalName>sample_eBUG</finalName>
+ <finalName>sample_fsw</finalName>
+ <!-- <finalName>sample_eBUG</finalName>-->
<!-- <finalName>sample_BUYdiff</finalName>-->
<archive>
<manifest>
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>
+ <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>-->
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpYdiff</mainClass>-->
</manifest>
</archive>
diff --git a/server/sample_fsw-jar-with-dependencies.jar b/server/sample_fsw-jar-with-dependencies.jar
index 3ca5857146d..fedc6291223 100644
Binary files a/server/sample_fsw-jar-with-dependencies.jar and b/server/sample_fsw-jar-with-dependencies.jar differ
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
index f2e2c0d68d9..e430b5184d0 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
@@ -99,7 +99,7 @@ public class Tool {
boolean directMore = false;
int iterNum = 0;
- int maxIterNum = 50;
+ int maxIterNum = 100;
// First binary search loop to find the initial range
double base = 2;
(iotdb) 16/44: array
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 65d259e51114a60ec20cda67423df30c41b18de5
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jan 22 14:48:53 2025 +0800
array
---
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 57 ++++++++-------
...d.java => eBUG_sortedEliminatedDeprecated.java} | 60 +++++++--------
...va => eBUG_sortedEliminatedDeprecated_tmp.java} | 85 +++++++++++-----------
.../org/apache/iotdb/db/query/eBUG/eBUG_tmp.java | 83 +++++++++++----------
4 files changed, 143 insertions(+), 142 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 1596f32a726..d4a8ebfc36f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -19,7 +19,9 @@
package org.apache.iotdb.db.query.eBUG;
-import java.io.*;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
import java.util.*;
import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
@@ -27,9 +29,12 @@ import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
public class eBUG {
- public static List<Point> findEliminated(Polyline lineToSimplify, TimestampPriorityQueue recentEliminatedOrdered,
+ public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
int pa_idx, int pi_idx, int pb_idx) {
// TODO 复杂度分析
+ // 从最近淘汰的e个点里寻找位于pa~pb之间的：e
+ // 把上一步找到的点和pa pi pb一起按照时间戳递增排序：记c=b-a。如果e<c，那么eloge，否则c
+ // 后面鞋带公式计算面积：min(e+3,c)
// 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
// pi: the point whose significance needs updating
// pa: left adjacent non-eliminated point of pi
@@ -51,7 +56,7 @@ public class eBUG {
// return res; // pa pi pb已经按照时间戳递增
// }
- if (recentEliminatedOrdered.size() >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
+ if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
return res; // 已经按照时间戳递增
@@ -60,26 +65,23 @@ public class eBUG {
// 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
// System.out.println("k>=[c=(b-a-2)]>e, search among e");
res.add(pa);
+ res.add(pi);
+ res.add(pb);
// 包括了c>e=0的情况
- boolean addPi = false;
- for (Point p : recentEliminatedOrdered.getQueue()) { // 已经按时间戳递增排序，为了后面计算total areal displacement需要
- if (p.x <= pa.x) {
- continue;
- }
- if (p.x >= pb.x) {
+ for (int idx : recentEliminated) { // e points
+ if (idx == 0) { // init, not filled by real eliminated points yet
break;
}
- if (p.x > pi.x && !addPi) {
- res.add(pi);
- addPi = true;
+ Point p = lineToSimplify.get(idx);
+ if (p.x > pa.x && p.x < pb.x) {
+ res.add(p);
}
- res.add(p);
- }
- if (!addPi) { // may happen when no eliminated points found above
- res.add(pi);
}
- res.add(pb);
+
+ // 按照时间戳递增排序，这是为了后面计算total areal displacement需要
+ res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 TODO 这样的话时间复杂度要变成e*log(e)了吧？
+
return res;
}
@@ -92,9 +94,8 @@ public class eBUG {
// results.addAll(lineToSimplify.getVertices()); // TODO debug
// TODO 需要一个东西来记录最近淘汰的点依次是谁
-// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
-// int recentEliminatedIdx = 0; // index in the array, circulate
- TimestampPriorityQueue recentEliminatedOrdered = new TimestampPriorityQueue(e);
+ int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
+ int recentEliminatedIdx = 0; // index in the array, circulate
int total = lineToSimplify.size();
if (total < 3) {
@@ -142,12 +143,11 @@ public class eBUG {
System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
}
if (e > 0) { // otherwise e=0
-// recentEliminated[recentEliminatedIdx] = tri.indices[1];
-// recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
- recentEliminatedOrdered.insert(lineToSimplify.get(tri.indices[1]));
+ recentEliminated[recentEliminatedIdx] = tri.indices[1];
+ recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
}
if (debug) {
- System.out.println("the e most recently eliminated points:" + recentEliminatedOrdered);
+ System.out.println("the e most recently eliminated points:" + Arrays.toString(recentEliminated));
}
// 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
@@ -176,7 +176,7 @@ public class eBUG {
tri.prev.indices[2] = tri.indices[2];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
tri.prev.indices[0],
tri.prev.indices[1],
tri.prev.indices[2]); // TODO complexity ?
@@ -225,7 +225,7 @@ public class eBUG {
tri.next.indices[0] = tri.indices[0];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
tri.next.indices[0],
tri.next.indices[1],
tri.next.indices[2]); // TODO complexity
@@ -298,8 +298,9 @@ public class eBUG {
// List<Point> results = new ArrayList<>();
// 计算运行时间
// int eParam = 10;
- try (PrintWriter writer = new PrintWriter(new File("exp2.csv"))) {
- int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000};
+ try (PrintWriter writer = new PrintWriter(new File("exp.csv"))) {
+ int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 30_0000, 50_0000, 60_0000,
+ 70_0000, 80_0000, 90_0000, 100_0000, 150_0000, 200_0000, 250_0000, 300_0000};
// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
for (int eParam : eParamList) {
long startTime = System.currentTimeMillis();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated.java
similarity index 91%
rename from server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old.java
rename to server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated.java
index 180f277c44e..61c508added 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated.java
@@ -19,22 +19,17 @@
package org.apache.iotdb.db.query.eBUG;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.PrintWriter;
+import java.io.*;
import java.util.*;
import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-public class eBUG_old {
- public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
+public class eBUG_sortedEliminatedDeprecated {
+ public static List<Point> findEliminated(Polyline lineToSimplify, TimestampPriorityQueue recentEliminatedOrdered,
int pa_idx, int pi_idx, int pb_idx) {
// TODO 复杂度分析
- // 从最近淘汰的e个点里寻找位于pa~pb之间的：e
- // 把上一步找到的点和pa pi pb一起按照时间戳递增排序：记c=b-a。如果e<c，那么eloge，否则c
- // 后面鞋带公式计算面积：min(e+3,c)
// 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
// pi: the point whose significance needs updating
// pa: left adjacent non-eliminated point of pi
@@ -56,7 +51,7 @@ public class eBUG_old {
// return res; // pa pi pb已经按照时间戳递增
// }
- if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
+ if (recentEliminatedOrdered.size() >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
return res; // 已经按照时间戳递增
@@ -65,23 +60,26 @@ public class eBUG_old {
// 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
// System.out.println("k>=[c=(b-a-2)]>e, search among e");
res.add(pa);
- res.add(pi);
- res.add(pb);
// 包括了c>e=0的情况
- for (int idx : recentEliminated) { // e points
- if (idx == 0) { // init, not filled by real eliminated points yet
+ boolean addPi = false;
+ for (Point p : recentEliminatedOrdered.getQueue()) { // 已经按时间戳递增排序，为了后面计算total areal displacement需要
+ if (p.x <= pa.x) {
+ continue;
+ }
+ if (p.x >= pb.x) {
break;
}
- Point p = lineToSimplify.get(idx);
- if (p.x > pa.x && p.x < pb.x) {
- res.add(p);
+ if (p.x > pi.x && !addPi) {
+ res.add(pi);
+ addPi = true;
}
+ res.add(p);
}
-
- // 按照时间戳递增排序，这是为了后面计算total areal displacement需要
- res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 TODO 这样的话时间复杂度要变成e*log(e)了吧？
-
+ if (!addPi) { // may happen when no eliminated points found above
+ res.add(pi);
+ }
+ res.add(pb);
return res;
}
@@ -94,8 +92,9 @@ public class eBUG_old {
// results.addAll(lineToSimplify.getVertices()); // TODO debug
// TODO 需要一个东西来记录最近淘汰的点依次是谁
- int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
- int recentEliminatedIdx = 0; // index in the array, circulate
+// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
+// int recentEliminatedIdx = 0; // index in the array, circulate
+ TimestampPriorityQueue recentEliminatedOrdered = new TimestampPriorityQueue(e);
int total = lineToSimplify.size();
if (total < 3) {
@@ -143,11 +142,12 @@ public class eBUG_old {
System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
}
if (e > 0) { // otherwise e=0
- recentEliminated[recentEliminatedIdx] = tri.indices[1];
- recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
+// recentEliminated[recentEliminatedIdx] = tri.indices[1];
+// recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
+ recentEliminatedOrdered.insert(lineToSimplify.get(tri.indices[1]));
}
if (debug) {
- System.out.println("the e most recently eliminated points:" + Arrays.toString(recentEliminated));
+ System.out.println("the e most recently eliminated points:" + recentEliminatedOrdered);
}
// 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
@@ -176,7 +176,7 @@ public class eBUG_old {
tri.prev.indices[2] = tri.indices[2];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
tri.prev.indices[0],
tri.prev.indices[1],
tri.prev.indices[2]); // TODO complexity ?
@@ -225,7 +225,7 @@ public class eBUG_old {
tri.next.indices[0] = tri.indices[0];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
tri.next.indices[0],
tri.next.indices[1],
tri.next.indices[2]); // TODO complexity
@@ -298,9 +298,9 @@ public class eBUG_old {
// List<Point> results = new ArrayList<>();
// 计算运行时间
// int eParam = 10;
- try (PrintWriter writer = new PrintWriter(new File("exp.csv"))) {
- int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 30_0000, 50_0000, 60_0000,
- 70_0000, 80_0000, 90_0000, 100_0000, 150_0000, 200_0000, 250_0000, 300_0000};
+ try (PrintWriter writer = new PrintWriter(new File("exp2.csv"))) {
+ int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 50_0000, 60_0000, 70_0000,
+ 80_0000, 90_0000, 100_0000, 150_0000, 200_0000};
// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
for (int eParam : eParamList) {
long startTime = System.currentTimeMillis();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old_tmp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated_tmp.java
similarity index 88%
rename from server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old_tmp.java
rename to server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated_tmp.java
index 5cfafb59110..4d933cfcd2e 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old_tmp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_sortedEliminatedDeprecated_tmp.java
@@ -27,13 +27,10 @@ import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-public class eBUG_old_tmp {
- public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
+public class eBUG_sortedEliminatedDeprecated_tmp {
+ public static List<Point> findEliminated(Polyline lineToSimplify, TimestampPriorityQueue recentEliminatedOrdered,
int pa_idx, int pi_idx, int pb_idx) {
// TODO 复杂度分析
- // 从最近淘汰的e个点里寻找位于pa~pb之间的：e
- // 把上一步找到的点和pa pi pb一起按照时间戳递增排序：记c=b-a。如果e<c，那么eloge，否则c
- // 后面鞋带公式计算面积：min(e+3,c)
// 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
// pi: the point whose significance needs updating
// pa: left adjacent non-eliminated point of pi
@@ -55,7 +52,7 @@ public class eBUG_old_tmp {
// return res; // pa pi pb已经按照时间戳递增
// }
- if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
+ if (recentEliminatedOrdered.size() >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
return res; // 已经按照时间戳递增
@@ -64,23 +61,26 @@ public class eBUG_old_tmp {
// 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
// System.out.println("k>=[c=(b-a-2)]>e, search among e");
res.add(pa);
- res.add(pi);
- res.add(pb);
// 包括了c>e=0的情况
- for (int idx : recentEliminated) { // e points
- if (idx == 0) { // init, not filled by real eliminated points yet
+ boolean addPi = false;
+ for (Point p : recentEliminatedOrdered.getQueue()) { // 已经按时间戳递增排序，为了后面计算total areal displacement需要
+ if (p.x <= pa.x) {
+ continue;
+ }
+ if (p.x >= pb.x) {
break;
}
- Point p = lineToSimplify.get(idx);
- if (p.x > pa.x && p.x < pb.x) {
- res.add(p);
+ if (p.x > pi.x && !addPi) {
+ res.add(pi);
+ addPi = true;
}
+ res.add(p);
}
-
- // 按照时间戳递增排序，这是为了后面计算total areal displacement需要
- res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 TODO 这样的话时间复杂度要变成e*log(e)了吧？
-
+ if (!addPi) { // may happen when no eliminated points found above
+ res.add(pi);
+ }
+ res.add(pb);
return res;
}
@@ -93,8 +93,9 @@ public class eBUG_old_tmp {
// results.addAll(lineToSimplify.getVertices()); // TODO debug
// TODO 需要一个东西来记录最近淘汰的点依次是谁
- int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
- int recentEliminatedIdx = 0; // index in the array, circulate
+// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
+// int recentEliminatedIdx = 0; // index in the array, circulate
+ TimestampPriorityQueue recentEliminatedOrdered = new TimestampPriorityQueue(e);
int total = lineToSimplify.size();
if (total < 3) {
@@ -142,11 +143,12 @@ public class eBUG_old_tmp {
System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
}
if (e > 0) { // otherwise e=0
- recentEliminated[recentEliminatedIdx] = tri.indices[1];
- recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
+// recentEliminated[recentEliminatedIdx] = tri.indices[1];
+// recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
+ recentEliminatedOrdered.insert(lineToSimplify.get(tri.indices[1]));
}
if (debug) {
- System.out.println("the e most recently eliminated points:" + Arrays.toString(recentEliminated));
+ System.out.println("the e most recently eliminated points:" + recentEliminatedOrdered);
}
// 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
@@ -175,7 +177,7 @@ public class eBUG_old_tmp {
tri.prev.indices[2] = tri.indices[2];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
tri.prev.indices[0],
tri.prev.indices[1],
tri.prev.indices[2]); // TODO complexity ?
@@ -224,7 +226,7 @@ public class eBUG_old_tmp {
tri.next.indices[0] = tri.indices[0];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
tri.next.indices[0],
tri.next.indices[1],
tri.next.indices[2]); // TODO complexity
@@ -264,6 +266,7 @@ public class eBUG_old_tmp {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
+// int n = 1000_0000;
int n = 10000;
int p = 10;
@@ -279,31 +282,29 @@ public class eBUG_old_tmp {
polylineList.add(polylineBatch);
}
-// try (FileWriter writer = new FileWriter("raw.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < polyline.size(); i++) {
-// Point point = polyline.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < polyline.size(); i++) {
+ Point point = polyline.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
System.out.println("---------------------------------");
// List<Point> results = new ArrayList<>();
// 计算运行时间
-// int eParam = 10;
-// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
- int eParam = 22;
+ int eParam = 0;
long startTime = System.currentTimeMillis();
List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
long endTime = System.currentTimeMillis();
- System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
System.out.println(results.size());
if (results.size() <= 100) {
@@ -314,7 +315,7 @@ public class eBUG_old_tmp {
}
}
- try (FileWriter writer = new FileWriter("fast2.csv")) {
+ try (FileWriter writer = new FileWriter("fast.csv")) {
// 写入CSV头部
writer.append("x,y,z\n");
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_tmp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_tmp.java
index a04c4c17920..93267df65a7 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_tmp.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_tmp.java
@@ -28,9 +28,12 @@ import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
public class eBUG_tmp {
- public static List<Point> findEliminated(Polyline lineToSimplify, TimestampPriorityQueue recentEliminatedOrdered,
+ public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
int pa_idx, int pi_idx, int pb_idx) {
// TODO 复杂度分析
+ // 从最近淘汰的e个点里寻找位于pa~pb之间的：e
+ // 把上一步找到的点和pa pi pb一起按照时间戳递增排序：记c=b-a。如果e<c，那么eloge，否则c
+ // 后面鞋带公式计算面积：min(e+3,c)
// 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
// pi: the point whose significance needs updating
// pa: left adjacent non-eliminated point of pi
@@ -52,7 +55,7 @@ public class eBUG_tmp {
// return res; // pa pi pb已经按照时间戳递增
// }
- if (recentEliminatedOrdered.size() >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
+ if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
return res; // 已经按照时间戳递增
@@ -61,26 +64,23 @@ public class eBUG_tmp {
// 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
// System.out.println("k>=[c=(b-a-2)]>e, search among e");
res.add(pa);
+ res.add(pi);
+ res.add(pb);
// 包括了c>e=0的情况
- boolean addPi = false;
- for (Point p : recentEliminatedOrdered.getQueue()) { // 已经按时间戳递增排序，为了后面计算total areal displacement需要
- if (p.x <= pa.x) {
- continue;
- }
- if (p.x >= pb.x) {
+ for (int idx : recentEliminated) { // e points
+ if (idx == 0) { // init, not filled by real eliminated points yet
break;
}
- if (p.x > pi.x && !addPi) {
- res.add(pi);
- addPi = true;
+ Point p = lineToSimplify.get(idx);
+ if (p.x > pa.x && p.x < pb.x) {
+ res.add(p);
}
- res.add(p);
- }
- if (!addPi) { // may happen when no eliminated points found above
- res.add(pi);
}
- res.add(pb);
+
+ // 按照时间戳递增排序，这是为了后面计算total areal displacement需要
+ res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 TODO 这样的话时间复杂度要变成e*log(e)了吧？
+
return res;
}
@@ -93,9 +93,8 @@ public class eBUG_tmp {
// results.addAll(lineToSimplify.getVertices()); // TODO debug
// TODO 需要一个东西来记录最近淘汰的点依次是谁
-// int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
-// int recentEliminatedIdx = 0; // index in the array, circulate
- TimestampPriorityQueue recentEliminatedOrdered = new TimestampPriorityQueue(e);
+ int[] recentEliminated = new int[e]; // init 0 points to the first point, skipped in findEliminated
+ int recentEliminatedIdx = 0; // index in the array, circulate
int total = lineToSimplify.size();
if (total < 3) {
@@ -143,12 +142,11 @@ public class eBUG_tmp {
System.out.println("(1) eliminate " + lineToSimplify.get(tri.indices[1]));
}
if (e > 0) { // otherwise e=0
-// recentEliminated[recentEliminatedIdx] = tri.indices[1];
-// recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
- recentEliminatedOrdered.insert(lineToSimplify.get(tri.indices[1]));
+ recentEliminated[recentEliminatedIdx] = tri.indices[1];
+ recentEliminatedIdx = (recentEliminatedIdx + 1) % e; // 0~e-1 circulate
}
if (debug) {
- System.out.println("the e most recently eliminated points:" + recentEliminatedOrdered);
+ System.out.println("the e most recently eliminated points:" + Arrays.toString(recentEliminated));
}
// 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
@@ -177,7 +175,7 @@ public class eBUG_tmp {
tri.prev.indices[2] = tri.indices[2];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
tri.prev.indices[0],
tri.prev.indices[1],
tri.prev.indices[2]); // TODO complexity ?
@@ -226,7 +224,7 @@ public class eBUG_tmp {
tri.next.indices[0] = tri.indices[0];
// e parameter
- List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminatedOrdered,
+ List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
tri.next.indices[0],
tri.next.indices[1],
tri.next.indices[2]); // TODO complexity
@@ -266,7 +264,6 @@ public class eBUG_tmp {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
-// int n = 1000_0000;
int n = 10000;
int p = 10;
@@ -282,29 +279,31 @@ public class eBUG_tmp {
polylineList.add(polylineBatch);
}
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+// try (FileWriter writer = new FileWriter("raw.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
+//
+// // 写入每个点的数据
+// for (int i = 0; i < polyline.size(); i++) {
+// Point point = polyline.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// }
+// System.out.println("Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
+// }
System.out.println("---------------------------------");
// List<Point> results = new ArrayList<>();
// 计算运行时间
- int eParam = 0;
+// int eParam = 10;
+// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
+ int eParam = 22;
long startTime = System.currentTimeMillis();
List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
long endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
System.out.println(results.size());
if (results.size() <= 100) {
@@ -315,7 +314,7 @@ public class eBUG_tmp {
}
}
- try (FileWriter writer = new FileWriter("fast.csv")) {
+ try (FileWriter writer = new FileWriter("fast2.csv")) {
// 写入CSV头部
writer.append("x,y,z\n");
(iotdb) 04/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a3702eef02af92948dd0981140d5ebe31a5ed047
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Dec 20 01:10:24 2024 +0800
add
---
.../iotdb/db/query/simpiece/Visval_standard.java | 50 +++++++++++-----------
1 file changed, 25 insertions(+), 25 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
index 095a8e234f7..b5033b86097 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
@@ -148,7 +148,7 @@ public class Visval_standard {
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random();
int n = 1000_000;
- int p = 1000;
+ int p = 10;
for (int i = 0; i < n; i += p) {
Polyline polylineBatch = new Polyline();
for (int j = i; j < Math.min(i + p, n); j++) {
@@ -159,30 +159,6 @@ public class Visval_standard {
polylineList.add(polylineBatch);
}
-// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842, 23089};
-// for (int i = 0; i < vlist.length; i++) {
-// polyline.addVertex(new vPoint(i, vlist[i]));
-// }
-
-// ArrayList<Double> v = new ArrayList<>();
-// String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
-// try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
-// String line;
-// int count = 0;
-// while ((line = br.readLine()) != null && count < 1000) {
-// String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
-// v.add(Double.parseDouble(columns[0])); // 读取第一列数据
-// count++;
-// }
-// } catch (Exception e) {
-// e.printStackTrace();
-// }
-// // 打印数据长度
-// System.out.println("Data length: " + v.size());
-// for (int i = 0; i < v.size(); i++) {
-// polyline.addVertex(new vPoint(i, v.get(i)));
-// }
-
System.out.println("---------------------------------");
List<vPoint> results = new ArrayList<>();
// 计算运行时间
@@ -227,5 +203,29 @@ public class Visval_standard {
System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
}
+
+// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842, 23089};
+// for (int i = 0; i < vlist.length; i++) {
+// polyline.addVertex(new vPoint(i, vlist[i]));
+// }
+
+// ArrayList<Double> v = new ArrayList<>();
+// String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
+// try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+// String line;
+// int count = 0;
+// while ((line = br.readLine()) != null && count < 1000) {
+// String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
+// v.add(Double.parseDouble(columns[0])); // 读取第一列数据
+// count++;
+// }
+// } catch (Exception e) {
+// e.printStackTrace();
+// }
+// // 打印数据长度
+// System.out.println("Data length: " + v.size());
+// for (int i = 0; i < v.size(); i++) {
+// polyline.addVertex(new vPoint(i, v.get(i)));
+// }
}
(iotdb) 18/44: before
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1f328cbe744f4c10751adafd38c67d68cd8a8875
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jan 22 21:59:37 2025 +0800
before
---
.../org/apache/iotdb/db/query/eBUG/Polyline.java | 4 +
.../java/org/apache/iotdb/db/query/eBUG/Tmp.java | 41 +++++
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 185 ++++++---------------
3 files changed, 97 insertions(+), 133 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
index 9fa11b192cb..bed14731832 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Polyline.java
@@ -24,4 +24,8 @@ public class Polyline {
public Point get(int index) {
return vertices.get(index);
}
+
+ public void clear() {
+ vertices.clear();
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java
new file mode 100644
index 00000000000..ad95ee6ec51
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tmp.java
@@ -0,0 +1,41 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
+import java.util.Comparator;
+import java.util.PriorityQueue;
+import java.util.Random;
+
+public class Tmp {
+ public static void main(String[] args) {
+ int seed = 10;
+ Random rand = new Random(seed);
+ int eParam = 1;
+ try (PrintWriter writer = new PrintWriter(new File("tmp.csv"))) {
+ for (int n = 100_0000; n < 100000_0000; n += 5000_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
+// for (int n = 19000000; n < 3000_0000; n +=200_0000) {
+ PriorityQueue<Point> heap = new PriorityQueue<>(Comparator.comparingDouble(t -> t.y));
+ for (int i = 0; i < n; i++) {
+ double v = rand.nextInt(1000000);
+ heap.add(new Point(i, v));
+ }
+
+ long startTime = System.currentTimeMillis();
+ long sum = 0;
+ while (!heap.isEmpty()) {
+ Point p = heap.poll();
+ sum += p.x;
+ }
+ long endTime = System.currentTimeMillis();
+ System.out.println(n + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+ writer.println(n + "," + "0" + "," + (endTime - startTime));
+ System.out.println(sum);
+ }
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+
+ System.out.println("finish");
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 8c4d41f37f0..77e21a27067 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -49,23 +49,17 @@ public class eBUG {
List<Point> res = new ArrayList<>();
-// if (recentEliminated.length == 0) { // e=0
-// System.out.println("e=0");
-// res.add(pa);
-// res.add(pi);
-// res.add(pb);
-// return res; // pa pi pb已经按照时间戳递增
-// }
-
+ // TODO note: here may happen because e=c=0 or just 0<c<=e
if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
return res; // 已经按照时间戳递增
}
+ // TODO note: here may happen because c>e=0 or c>e>0. the latter needs search among recentEliminated and sort by timestamp
// 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
// System.out.println("k>=[c=(b-a-2)]>e, search among e");
- res.add(pa);
+ res.add(pa); // 注意pa pi pb已经按照时间戳递增
res.add(pi);
res.add(pb);
@@ -81,7 +75,9 @@ public class eBUG {
}
// 按照时间戳递增排序，这是为了后面计算total areal displacement需要
- res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 这样的话时间复杂度变成e*log(e)
+ if (recentEliminated.length > 0) { // 否则e=0，不需要排序pa pi pb三个点
+ res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 这样的话时间复杂度变成e*log(e)
+ }
return res;
}
@@ -168,7 +164,7 @@ public class eBUG {
tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
-// tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
+ // tri.prev = newPre; // can omit, because already done by new Triangle(tri.prev)
// 2. 处理pi被淘汰引起tri.prev被更新的事情
// 前一个三角形连到后一个三角形
@@ -214,7 +210,7 @@ public class eBUG {
tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
-// tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
+ // tri.next = newNext; // omit, because already done by new Triangle(tri.prev)
if (tri.prev != null) {
tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
@@ -262,145 +258,68 @@ public class eBUG {
}
public static void main(String[] args) {
- Polyline polyline = new Polyline();
- List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random(1);
- int n = 1000_0000;
-
- int p = 10;
- for (int i = 0; i < n; i += p) {
- Polyline polylineBatch = new Polyline();
- for (int j = i; j < Math.min(i + p, n); j++) {
- double v = rand.nextInt(1000000);
-
- polyline.addVertex(new Point(j, v));
-
- polylineBatch.addVertex(new Point(j, v));
- }
- polylineList.add(polylineBatch);
- }
-// try (FileWriter writer = new FileWriter("raw.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < polyline.size(); i++) {
-// Point point = polyline.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
+ int eParam = 2000_0000;
+ try (PrintWriter writer = new PrintWriter(new File("exp_varyN_e" + eParam + ".csv"))) {
+ for (int n = 100_0000; n < 2000_0000; n += 200_0000) { // 超过两千万就都变得慢多了，感觉可能是内存有限的原因，而不是算法复杂度
+ // TODO 注意 要有一个点是n=300w
+
+ Polyline polyline = new Polyline();
+ int seed = 1;
+ Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+ for (int i = 0; i < n; i++) {
+ double v = rand.nextInt(1000000);
+ polyline.addVertex(new Point(i, v));
+ }
- System.out.println("---------------------------------");
-// List<Point> results = new ArrayList<>();
- // 计算运行时间
-// int eParam = 10;
- try (PrintWriter writer = new PrintWriter(new File("exp.csv"))) {
-// int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 30_0000, 50_0000, 60_0000,
-// 70_0000, 80_0000, 90_0000, 100_0000, 150_0000, 200_0000, 250_0000, 300_0000};
- for (int eParam = 0; eParam < n*1.5; eParam += 2_0000) {
-// for (int eParam : eParamList) {
long startTime = System.currentTimeMillis();
List<Point> results = buildEffectiveArea(polyline, eParam, false);
- // 输出结果
long endTime = System.currentTimeMillis();
+
System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(results.size());
writer.println(n + "," + eParam + "," + (endTime - startTime));
+ System.out.println(results.size());
+
+ // clear
+ polyline.clear();
+ results.clear();
+ polyline = null;
+ results = null;
+ System.gc();
}
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
-// if (results.size() <= 100) {
-// System.out.println("+++++++++++++++++++");
-// for (int i = 0; i < results.size(); i++) {
-// Point point = results.get(i);
-// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
-// }
+// int n = 300_0000;
+// int seed = 1;
+// Random rand = new Random(seed); // TODO 注意seed在每轮里重设
+// Polyline polyline = new Polyline(); // 注意：如果是固定n变化e实验，这个数据要一次性生成，不然每次随机不一样
+// for (int i = 0; i < n; i++) {
+// double v = rand.nextInt(1000000);
+// polyline.addVertex(new Point(i, v));
// }
-
-// try (FileWriter writer = new FileWriter("fast.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
//
-// // 写入每个点的数据
-// for (int i = 0; i < results.size(); i++) {
-// Point point = results.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
-
-// System.out.println("---------------------------------");
-// List<List<Point>> resultsBatchList = new ArrayList<>();
-// // 计算运行时间
-// int cnt = 0;
-// startTime = System.currentTimeMillis();
-// for (Polyline polylineBatch : polylineList) {
-// List<Point> resultsBatch = new ArrayList<>();
-// buildEffectiveArea(polylineBatch, resultsBatch);
-// cnt += resultsBatch.size();
-// resultsBatchList.add(resultsBatch);
-// }
-// // 输出结果
-// endTime = System.currentTimeMillis();
-// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
-// System.out.println(cnt);
+// try (PrintWriter writer = new PrintWriter(new File("exp_varyE_n" + n + ".csv"))) {
+// int[] eParamList = {0, 1, 2, 10, 1000, 10000, 10_0000, 30_0000, 50_0000, 60_0000, 80_0000, 100_0000, 150_0000, 200_0000};
+// for (int eParam : eParamList) {
+// eParam *= 3; // TODO 注意 因为n=300w而不是100w
//
-// System.out.println("---------------------------------");
-// // 使用 Stream API 合并所有列表
-// List<Point> mergedList = resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
-// int sameCnt = 0;
-// for (int i = 0; i < mergedList.size(); i++) {
-// if (mergedList.get(i).z == results.get(i).z) {
-// sameCnt += 1;
-// }
-// }
-// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
+// long startTime = System.currentTimeMillis();
+// List<Point> results = buildEffectiveArea(polyline, eParam, false);
+// long endTime = System.currentTimeMillis();
//
-// try (FileWriter writer = new FileWriter("batch.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
+// System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+// writer.println(n + "," + eParam + "," + (endTime - startTime));
+// System.out.println(results.size());
//
-// // 写入每个点的数据
-// for (int i = 0; i < mergedList.size(); i++) {
-// Point point = mergedList.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// // clear 注意不能把原始数据清除，还要接着用
+// System.gc();
// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
+// } catch (FileNotFoundException e) {
+// throw new RuntimeException(e);
// }
- }
- // float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
- // 23089};
- // for (int i = 0; i < vlist.length; i++) {
- // polyline.addVertex(new vPoint(i, vlist[i]));
- // }
-
- // ArrayList<Double> v = new ArrayList<>();
- // String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
- // try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
- // String line;
- // int count = 0;
- // while ((line = br.readLine()) != null && count < 1000) {
- // String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
- // v.add(Double.parseDouble(columns[0])); // 读取第一列数据
- // count++;
- // }
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // // 打印数据长度
- // System.out.println("Data length: " + v.size());
- // for (int i = 0; i < v.size(); i++) {
- // polyline.addVertex(new vPoint(i, v.get(i)));
- // }
+ System.out.println("finish");
+ }
}
(iotdb) 29/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2a22359d4bccd402a123e0bc254c0da0e9a1eafb
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jan 28 00:38:18 2025 +0800
add
---
.../java/org/apache/iotdb/db/query/eBUG/DNSL1.java | 97 ++++++++++++++++++++++
.../java/org/apache/iotdb/db/query/eBUG/DP.java | 57 +++++++------
.../org/apache/iotdb/db/query/eBUG/Test10.java | 62 ++++++++++++++
3 files changed, 191 insertions(+), 25 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
new file mode 100644
index 00000000000..df2c09d35ab
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
@@ -0,0 +1,97 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.DP.dynamic_programming;
+
+public class DNSL1 {
+ // 默认使用linear interpolation连接分段首尾点来近似
+ // 默认使用L1 error
+ // 使用divide and conquer算法
+ public static List<Point> reducePoints(List<Point> points, int m, boolean debug) {
+ int n = points.size();
+ int k = m - 1; // 分段数
+ int intervalPts = (int) Math.pow((double) n / k, (double) 2 / 3); // divide batch点数
+
+ if (debug) {
+ System.out.println("interval point length=" + intervalPts);
+ }
+
+ // divide into intervalPts equal-length intervals
+ List<Point> allSampledPoints = new ArrayList<>();
+ for (int start = 0; start < n; start += intervalPts) {
+ int end = Math.min(start + intervalPts, n);
+ List<Point> batch = points.subList(start, end); // 左闭右开
+ if (debug) {
+ System.out.println("Processing batch: [" + start + "," + end + ")");
+ }
+ List<Point> sampledForBatch = dynamic_programming(batch, k, DP.ERROR.L1, false);
+ allSampledPoints.addAll(sampledForBatch); // 把上一步的结果加到大的容器里
+ }
+
+ // 在大的容器上最后执行DP.dynamic_programming得到最后的m个采样点
+ if (debug) {
+ System.out.println("number of points from batch sampling: " + allSampledPoints.size());
+ }
+ List<Point> finalSampledPoints = dynamic_programming(allSampledPoints, k, DP.ERROR.L1, false);
+ if (debug) {
+ System.out.println("result point number: " + finalSampledPoints.size());
+ }
+ return finalSampledPoints;
+ }
+
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "raw.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 1000;
+// List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+// int m = (int) Math.pow(points.size(), 0.5);
+ int m = 10;
+
+// long startTime = System.currentTimeMillis();
+// List<Point> sampled = dynamic_programming(points, m - 1, DP.ERROR.area, false);
+// long endTime = System.currentTimeMillis();
+// System.out.println("Time taken: " + (endTime - startTime) + "ms");
+// for (Point p : sampled) {
+// System.out.println(p);
+// }
+// System.out.println(sampled.size());
+
+ long startTime2 = System.currentTimeMillis();
+// System.out.println(points.size() * m / (Math.pow(points.size() * 1.0 / (m - 1), 2 / 3.0)));
+// System.out.println(10000000 * (2 / (Math.pow(10000000 * 1.0 / (2 - 1), 2 / 3.0))));
+ List<Point> sampled2 = reducePoints(points, m, true);
+ long endTime2 = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime2 - startTime2) + "ms");
+// for (Point p : sampled2) {
+// System.out.println(p);
+// }
+// System.out.println(sampled2.size());
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java
index a6a9647fb4b..62c81cd10ea 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java
@@ -19,6 +19,7 @@ public class DP { // Dynamic-Programming
public static double[][] prepareCostTable(List<Point> points, ERROR errorType, boolean debug) {
int N = points.size();
double[][] dists = new double[N][N]; // 默认0，对角线元素已经都是0了
+ // 一个 double[47700][47700] 矩阵大约需要 16.94 GB 的内存。
// 外层循环i：range长度=right boundary-left boundary
// 内层循环j: 矩阵逐行
@@ -89,12 +90,18 @@ public class DP { // Dynamic-Programming
System.out.println(">>>分段数i+1" + (i + 1) + ",end pos j=" + j);
}
double[] choices = new double[j + 1]; // java从0开始
+ int bestIndex = -1;
+ double bestVal = Double.MAX_VALUE; // 越小越好
for (int xtmp = 0; xtmp < j + 1; xtmp++) {
if (errorType == ERROR.L_infy) {
choices[xtmp] = Math.max(kSegDist[i - 1][xtmp], dists[xtmp][j]);
} else {
choices[xtmp] = kSegDist[i - 1][xtmp] + dists[xtmp][j];
}
+ if (choices[xtmp] < bestVal) {
+ bestVal = choices[xtmp];
+ bestIndex = xtmp;
+ }
}
if (debug) {
for (int xtmp = 0; xtmp < j + 1; xtmp++) { // 遍历从 0 到 j 的每个元素
@@ -112,12 +119,8 @@ public class DP { // Dynamic-Programming
}
}
- int bestIndex = getIndexOfMin(choices);
- double bestVal = choices[bestIndex];
-
// Store the sub-problem solution
kSegPath[i][j] = bestIndex;
-
kSegDist[i][j] = bestVal;
if (debug) {
@@ -162,23 +165,27 @@ public class DP { // Dynamic-Programming
return sDs;
}
- // Helper method to get index of minimum value
- private static int getIndexOfMin(double[] array) {
- double minVal = array[0];
- int minIndex = 0;
- for (int i = 1; i < array.length; i++) {
- if (array[i] < minVal) {
- minVal = array[i];
- minIndex = i;
- }
- }
- return minIndex;
- }
+// // Helper method to get index of minimum value
+// private static int getIndexOfMin(double[] array) {
+// double minVal = array[0];
+// int minIndex = 0;
+// for (int i = 1; i < array.length; i++) {
+// if (array[i] < minVal) {
+// minVal = array[i];
+// minIndex = i;
+// }
+// }
+// return minIndex;
+// }
// Method to calculate joint segment error based on error type
private static double joint_segment_error(List<Point> points, int lx, int rx, ERROR errorType) {
// 默认joint=true, residual=true，即使用linear interpolation近似分段
// lx~rx 左闭右闭
+ if (lx == rx) {
+ return 0;
+ }
+
if (errorType != ERROR.area) {
double x1 = points.get(lx).x;
double y1 = points.get(lx).y;
@@ -240,14 +247,14 @@ public class DP { // Dynamic-Programming
boolean hasHeader = true;
int timeIdx = 0;
int valueIdx = 1;
- int N = -1;
- List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
-// Polyline polyline = new Polyline();
-// for (int i = 0; i < N; i += 1) {
-// double v = rand.nextInt(1000);
-// polyline.addVertex(new Point(i, v));
-// }
-// List<Point> points = polyline.getVertices();
+ int N = 100;
+// List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
try (FileWriter writer = new FileWriter("raw.csv")) {
// 写入CSV头部
writer.append("x,y,z\n");
@@ -263,7 +270,7 @@ public class DP { // Dynamic-Programming
long startTime = System.currentTimeMillis();
// double[][] ad2 = prepareKSegments(points, ERROR.L1, false);
- int m = 4;
+ int m = 10;
List<Point> sampled = dynamic_programming(points, m - 1, ERROR.area, false);
long endTime = System.currentTimeMillis();
System.out.println("Time taken: " + (endTime - startTime) + "ms");
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test10.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test10.java
new file mode 100644
index 00000000000..4dba96d0e3e
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Test10.java
@@ -0,0 +1,62 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.iotdb.db.query.eBUG.DP.dynamic_programming;
+
+public class Test10 {
+ // 用于验证java DP正确性
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "raw.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 100;
+// List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+
+ long startTime = System.currentTimeMillis();
+// double[][] ad2 = prepareKSegments(points, ERROR.L1, false);
+ int m = 10;
+ List<Point> sampled = dynamic_programming(points, m - 1, DP.ERROR.area, false);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ for (Point p : sampled) {
+ System.out.println(p);
+ }
+ System.out.println(sampled.size());
+
+// try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+// // 写入字符串
+// for (int i = 0; i < sampled.size(); i++) {
+// writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+// }
+//
+// } catch (FileNotFoundException e) {
+// e.printStackTrace();
+// }
+ }
+}
(iotdb) 38/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f2f408ba529c373284bcb12798dc0c08605d25ee
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jan 29 19:38:17 2025 +0800
add
---
server/sample_ltd-jar-with-dependencies.jar | Bin 40785602 -> 40794458 bytes
.../java/org/apache/iotdb/db/query/eBUG/LTD.java | 282 ++++++++++++++-------
.../org/apache/iotdb/db/query/eBUG/LTDBucket.java | 68 +++++
.../eBUG/{LTD.java => LTD_slow_deprecated.java} | 37 ++-
4 files changed, 279 insertions(+), 108 deletions(-)
diff --git a/server/sample_ltd-jar-with-dependencies.jar b/server/sample_ltd-jar-with-dependencies.jar
index f4b6c621844..db15b02bb3b 100644
Binary files a/server/sample_ltd-jar-with-dependencies.jar and b/server/sample_ltd-jar-with-dependencies.jar differ
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD.java
index 6eb2051411c..bd91813b720 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD.java
@@ -1,10 +1,7 @@
package org.apache.iotdb.db.query.eBUG;
import java.io.*;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
+import java.util.*;
public class LTD {
public static Point calculateAveragePoint(List<Point> points, int startClosed, int endClosed) {
@@ -52,108 +49,200 @@ public class LTD {
return sse;
}
- public static List<Integer> getLtdBinIdxs(List<Point> points, int m, int maxIter, boolean debug) {
- int numOfIterations = (int) (points.size() * 1.0 / m * 10);
+ public static List<Integer> getLtdBinIdxs(List<Point> points, int m, int maxIter, boolean debug)
+ throws IOException {
+ if (m < 6) {
+ throw new IOException("m at least 6");
+ }
+ int n = points.size();
+ int numOfIterations;
if (maxIter >= 0) {
- // numOfIterations = Math.min(numOfIterations, maxIter);
numOfIterations = maxIter; // overwrite
+ } else {
+ numOfIterations = (int) (n * 1.0 / m * 10);
}
- double blockSize = (points.size() - 3) * 1.0 / (m - 2);
+ if (debug) {
+ System.out.println("numOfIterations=" + numOfIterations);
+ }
+
+ int nbins = m - 2; // 留出全局首尾点
- List<Integer> offset = new LinkedList<>();
- for (double i = 1; i < points.size(); i += blockSize) {
+ double blockSize = (n - 3) * 1.0 / nbins;
+ List<Integer> offset = new LinkedList<>(); // nbins个分桶，nbins+1个桶边界
+ for (double i = 1; i < n; i += blockSize) {
offset.add((int) i); // 1~n-2, 这样最后一个offset+1才不会超出边界
}
if (debug) {
- System.out.println("numOfIterations=" + numOfIterations);
- System.out.println(offset);
+ System.out.println("init bins: " + offset);
}
- List<Double> sse = new LinkedList<>();
-
- // Initialization
- for (int i = 0; i < m - 2; i++) {
- // with one extra point overlapping for each adjacent bucket
- sse.add(calculateSSEForBucket(points, offset.get(i) - 1, offset.get(i + 1) + 1));
+ LTDBucket[] buckets = new LTDBucket[nbins];
+ double lastSSE = -1;
+ for (int i = 0; i < nbins - 1; i++) {
+ double sse;
+ if (i == 0) {
+ sse = calculateSSEForBucket(points, offset.get(i) - 1, offset.get(i + 1) + 1);
+ } else {
+ sse = lastSSE;
+ }
+ double sse_next = calculateSSEForBucket(points, offset.get(i + 1) - 1, offset.get(i + 2) + 1);
+ double sum = sse + sse_next;
+ buckets[i] = new LTDBucket(offset.get(i), offset.get(i + 1), sse, sum);
+ lastSSE = sse_next;
+ }
+ // the last bucket sumOf2SSE set as infinity meaning never merged
+ buckets[nbins - 1] =
+ new LTDBucket(offset.get(nbins - 1), offset.get(nbins), lastSSE, Double.MAX_VALUE);
+
+ // 设置前后关系
+ LTDBucket starter = new LTDBucket(0, 0, 0, 0);
+ starter.next = buckets[0];
+ for (int i = 0; i < nbins; i++) {
+ buckets[i].prev = (i == 0 ? starter : buckets[i - 1]);
+ buckets[i].next = (i == nbins - 1 ? null : buckets[i + 1]);
}
+ // System.out.println("begin creating heap...");
+
+ // 使用优先队列构建
+ PriorityQueue<LTDBucket> splitHeap =
+ new PriorityQueue<>((p1, p2) -> Double.compare(p2.sse, p1.sse));
+ // 越大的排在前面
+ Collections.addAll(splitHeap, buckets);
+ PriorityQueue<LTDBucket> mergeHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(p -> p.sumOf2SSE));
+ // 越小的排在前面
+ Collections.addAll(mergeHeap, buckets);
+
+ // System.out.println("begin iterating...");
+
for (int c = 0; c < numOfIterations; c++) {
- // Find the bucket to be split
- int maxSSEIndex = -1;
- double maxSSE = Double.NEGATIVE_INFINITY;
- for (int i = 0; i < m - 2; i++) {
- if (offset.get(i + 1) - offset.get(i) <= 1) {
- continue;
- }
- if (sse.get(i) > maxSSE) {
- maxSSE = sse.get(i);
- maxSSEIndex = i;
- }
+ if (debug) {
+ System.out.println("--------------[" + c + "]----------------");
}
- if (maxSSEIndex < 0) {
- if (debug) {
- System.out.println(c);
- System.out.println(maxSSEIndex);
- System.out.println("break max");
- }
+ if (splitHeap.isEmpty() || mergeHeap.isEmpty()) {
break;
}
-
- // Find the buckets to be merged
- int minSSEIndex = -1;
- double minSSE = Double.POSITIVE_INFINITY;
- for (int i = 0; i < m - 3; i++) {
- if (i == maxSSEIndex || i + 1 == maxSSEIndex) {
- continue;
- }
- if (sse.get(i) + sse.get(i + 1) < minSSE) {
- minSSE = sse.get(i) + sse.get(i + 1);
- minSSEIndex = i;
+ // Find the bucket to be split and buckets to be merged
+ LTDBucket bucket_split = splitHeap.poll();
+ if (bucket_split.isDeleted) {
+ c--;
+ continue;
+ }
+ LTDBucket buckets_merged = mergeHeap.poll();
+ if (buckets_merged.isDeleted) {
+ c--;
+ splitHeap.add(bucket_split); // 前面poll出来的未被删除的要加回去！
+ continue;
+ }
+ // TODO
+ if (bucket_split.startIdx == buckets_merged.startIdx) {
+ if (buckets_merged.next.next != null) {
+ buckets_merged = buckets_merged.next;
+ } else {
+ buckets_merged = buckets_merged.prev;
}
}
- if (minSSEIndex < 0) {
- if (debug) {
- System.out.println(c);
- System.out.println(minSSEIndex);
- System.out.println("break min");
+ if (bucket_split.endIdx == buckets_merged.getMergedEndIdx()) {
+ if (buckets_merged.prev != starter) {
+ buckets_merged = buckets_merged.prev;
+ } else { // 假设nbins至少>=4, 于是buckets_merged.next.next不为null
+ buckets_merged = buckets_merged.next.next;
}
- break;
}
- // Split
- int startIdx = offset.get(maxSSEIndex);
- int endIdx = offset.get(maxSSEIndex + 1);
- int middleIdx = (startIdx + endIdx) / 2;
- offset.add(maxSSEIndex + 1, middleIdx);
-
- // Update SSE affected by split
- sse.set(
- maxSSEIndex,
- calculateSSEForBucket(
- points, offset.get(maxSSEIndex) - 1, offset.get(maxSSEIndex + 1) + 1));
-
- double newSse =
- calculateSSEForBucket(
- points, offset.get(maxSSEIndex + 1) - 1, offset.get(maxSSEIndex + 2) + 1);
+ // split
+ if (debug) {
+ System.out.println("+++To split bucket: " + bucket_split);
+ }
+ int startIdx = bucket_split.startIdx;
+ int endIdx = bucket_split.endIdx;
+ int middleIdx = (int) ((startIdx + endIdx) * 1.0 / 2);
+ double sse1 = calculateSSEForBucket(points, startIdx - 1, middleIdx + 1);
+ double sse2 = calculateSSEForBucket(points, middleIdx - 1, endIdx + 1);
+
+ LTDBucket newBucket =
+ new LTDBucket(middleIdx, endIdx, sse2, sse2 + bucket_split.getNextSSE());
+ newBucket.prev = bucket_split;
+ newBucket.next = bucket_split.next;
+ if (newBucket.next != null) {
+ newBucket.next.prev = newBucket;
+ }
+ bucket_split.next = newBucket;
+
+ bucket_split.isDeleted = true;
+ LTDBucket replaceBucket = new LTDBucket(bucket_split);
+ replaceBucket.endIdx = middleIdx;
+ replaceBucket.sse = sse1;
+ replaceBucket.sumOf2SSE = sse1 + sse2;
+ replaceBucket.next = newBucket;
+ // 更新前一个桶的sumOf2SSE 注意这意味着前一个桶也要更新heap！
+ if (replaceBucket.prev != starter) {
+ replaceBucket.prev.isDeleted = true;
+ LTDBucket preReplaceBucket = new LTDBucket(replaceBucket.prev);
+ preReplaceBucket.sumOf2SSE = preReplaceBucket.sse + sse1;
+ splitHeap.add(preReplaceBucket);
+ mergeHeap.add(preReplaceBucket);
+ }
- sse.add(maxSSEIndex + 1, newSse);
+ splitHeap.add(newBucket);
+ splitHeap.add(replaceBucket);
+ mergeHeap.add(newBucket);
+ mergeHeap.add(replaceBucket);
- // Merge
- if (minSSEIndex > maxSSEIndex) {
- minSSEIndex += 1; // Adjust index
+ if (debug) {
+ System.out.println("\tsplit into: " + replaceBucket + "," + newBucket);
}
- offset.remove(minSSEIndex + 1);
-
- sse.set(
- minSSEIndex,
- calculateSSEForBucket(
- points, offset.get(minSSEIndex) - 1, offset.get(minSSEIndex + 1) + 1));
- sse.remove(minSSEIndex + 1);
+ // merge
+ if (debug) {
+ System.out.println("---To merge bucket: " + buckets_merged + "," + buckets_merged.next);
+ }
+ startIdx = buckets_merged.startIdx;
+ endIdx = buckets_merged.getMergedEndIdx();
+ double sse3 = calculateSSEForBucket(points, startIdx - 1, endIdx + 1);
+ buckets_merged.isDeleted = true;
+ buckets_merged.next.isDeleted = true;
+ LTDBucket mergedBucket = new LTDBucket(buckets_merged);
+ mergedBucket.endIdx = endIdx;
+ mergedBucket.sse = sse3;
+ mergedBucket.next = buckets_merged.next.next;
+ if (mergedBucket.next != null) {
+ mergedBucket.next.prev = mergedBucket;
+ }
+ // 更新自己的sumOf2SSE
+ mergedBucket.sumOf2SSE = sse3 + mergedBucket.getNextSSE(); // 如果next为null，这一项就是Infinity
+ // 更新前一个分桶的sumOf2SSE 注意这意味着前一个桶也要更新heap！
+ if (mergedBucket.prev != starter) {
+ mergedBucket.prev.isDeleted = true;
+ LTDBucket preReplaceBucket = new LTDBucket(mergedBucket.prev);
+ preReplaceBucket.sumOf2SSE = preReplaceBucket.sse + sse3;
+ splitHeap.add(preReplaceBucket);
+ mergeHeap.add(preReplaceBucket);
+ }
+ splitHeap.add(mergedBucket);
+ mergeHeap.add(mergedBucket);
+ if (debug) {
+ System.out.println("\tmerged bucket: " + mergedBucket);
+ }
+ // System.out.println("----------" + splitHeap.size() + "-------------");
+ // System.out.println("----------" + mergeHeap.size() + "-------------");
}
- // Convert ArrayList to int[] and return
- return offset;
+ List<Integer> res = new ArrayList<>();
+ LTDBucket bucket = starter.next;
+ while (true) {
+ res.add(bucket.startIdx);
+ if (bucket.next == null) {
+ res.add(bucket.endIdx); // 收尾
+ break;
+ }
+ bucket = bucket.next;
+ }
+ if (debug) {
+ System.out.println(res);
+ }
+ return res;
}
public static List<Point> LTTB(List<Point> points, List<Integer> bins) {
@@ -185,25 +274,26 @@ public class LTD {
return res;
}
- public static List<Point> LTD(List<Point> points, int m, int maxIter, boolean debug) {
+ public static List<Point> LTD(List<Point> points, int m, int maxIter, boolean debug)
+ throws IOException {
List<Integer> bins = getLtdBinIdxs(points, m, maxIter, debug);
return LTTB(points, bins);
}
- public static void main(String[] args) {
+ public static void main(String[] args) throws IOException {
Random rand = new Random(10);
String input = "D:\\datasets\\regular\\tmp2.csv";
boolean hasHeader = true;
int timeIdx = 0;
int valueIdx = 1;
- int N = 10000;
- List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- // Polyline polyline = new Polyline();
- // for (int i = 0; i < N; i += 1) {
- // double v = rand.nextInt(1000);
- // polyline.addVertex(new Point(i, v));
- // }
- // List<Point> points = polyline.getVertices();
+ int N = 1000_0;
+ // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
try (FileWriter writer = new FileWriter("raw.csv")) {
// 写入CSV头部
writer.append("x,y,z\n");
@@ -217,17 +307,17 @@ public class LTD {
System.out.println("Error writing to CSV file: " + e.getMessage());
}
- int m = 10;
- int maxIter = 10;
+ int m = 1000;
+ int maxIter = -1;
long startTime = System.currentTimeMillis();
// List<Integer> bins = getLtdBinIdxs(points, m, true);
- List<Point> sampled = LTD(points, m, maxIter, true);
+ List<Point> sampled = LTD(points, m, maxIter, false);
long endTime = System.currentTimeMillis();
System.out.println("Time taken: " + (endTime - startTime) + "ms");
- for (Point p : sampled) {
- System.out.println(p);
- }
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
// 写入字符串
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTDBucket.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTDBucket.java
new file mode 100644
index 00000000000..59998b599ea
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTDBucket.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.db.query.eBUG;
+
+public class LTDBucket {
+
+ public int startIdx, endIdx;
+ public double sse, sumOf2SSE;
+
+ public LTDBucket prev;
+ public LTDBucket next;
+
+ public boolean isDeleted;
+
+ public LTDBucket(int startIdx, int endIdx, double sse, double sumOf2SSE) {
+ this.startIdx = startIdx;
+ this.endIdx = endIdx;
+ this.sse = sse;
+ this.sumOf2SSE = sumOf2SSE;
+ this.prev = null;
+ this.next = null;
+ this.isDeleted = false;
+ }
+
+ public LTDBucket(LTDBucket ltdBucket) {
+ this.startIdx = ltdBucket.startIdx;
+ this.endIdx = ltdBucket.endIdx;
+ this.sse = ltdBucket.sse;
+ this.sumOf2SSE = ltdBucket.sumOf2SSE;
+ this.prev = ltdBucket.prev;
+ this.next = ltdBucket.next;
+ this.isDeleted = false;
+
+ if (ltdBucket.prev != null) {
+ ltdBucket.prev.next = this;
+ }
+ if (ltdBucket.next != null) {
+ ltdBucket.next.prev = this;
+ }
+ }
+
+ public int getStartIdx() {
+ return startIdx;
+ }
+
+ public int getEndIdx() {
+ return endIdx;
+ }
+
+ public int getMergedEndIdx() {
+ if (this.next == null) {
+ return this.endIdx;
+ } else {
+ return this.next.endIdx;
+ }
+ }
+
+ public double getNextSSE() {
+ if (this.next == null) {
+ return Double.MAX_VALUE;
+ } else {
+ return this.next.sse;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "[" + startIdx + ", " + endIdx + "]: " + sse + "," + sumOf2SSE;
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD_slow_deprecated.java
similarity index 89%
copy from server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD.java
copy to server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD_slow_deprecated.java
index 6eb2051411c..7746a5c8f8d 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/LTD_slow_deprecated.java
@@ -6,7 +6,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Random;
-public class LTD {
+public class LTD_slow_deprecated {
public static Point calculateAveragePoint(List<Point> points, int startClosed, int endClosed) {
double sumX = 0;
double sumY = 0;
@@ -78,6 +78,9 @@ public class LTD {
}
for (int c = 0; c < numOfIterations; c++) {
+ if (debug) {
+ System.out.println("--------------[" + c + "]----------------");
+ }
// Find the bucket to be split
int maxSSEIndex = -1;
double maxSSE = Double.NEGATIVE_INFINITY;
@@ -123,6 +126,9 @@ public class LTD {
// Split
int startIdx = offset.get(maxSSEIndex);
int endIdx = offset.get(maxSSEIndex + 1);
+ if (debug) {
+ System.out.println("+++To split bucket: " + startIdx + "," + endIdx + ":" + maxSSE);
+ }
int middleIdx = (startIdx + endIdx) / 2;
offset.add(maxSSEIndex + 1, middleIdx);
@@ -144,6 +150,10 @@ public class LTD {
}
offset.remove(minSSEIndex + 1);
+ if (debug) {
+ System.out.println("---To merge bucket: " + offset.get(minSSEIndex) + ":" + minSSE);
+ }
+
sse.set(
minSSEIndex,
calculateSSEForBucket(
@@ -153,6 +163,9 @@ public class LTD {
}
// Convert ArrayList to int[] and return
+ if (debug) {
+ System.out.println(offset);
+ }
return offset;
}
@@ -196,14 +209,14 @@ public class LTD {
boolean hasHeader = true;
int timeIdx = 0;
int valueIdx = 1;
- int N = 10000;
- List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- // Polyline polyline = new Polyline();
- // for (int i = 0; i < N; i += 1) {
- // double v = rand.nextInt(1000);
- // polyline.addVertex(new Point(i, v));
- // }
- // List<Point> points = polyline.getVertices();
+ int N = 100_0000;
+ // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
try (FileWriter writer = new FileWriter("raw.csv")) {
// 写入CSV头部
writer.append("x,y,z\n");
@@ -217,11 +230,11 @@ public class LTD {
System.out.println("Error writing to CSV file: " + e.getMessage());
}
- int m = 10;
- int maxIter = 10;
+ int m = 1000;
+ int maxIter = -1;
long startTime = System.currentTimeMillis();
// List<Integer> bins = getLtdBinIdxs(points, m, true);
- List<Point> sampled = LTD(points, m, maxIter, true);
+ List<Point> sampled = LTD(points, m, maxIter, false);
long endTime = System.currentTimeMillis();
System.out.println("Time taken: " + (endTime - startTime) + "ms");
(iotdb) 28/44: dp
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 364f0d8d0b6f2780e69203b5a5240be3bc3b337b
Author: Lei Rui <10...@qq.com>
AuthorDate: Mon Jan 27 21:53:44 2025 +0800
dp
---
server/pom.xml | 10 +-
server/sample_eBUG-jar-with-dependencies.jar | Bin 38715994 -> 38716020 bytes
.../java/org/apache/iotdb/db/query/eBUG/DP.java | 286 +++++++++
.../java/org/apache/iotdb/db/query/eBUG/Tool.java | 712 +++++++++++----------
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 44 +-
5 files changed, 694 insertions(+), 358 deletions(-)
diff --git a/server/pom.xml b/server/pom.xml
index 66ccebdb43a..b0b6be3c539 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -281,13 +281,13 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
- <finalName>sample_fsw</finalName>
- <!-- <finalName>sample_eBUG</finalName>-->
- <!-- <finalName>sample_BUL2</finalName>-->
+ <!-- <finalName>sample_fsw</finalName>-->
+ <finalName>sample_eBUG</finalName>
+ <!-- <finalName>sample_BUL2</finalName>-->
<archive>
<manifest>
- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>
- <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>-->
+ <!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_FSW</mainClass>-->
+ <mainClass>org.apache.iotdb.db.query.eBUG.sample_eBUG</mainClass>
<!-- <mainClass>org.apache.iotdb.db.query.eBUG.sample_bottomUpL2</mainClass>-->
</manifest>
</archive>
diff --git a/server/sample_eBUG-jar-with-dependencies.jar b/server/sample_eBUG-jar-with-dependencies.jar
index a37acfd0590..0df972fa4a5 100644
Binary files a/server/sample_eBUG-jar-with-dependencies.jar and b/server/sample_eBUG-jar-with-dependencies.jar differ
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java
new file mode 100644
index 00000000000..a6a9647fb4b
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DP.java
@@ -0,0 +1,286 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.*;
+
+// Some public references about dynamic programming:
+// https://justinwillmert.com/articles/2014/bellman-k-segmentation-algorithm/
+// https://github.com/CrivelliLab/Protein-Structure-DL
+
+public class DP { // Dynamic-Programming
+
+ // Enum to represent error types
+ public enum ERROR {
+ L1, L2, L_infy, area
+ }
+
+ // precomputing error measures of all possible segments, e.g., ad2 table
+ public static double[][] prepareCostTable(List<Point> points, ERROR errorType, boolean debug) {
+ int N = points.size();
+ double[][] dists = new double[N][N]; // 默认0，对角线元素已经都是0了
+
+ // 外层循环i：range长度=right boundary-left boundary
+ // 内层循环j: 矩阵逐行
+ for (int i = 1; i < N; i++) { // TODO
+ for (int j = 0; j < N - i; j++) {
+ // j = left boundary, r = right boundary, 左闭右闭
+ int r = i + j; // r<N
+ if (debug) {
+ System.out.println(">>>> i=" + i + ", j=" + j + ", r=" + r);
+ }
+
+ // lx=j,rx=r, 从lx=j到rx=r（左闭右闭）的linear interpolation（即连接lx和rx点）的errorType类型的误差
+ double mc = joint_segment_error(points, j, r, errorType);
+ dists[j][r] = mc;
+ if (debug) {
+ System.out.println("dists=");
+ Tool.printArray(dists);
+ System.out.println("---------------------");
+ }
+ }
+ }
+ return dists;
+ }
+
+ // 注意k是分段数，不是采点数
+ public static List<Point> dynamic_programming(List<Point> points, int k, ERROR errorType, boolean debug) {
+ int N = points.size();
+
+ // 预计算dists（即ad2 table），dists[i][j]代表对子序列i:j左闭右闭直接连接pi和pj的近似误差
+ double[][] dists = prepareCostTable(points, errorType, debug);
+ if (debug) {
+ Tool.printArray(dists);
+ System.out.println("----------------------------------");
+ }
+
+ // 创建dp table。kSegDist[i][j]代表把子序列T[0:j]左闭右闭分成i段(java从0开始计数)的最小误差
+ double[][] kSegDist = new double[k][N];
+ // Initialize the case k=1 directly from the pre-computed distances
+ // 注意java从0开始，所以是0 index第一行代表分段数1
+ System.arraycopy(dists[0], 0, kSegDist[0], 0, kSegDist[0].length);
+ if (debug) {
+ System.out.println("k_seg_dist=");
+ Tool.printArray(kSegDist);
+ }
+
+ // 创建path table。记录子问题的解。kSegPath[i][j]代表把T[0:j]分成i段这个问题的最优子问题的位置
+ // 子问题是把T[0:kSegPath[i][j]]分成i-1段，以及最后一段T[kSegPath[i][j]:end]
+ int[][] kSegPath = new int[k][N];
+ // 第一行只分一段，所以不管终点是哪个（列），左边的闭合起点都是0
+ // 虽然java数组默认初始是0，但是这里还是显式赋值
+ Arrays.fill(kSegPath[0], 0);
+ if (debug) {
+ System.out.println("k_seg_path=");
+ Tool.printArray(kSegPath);
+ }
+
+ // 外层循环i：分段数，注意python从0开始，所以实际含义是2~k个分段数
+ for (int i = 1; i < k; i++) {
+ // 内层循环j：从第一个点开始到j终点（闭合）的序列
+ // 所以含义是：找到从第一个点开始到j终点（闭合）的序列的分成(i+1)段的最佳分段方案（误差最小）
+ for (int j = 0; j < N; j++) {
+ // 动态规划
+ // 注意linear interpolation不需要单点成为一个分段的情况
+ // TODO 误差类型
+ // TODO ghost修复
+ // 从0:j的序列分成i段的问题：遍历所有可能的子问题组合解
+ if (debug) {
+ System.out.println(">>>分段数i+1" + (i + 1) + ",end pos j=" + j);
+ }
+ double[] choices = new double[j + 1]; // java从0开始
+ for (int xtmp = 0; xtmp < j + 1; xtmp++) {
+ if (errorType == ERROR.L_infy) {
+ choices[xtmp] = Math.max(kSegDist[i - 1][xtmp], dists[xtmp][j]);
+ } else {
+ choices[xtmp] = kSegDist[i - 1][xtmp] + dists[xtmp][j];
+ }
+ }
+ if (debug) {
+ for (int xtmp = 0; xtmp < j + 1; xtmp++) { // 遍历从 0 到 j 的每个元素
+ if (errorType == ERROR.L_infy) {
+ System.out.printf(" max((k_seg_dist[%d, %d] = %f), (dists[%d, %d] = %f)) --> %f%n",
+ i - 1, xtmp, kSegDist[i - 1][xtmp],
+ xtmp, j, dists[xtmp][j],
+ Math.max(kSegDist[i - 1][xtmp], dists[xtmp][j]));
+ } else {
+ System.out.printf(" (k_seg_dist[%d, %d] = %f) + (dists[%d, %d] = %f) --> %f%n",
+ i - 1, xtmp, kSegDist[i - 1][xtmp],
+ xtmp, j, dists[xtmp][j],
+ kSegDist[i - 1][xtmp] + dists[xtmp][j]);
+ }
+ }
+ }
+
+ int bestIndex = getIndexOfMin(choices);
+ double bestVal = choices[bestIndex];
+
+ // Store the sub-problem solution
+ kSegPath[i][j] = bestIndex;
+
+ kSegDist[i][j] = bestVal;
+
+ if (debug) {
+ System.out.println("kSegDist[" + i + "][" + j + "] = " + bestVal);
+ System.out.println("kSegDist=");
+ Tool.printArray(kSegDist);
+ System.out.println("kSegPath=");
+ Tool.printArray(kSegPath);
+ }
+ }
+ }
+
+ // 开始回溯构建采样结果
+ List<Point> sDs = new ArrayList<>(); // k+1个采样点
+ List<Integer> sDs_idx = new ArrayList<>();
+ int rhs = N - 1;
+ sDs.add(points.get(rhs)); // 全局尾点
+ sDs_idx.add(rhs);
+
+ for (int i = k - 1; i >= 0; i--) {
+ int lhs = kSegPath[i][rhs];
+ sDs.add(points.get(lhs));
+ sDs_idx.add(lhs);
+ rhs = lhs;
+ }
+
+ // 反转列表
+ Collections.reverse(sDs);
+ Collections.reverse(sDs_idx);
+
+ if (debug) {
+ System.out.println(sDs);
+ System.out.println(">>>>>ad2[][]=");
+ Tool.printArray(dists);
+ System.out.println(">>>>>dp[][]=");
+ Tool.printTransposeArray(kSegDist);
+ System.out.println(">>>>>path[][]=");
+ Tool.printTransposeArray(kSegPath);
+ System.out.println(error(points, sDs_idx, errorType));
+ }
+
+ return sDs;
+ }
+
+ // Helper method to get index of minimum value
+ private static int getIndexOfMin(double[] array) {
+ double minVal = array[0];
+ int minIndex = 0;
+ for (int i = 1; i < array.length; i++) {
+ if (array[i] < minVal) {
+ minVal = array[i];
+ minIndex = i;
+ }
+ }
+ return minIndex;
+ }
+
+ // Method to calculate joint segment error based on error type
+ private static double joint_segment_error(List<Point> points, int lx, int rx, ERROR errorType) {
+ // 默认joint=true, residual=true，即使用linear interpolation近似分段
+ // lx~rx 左闭右闭
+ if (errorType != ERROR.area) {
+ double x1 = points.get(lx).x;
+ double y1 = points.get(lx).y;
+ double x2 = points.get(rx).x;
+ double y2 = points.get(rx).y;
+
+ // linear interpolation:
+ double k = (y2 - y1) / (x2 - x1);
+ double b = (y1 * x2 - y2 * x1) / (x2 - x1);
+
+ double tmp = 0;
+ if (errorType == ERROR.L2) {
+ for (int i = lx; i <= rx; i++) {
+ double e = (k * points.get(i).x + b - points.get(i).y) * (k * points.get(i).x + b - points.get(i).y);
+ tmp += e;
+ }
+ } else if (errorType == ERROR.L1) {
+ for (int i = lx; i <= rx; i++) {
+ double e = Math.abs(k * points.get(i).x + b - points.get(i).y);
+ tmp += e;
+ }
+ } else if (errorType == ERROR.L_infy) {
+ for (int i = lx; i <= rx; i++) {
+ double e = Math.abs(k * points.get(i).x + b - points.get(i).y); // 注意绝对值
+ if (e > tmp) {
+ tmp = e;
+ }
+ }
+ }
+ return tmp;
+ } else { // AD
+ List<Point> linearInterpolation = new ArrayList<>();
+ linearInterpolation.add(points.get(lx));
+ linearInterpolation.add(points.get(rx));
+ return Tool.total_areal_displacement(points.subList(lx, rx + 1), // 注意lx,rx左闭右闭
+ linearInterpolation, false);
+ }
+ }
+
+ public static double error(List<Point> points, List<Integer> sampledIdx, ERROR errorType) {
+ double res = 0;
+ for (int i = 0; i < sampledIdx.size() - 1; i++) {
+ int lx = sampledIdx.get(i);
+ int rx = sampledIdx.get(i + 1);
+ double e = joint_segment_error(points, lx, rx, errorType);
+ if (errorType == ERROR.L_infy) {
+ res = Math.max(res, e);
+ } else {
+ res += e;
+ }
+ }
+ return res;
+ }
+
+ // Example usage
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "D:\\LabSync\\iotdb\\我的Gitbook基地\\RUI Lei gitbook\\士论\\49. visval改进\\notebook\\raw.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = -1;
+ List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+// Polyline polyline = new Polyline();
+// for (int i = 0; i < N; i += 1) {
+// double v = rand.nextInt(1000);
+// polyline.addVertex(new Point(i, v));
+// }
+// List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ long startTime = System.currentTimeMillis();
+// double[][] ad2 = prepareKSegments(points, ERROR.L1, false);
+ int m = 4;
+ List<Point> sampled = dynamic_programming(points, m - 1, ERROR.area, false);
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime - startTime) + "ms");
+
+ for (Point p : sampled) {
+ System.out.println(p);
+ }
+ System.out.println(sampled.size());
+
+// try (PrintWriter writer = new PrintWriter(new File("output.csv"))) {
+// // 写入字符串
+// for (int i = 0; i < sampled.size(); i++) {
+// writer.println(sampled.get(i).x + "," + sampled.get(i).y);
+// }
+//
+// } catch (FileNotFoundException e) {
+// e.printStackTrace();
+// }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
index a61f091198a..9a54f835ec3 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
@@ -11,363 +11,407 @@ import java.util.List;
import java.util.stream.Stream;
public class Tool {
- // Assumed interface for the simplification function
- interface SimplifyFunction {
- List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs)
- throws IOException;
- }
-
- // Method to generate the output file name based on input path
- public static String generateOutputFileName(
- String inputFilePath, String outDir, String customize) {
- // Get the file name without extension
- Path path = Paths.get(inputFilePath);
- String fileNameWithoutExtension = path.getFileName().toString();
- String name = fileNameWithoutExtension.substring(0, fileNameWithoutExtension.lastIndexOf('.'));
-
- // Get the file extension
- String extension =
- path.getFileName().toString().substring(fileNameWithoutExtension.lastIndexOf('.'));
-
- // Create output file path by appending '-ds' before the extension
- // String outputFile = name + "-ds-e" + e + extension;
- String outputFile = name + customize + extension;
-
- if (outDir == null) { // 表示使用input文件所在的文件夹
- // Handle path compatibility for different operating systems (Linux/Windows)
- return path.getParent().resolve(outputFile).toString();
- } else {
- Path out = Paths.get(outDir);
- return out.resolve(outputFile).toString();
- }
- }
-
- public static double getParam(
- List<Point> points,
- int m,
- SimplifyFunction simplifyFunc,
- double tolerantPts,
- Object... kwargs)
- throws IOException {
- // double x = 1; 不要从1开始，因为对于swab来说使用如此小的maxError会很慢，宁可让maxError从大到小
- double x = 10;
- boolean directLess = false;
- boolean directMore = false;
-
- int iterNum = 0;
- int maxIterNum = 50;
-
- // First binary search loop to find the initial range
- double base = 2;
- while (true) {
- // System.out.println("x=" + x);
- iterNum++;
- if (iterNum > maxIterNum) {
- // System.out.println("reach maxIterNum1");
- break; // Avoid infinite loops for special cases
- }
- List<Point> res = simplifyFunc.reducePoints(points, x, kwargs);
- int length = res.size();
- // System.out.println(length);
-
- if (Math.abs(length - m) <= tolerantPts) {
- return x; // Return the found parameter
- }
-
- if (length > m) {
- if (directMore) {
- break; // Reach the first more
- }
- if (!directLess) {
- directLess = true;
+
+ public static void printArray(double[][] array) {
+ // 遍历每一行
+ for (double[] row : array) {
+ for (double value : row) {
+ // 格式化输出：%10.2f 表示右对齐，占10个字符，保留2位小数
+ System.out.printf("%10.2f ", value);
+ }
+ System.out.println(); // 换行
}
+ }
- x *= base; // Need less points, increase x
- } else {
- if (directLess) {
- break; // Reach the first less
+ public static void printTransposeArray(double[][] array) {
+ // 遍历每一列
+ for (int i = 0; i < array[0].length; i++) {
+ for (int j = 0; j < array.length; j++) {
+ // 格式化输出：%10.2f 表示右对齐，占10个字符，保留2位小数
+ System.out.printf("%10.2f ", array[j][i]);
+ }
+ System.out.println(); // 换行
}
- if (!directMore) {
- directMore = true;
+ }
+
+ public static void printArray(int[][] array) {
+ // 遍历每一行
+ for (int[] row : array) {
+ for (int value : row) {
+ System.out.printf("%10d ", value);
+ }
+ System.out.println(); // 换行
}
+ }
- x /= base; // Need more points, decrease x
- }
+ public static void printTransposeArray(int[][] array) {
+ // 遍历每一列
+ for (int i = 0; i < array[0].length; i++) {
+ for (int j = 0; j < array.length; j++) {
+ // 格式化输出：%10.2f 表示右对齐，占10个字符，保留2位小数
+ System.out.printf("%10d ", array[j][i]);
+ }
+ System.out.println(); // 换行
+ }
}
- // Determine the initial left and right bounds
- double left = (directLess) ? x / base : x;
- double right = (directMore) ? x * base : x;
-
- // Refine the range with binary search
- iterNum = 0;
- while (true) {
- iterNum++;
- if (iterNum > maxIterNum) {
- // System.out.println("reach maxIterNum");
- break; // Avoid infinite loops
- }
-
- double mid = (left + right) / 2;
-
- List<Point> res = simplifyFunc.reducePoints(points, mid, kwargs);
- int length = res.size();
- // System.out.println(length);
-
- if (Math.abs(length - m) <= tolerantPts) {
- return mid; // Return the refined parameter
- }
-
- if (length > m) {
- left = mid; // Need less, narrow range to the left
- } else {
- right = mid; // Need more, narrow range to the right
- }
- // System.out.println(left + "," + right);
+ // Assumed interface for the simplification function
+ interface SimplifyFunction {
+ List<Point> reducePoints(List<Point> points, double epsilon, Object... kwargs)
+ throws IOException;
}
- return (left + right) / 2; // Return the average of left and right as the final parameter
- }
-
- public static List<Point> readFromFile(
- String input, boolean hasHeader, int timeIdx, int valueIdx, int N) {
- List<Point> res = new ArrayList<>();
-
- // Using Files.lines() for memory-efficient line-by-line reading
- try (Stream<String> lines = Files.lines(Paths.get(input))) {
- Iterator<String> iterator = lines.iterator();
- int lineNumber = 0;
-
- // Skip the header line if necessary
- if (hasHeader && iterator.hasNext()) {
- iterator.next(); // Skip the header
- }
-
- // Process each line
- while (iterator.hasNext()) {
- lineNumber++;
- String line = iterator.next();
- String[] columns = line.split(",");
-
- if (columns.length > Math.max(timeIdx, valueIdx)) {
- double time;
- if (timeIdx < 0) {
- time = lineNumber;
- } else {
- time = Double.parseDouble(columns[timeIdx]);
- }
- double value = Double.parseDouble(columns[valueIdx]);
- res.add(new Point(time, value));
- // System.out.println("Line " + lineNumber + " - Time: " + time + ",
- // Value: " + value);
+ // Method to generate the output file name based on input path
+ public static String generateOutputFileName(
+ String inputFilePath, String outDir, String customize) {
+ // Get the file name without extension
+ Path path = Paths.get(inputFilePath);
+ String fileNameWithoutExtension = path.getFileName().toString();
+ String name = fileNameWithoutExtension.substring(0, fileNameWithoutExtension.lastIndexOf('.'));
+
+ // Get the file extension
+ String extension =
+ path.getFileName().toString().substring(fileNameWithoutExtension.lastIndexOf('.'));
+
+ // Create output file path by appending '-ds' before the extension
+ // String outputFile = name + "-ds-e" + e + extension;
+ String outputFile = name + customize + extension;
+
+ if (outDir == null) { // 表示使用input文件所在的文件夹
+ // Handle path compatibility for different operating systems (Linux/Windows)
+ return path.getParent().resolve(outputFile).toString();
} else {
- System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
+ Path out = Paths.get(outDir);
+ return out.resolve(outputFile).toString();
+ }
+ }
+
+ public static double getParam(
+ List<Point> points,
+ int m,
+ SimplifyFunction simplifyFunc,
+ double tolerantPts,
+ Object... kwargs)
+ throws IOException {
+ // double x = 1; 不要从1开始，因为对于swab来说使用如此小的maxError会很慢，宁可让maxError从大到小
+ double x = 10;
+ boolean directLess = false;
+ boolean directMore = false;
+
+ int iterNum = 0;
+ int maxIterNum = 50;
+
+ // First binary search loop to find the initial range
+ double base = 2;
+ while (true) {
+ // System.out.println("x=" + x);
+ iterNum++;
+ if (iterNum > maxIterNum) {
+ // System.out.println("reach maxIterNum1");
+ break; // Avoid infinite loops for special cases
+ }
+ List<Point> res = simplifyFunc.reducePoints(points, x, kwargs);
+ int length = res.size();
+ // System.out.println(length);
+
+ if (Math.abs(length - m) <= tolerantPts) {
+ return x; // Return the found parameter
+ }
+
+ if (length > m) {
+ if (directMore) {
+ break; // Reach the first more
+ }
+ if (!directLess) {
+ directLess = true;
+ }
+
+ x *= base; // Need less points, increase x
+ } else {
+ if (directLess) {
+ break; // Reach the first less
+ }
+ if (!directMore) {
+ directMore = true;
+ }
+
+ x /= base; // Need more points, decrease x
+ }
}
- if (N > 0 && lineNumber >= N) {
- // N>0控制点数，否则N<=0表示读全部数据
- break;
+
+ // Determine the initial left and right bounds
+ double left = (directLess) ? x / base : x;
+ double right = (directMore) ? x * base : x;
+
+ // Refine the range with binary search
+ iterNum = 0;
+ while (true) {
+ iterNum++;
+ if (iterNum > maxIterNum) {
+ // System.out.println("reach maxIterNum");
+ break; // Avoid infinite loops
+ }
+
+ double mid = (left + right) / 2;
+
+ List<Point> res = simplifyFunc.reducePoints(points, mid, kwargs);
+ int length = res.size();
+ // System.out.println(length);
+
+ if (Math.abs(length - m) <= tolerantPts) {
+ return mid; // Return the refined parameter
+ }
+
+ if (length > m) {
+ left = mid; // Need less, narrow range to the left
+ } else {
+ right = mid; // Need more, narrow range to the right
+ }
+ // System.out.println(left + "," + right);
}
- }
- } catch (IOException e) {
- e.printStackTrace();
+
+ return (left + right) / 2; // Return the average of left and right as the final parameter
}
- return res;
- }
-
- // 计算三角形面积
- public static double triArea(Point d0, Point d1, Point d2) {
- double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
- return (dArea > 0.0) ? dArea : -dArea; // abs
- }
-
- // 计算简单多边形（边不交叉）的面积（鞋带公式）
- public static double calculatePolygonArea(List<Point> points) {
- // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
- int n = points.size();
- double area = 0;
- for (int i = 0; i < n; i++) {
- int next = (i + 1) % n;
- area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
+
+ public static List<Point> readFromFile(
+ String input, boolean hasHeader, int timeIdx, int valueIdx, int N) {
+ List<Point> res = new ArrayList<>();
+
+ // Using Files.lines() for memory-efficient line-by-line reading
+ try (Stream<String> lines = Files.lines(Paths.get(input))) {
+ Iterator<String> iterator = lines.iterator();
+ int lineNumber = 0;
+
+ // Skip the header line if necessary
+ if (hasHeader && iterator.hasNext()) {
+ iterator.next(); // Skip the header
+ }
+
+ // Process each line
+ while (iterator.hasNext()) {
+ lineNumber++;
+ String line = iterator.next();
+ String[] columns = line.split(",");
+
+ if (columns.length > Math.max(timeIdx, valueIdx)) {
+ double time;
+ if (timeIdx < 0) {
+ time = lineNumber;
+ } else {
+ time = Double.parseDouble(columns[timeIdx]);
+ }
+ double value = Double.parseDouble(columns[valueIdx]);
+ res.add(new Point(time, value));
+ // System.out.println("Line " + lineNumber + " - Time: " + time + ",
+ // Value: " + value);
+ } else {
+ System.out.println("Line " + lineNumber + " is malformed (not enough columns).");
+ }
+ if (N > 0 && lineNumber >= N) {
+ // N>0控制点数，否则N<=0表示读全部数据
+ break;
+ }
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return res;
}
- return Math.abs(area) / 2.0;
- }
-
- // 计算两个向量的叉积
- public static double crossProduct(double x1, double y1, double x2, double y2) {
- // >0: (x2,y2)在(x1,y1)的逆时针方向
- // <0: (x2,y2)在(x1,y1)的顺时针方向
- // =0: 平行或共线
- return x1 * y2 - y1 * x2;
- }
-
- // 判断两条线段是否相交并计算交点
- // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
- public static Object[] lineIntersection(Point[] L1, Point[] L2) {
- double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
- double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
-
- // 判断是否相交（叉积）
- double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
- double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
- double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
- double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
-
- // 如果叉积条件满足，表示有交点
- // d1*d2<0意味着P3、P4分别在L12的两边
- // d3*d4<0意味着P1、P2分别在L34的两边
- // 两个同时满足说明有交点
- if (d1 * d2 < 0 && d3 * d4 < 0) {
- double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
- double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
- double x = x1 + t1 * (x2 - x1);
- double y = y1 + t1 * (y2 - y1);
- return new Object[] {true, new Point(x, y)};
+
+ // 计算三角形面积
+ public static double triArea(Point d0, Point d1, Point d2) {
+ double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
+ return (dArea > 0.0) ? dArea : -dArea; // abs
}
- // 检查是否起点或终点重合
- if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
- return new Object[] {true, new Point(x1, y1)};
+ // 计算简单多边形（边不交叉）的面积（鞋带公式）
+ public static double calculatePolygonArea(List<Point> points) {
+ // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
+ int n = points.size();
+ double area = 0;
+ for (int i = 0; i < n; i++) {
+ int next = (i + 1) % n;
+ area += points.get(i).x * points.get(next).y - points.get(next).x * points.get(i).y;
+ }
+ return Math.abs(area) / 2.0;
}
- if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
- return new Object[] {true, new Point(x2, y2)};
+
+ // 计算两个向量的叉积
+ public static double crossProduct(double x1, double y1, double x2, double y2) {
+ // >0: (x2,y2)在(x1,y1)的逆时针方向
+ // <0: (x2,y2)在(x1,y1)的顺时针方向
+ // =0: 平行或共线
+ return x1 * y2 - y1 * x2;
}
- return new Object[] {false, null};
- }
-
- // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
- public static double total_areal_displacement(
- List<Point> points, List<Point> points2, boolean debug) {
- double totalArea = 0;
- int i = 0, j = 0; // i for points, j for points2
- Point prevIntersection = null;
- int prevI = -1, prevJ = -1;
-
- // List<double[]> intersectionCoords = new ArrayList<>();
- List<Double> areaList = new ArrayList<>();
-
- while (i < points.size() - 1 && j < points2.size() - 1) {
- if (debug) {
- System.out.println("--------- " + i + " " + j + " ------------");
- }
-
- // 当前线段
- Point[] L1 = {points.get(i), points.get(i + 1)};
- Point[] L2 = {points2.get(j), points2.get(j + 1)};
-
- // 判断是否有交点
- Object[] result = lineIntersection(L1, L2);
- boolean isIntersect = (boolean) result[0];
- Point intersection = (Point) result[1];
-
- if (isIntersect) {
- // intersectionCoords.add(intersection);
-
- if (prevIntersection != null) {
- // 构造多边形点集
- List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
- polygon.add(prevIntersection);
- if (debug) {
- System.out.println("- start intersection: " + prevIntersection);
- }
- polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
- // polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i +
- // 1))); // 添加当前线段的点，左闭右开
- if (debug) {
- System.out.println("- one side: " + points.subList(prevI, i + 1));
- }
- polygon.add(intersection);
- if (debug) {
- System.out.println("- end intersection: " + intersection);
- }
- List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
- Collections.reverse(tempPoints2); // 添加另一序列的点
- polygon.addAll(tempPoints2);
- if (debug) {
- System.out.println("- another side: " + tempPoints2);
- }
-
- // double[][] polygonArray = new double[polygon.size()][2];
- // for (int k = 0; k < polygon.size(); k++) {
- // polygonArray[k] = polygon.get(k);
- // }
-
- // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
- double area = calculatePolygonArea(polygon);
- if (debug) {
- System.out.println("Area = " + area);
- }
- totalArea += area;
- areaList.add(area);
+ // 判断两条线段是否相交并计算交点
+ // L1包含一个线段的两个端点，L2包含另一个线段的两个端点
+ public static Object[] lineIntersection(Point[] L1, Point[] L2) {
+ double x1 = L1[0].x, y1 = L1[0].y, x2 = L1[1].x, y2 = L1[1].y;
+ double x3 = L2[0].x, y3 = L2[0].y, x4 = L2[1].x, y4 = L2[1].y;
+
+ // 判断是否相交（叉积）
+ double d1 = crossProduct(x2 - x1, y2 - y1, x3 - x1, y3 - y1);
+ double d2 = crossProduct(x2 - x1, y2 - y1, x4 - x1, y4 - y1);
+ double d3 = crossProduct(x4 - x3, y4 - y3, x1 - x3, y1 - y3);
+ double d4 = crossProduct(x4 - x3, y4 - y3, x2 - x3, y2 - y3);
+
+ // 如果叉积条件满足，表示有交点
+ // d1*d2<0意味着P3、P4分别在L12的两边
+ // d3*d4<0意味着P1、P2分别在L34的两边
+ // 两个同时满足说明有交点
+ if (d1 * d2 < 0 && d3 * d4 < 0) {
+ double denominator = (y4 - y3) * (x2 - x1) - (x4 - x3) * (y2 - y1); // 不可能为0（平行或共线），因为已经判断相交了
+ double t1 = ((x4 - x3) * (y1 - y3) - (y4 - y3) * (x1 - x3)) / denominator;
+ double x = x1 + t1 * (x2 - x1);
+ double y = y1 + t1 * (y2 - y1);
+ return new Object[]{true, new Point(x, y)};
}
- prevIntersection = intersection;
- prevI = i + 1;
- prevJ = j + 1;
+ // 检查是否起点或终点重合
+ if ((x1 == x3 && y1 == y3) || (x1 == x4 && y1 == y4)) {
+ return new Object[]{true, new Point(x1, y1)};
+ }
+ if ((x2 == x3 && y2 == y3) || (x2 == x4 && y2 == y4)) {
+ return new Object[]{true, new Point(x2, y2)};
+ }
+
+ return new Object[]{false, null};
+ }
+
+ // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
+ public static double total_areal_displacement(
+ List<Point> points, List<Point> points2, boolean debug) {
+ double totalArea = 0;
+ int i = 0, j = 0; // i for points, j for points2
+ Point prevIntersection = null;
+ int prevI = -1, prevJ = -1;
+
+ // List<double[]> intersectionCoords = new ArrayList<>();
+ List<Double> areaList = new ArrayList<>();
+
+ while (i < points.size() - 1 && j < points2.size() - 1) {
+ if (debug) {
+ System.out.println("--------- " + i + " " + j + " ------------");
+ }
+
+ // 当前线段
+ Point[] L1 = {points.get(i), points.get(i + 1)};
+ Point[] L2 = {points2.get(j), points2.get(j + 1)};
+
+ // 判断是否有交点
+ Object[] result = lineIntersection(L1, L2);
+ boolean isIntersect = (boolean) result[0];
+ Point intersection = (Point) result[1];
+
+ if (isIntersect) {
+ // intersectionCoords.add(intersection);
+
+ if (prevIntersection != null) {
+ // 构造多边形点集
+ List<Point> polygon = new ArrayList<>(); // 按照顺时针/逆时针几何连接顺序枚举多边形的顶点
+ polygon.add(prevIntersection);
+ if (debug) {
+ System.out.println("- start intersection: " + prevIntersection);
+ }
+ polygon.addAll(points.subList(prevI, i + 1)); // 添加当前线段的点，左闭右开
+ // polygon.addAll(Arrays.asList(Arrays.copyOfRange(points, prevI, i +
+ // 1))); // 添加当前线段的点，左闭右开
+ if (debug) {
+ System.out.println("- one side: " + points.subList(prevI, i + 1));
+ }
+ polygon.add(intersection);
+ if (debug) {
+ System.out.println("- end intersection: " + intersection);
+ }
+ List<Point> tempPoints2 = points2.subList(prevJ, j + 1);
+ Collections.reverse(tempPoints2); // 添加另一序列的点
+ polygon.addAll(tempPoints2);
+ if (debug) {
+ System.out.println("- another side: " + tempPoints2);
+ }
+
+ // double[][] polygonArray = new double[polygon.size()][2];
+ // for (int k = 0; k < polygon.size(); k++) {
+ // polygonArray[k] = polygon.get(k);
+ // }
+
+ // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
+ double area = calculatePolygonArea(polygon);
+ if (debug) {
+ System.out.println("Area = " + area);
+ }
+ totalArea += area;
+ areaList.add(area);
+ }
+
+ prevIntersection = intersection;
+ prevI = i + 1;
+ prevJ = j + 1;
+ if (debug) {
+ System.out.println(
+ "This intersection = "
+ + intersection
+ + ", next polygon: side1 = "
+ + prevI
+ + ", side2 = "
+ + prevJ);
+ }
+ }
+
+ int currentI = i; // 临时存储i
+ int currentJ = j; // 临时存储j
+ if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
+ i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
+ j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
+ }
+ } // end of while
+
if (debug) {
- System.out.println(
- "This intersection = "
- + intersection
- + ", next polygon: side1 = "
- + prevI
- + ", side2 = "
- + prevJ);
+ System.out.println(areaList);
}
- }
-
- int currentI = i; // 临时存储i
- int currentJ = j; // 临时存储j
- if (points.get(currentI + 1).x <= points2.get(currentJ + 1).x) {
- i += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
- }
- if (points.get(currentI + 1).x >= points2.get(currentJ + 1).x) {
- j += 1; // 基于时间戳严格递增的假设，Line不会回头或者垂直
- }
- } // end of while
-
- if (debug) {
- System.out.println(areaList);
+
+ return totalArea;
}
- return totalArea;
- }
-
- // 测试方法
- public static void main(String[] args) {
- // 示例数据
- List<Point> points = new ArrayList<>();
- points.add(new Point(0, 0));
- points.add(new Point(1, 2));
- points.add(new Point(2, -20));
- points.add(new Point(3, 0));
- points.add(new Point(4, -1));
- points.add(new Point(5, -1.5));
- points.add(new Point(6, 0));
-
- List<Point> points2 = new ArrayList<>();
- // points2.add(points.get(0));
- // points2.add(points.get(3));
- // points2.add(points.get(6));
- points2.add(new Point(1, -10));
- points2.add(new Point(3, 0));
-
- double area = total_areal_displacement(points, points2, true);
- System.out.println("Total area: " + area);
-
- // points = new ArrayList<>();
- // points.add(new Point(0, 0));
- // points.add(new Point(1, 2));
- // points.add(new Point(1.5, -10));
- // double area = calculatePolygonArea(points);
- // System.out.println(area);
- //
- // Point[] L1 = new Point[2];
- // L1[0] = new Point(1, 2);
- // L1[1] = new Point(2, -2);
- // Point[] L2 = new Point[2];
- // L2[0] = new Point(0, 0);
- // L2[1] = new Point(3, 0);
- // Object[] res = lineIntersection(L1, L2);
- // System.out.println(res[1]);
- }
+ // 测试方法
+ public static void main(String[] args) {
+ // 示例数据
+ List<Point> points = new ArrayList<>();
+ points.add(new Point(0, 0));
+ points.add(new Point(1, 2));
+ points.add(new Point(2, -20));
+ points.add(new Point(3, 0));
+ points.add(new Point(4, -1));
+ points.add(new Point(5, -1.5));
+ points.add(new Point(6, 0));
+
+ List<Point> points2 = new ArrayList<>();
+ // points2.add(points.get(0));
+ // points2.add(points.get(3));
+ // points2.add(points.get(6));
+ points2.add(new Point(1, -10));
+ points2.add(new Point(3, 0));
+
+ double area = total_areal_displacement(points, points2, true);
+ System.out.println("Total area: " + area);
+
+ // points = new ArrayList<>();
+ // points.add(new Point(0, 0));
+ // points.add(new Point(1, 2));
+ // points.add(new Point(1.5, -10));
+ // double area = calculatePolygonArea(points);
+ // System.out.println(area);
+ //
+ // Point[] L1 = new Point[2];
+ // L1[0] = new Point(1, 2);
+ // L1[1] = new Point(2, -2);
+ // Point[] L2 = new Point[2];
+ // L2[0] = new Point(0, 0);
+ // L2[1] = new Point(3, 0);
+ // Object[] res = lineIntersection(L1, L2);
+ // System.out.println(res[1]);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 8da7cca5175..44fc3298d96 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -76,16 +76,18 @@ public class eBUG {
throw new IllegalArgumentException("Parameter e must be non-negative.");
}
- if (m > 2) {
- System.out.println(
- "online sampling mode, "
- + "returning "
- + m
- + " sampled points sorted by time in ascending order");
- } else {
- System.out.println(
- "offline precomputation mode, "
- + "returning each point sorted by dominated significance (DS) in ascending order");
+ if (debug) {
+ if (m > 2) {
+ System.out.println(
+ "online sampling mode, "
+ + "returning "
+ + m
+ + " sampled points sorted by time in ascending order");
+ } else {
+ System.out.println(
+ "offline precomputation mode, "
+ + "returning each point sorted by dominated significance (DS) in ascending order");
+ }
}
// List<Point> results = lineToSimplify.getVertices(); // 浅复制
@@ -177,14 +179,16 @@ public class eBUG {
}
// 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
- if (tri.area > previousEA) {
- previousEA = tri.area;
- }
- // results.get(tri.indices[1]).z = previousEA; // dominated significance
- points.get(tri.indices[1]).z = previousEA; // dominated significance
- resultsBottomUpEliminated.add(points.get(tri.indices[1])); // TODO add的是Point引用，所以没有多用一倍的空间
- if (debug) {
- System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
+ if (m <= 2) { // precomputation mode
+ if (tri.area > previousEA) {
+ previousEA = tri.area;
+ }
+ // results.get(tri.indices[1]).z = previousEA; // dominated significance
+ points.get(tri.indices[1]).z = previousEA; // dominated significance
+ resultsBottomUpEliminated.add(points.get(tri.indices[1])); // TODO add的是Point引用，所以没有多用一倍的空间
+ if (debug) {
+ System.out.println(Arrays.toString(tri.indices) + ", Dominated Sig=" + previousEA);
+ }
}
// 更新相邻三角形
@@ -326,6 +330,8 @@ public class eBUG {
if (m > 2) { // online sampling mode
// 把滞后的淘汰点施加上去，然后返回在线采样结果（也就是返回剩余未被淘汰的点）
+ // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ // 不过既然m>2在线采样模式，所以其实淘汰点的DS本身也没有记录
for (Point p : laggedEliminatedPoints) {
p.markEliminated();
if (debug) {
@@ -337,7 +343,7 @@ public class eBUG {
Point end = points.get(points.size() - 1);
while (start
!= end) { // Point类里增加prev&next指针，这样T'_max{0,k-e}里点的连接关系就有了，这样从Pa开始沿着指针，遍历点数一定不超过e+3
- onlineSampled.add(start); // 注意未淘汰的点的Dominated significance尚未赋值，还都是infinity
+ onlineSampled.add(start);
start = start.next; // when e=0, only traversing three points pa pi pb
}
onlineSampled.add(end);
(iotdb) 12/44: before
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 85ef410387407ce78dbdd2d851fb426f9c3351de
Author: Lei Rui <10...@qq.com>
AuthorDate: Tue Jan 21 21:08:16 2025 +0800
before
---
.../java/org/apache/iotdb/db/query/eBUG/Tool.java | 8 ++--
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 56 +++++++++++++++++-----
2 files changed, 47 insertions(+), 17 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
index 0587b2fa39e..f7c5bbfb668 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/Tool.java
@@ -12,9 +12,9 @@ public class Tool {
return (dArea > 0.0) ? dArea : -dArea; // abs
}
- // 计算多边形面积（鞋带公式）
+ // 计算简单多边形（边不交叉）的面积（鞋带公式）
public static double calculatePolygonArea(List<Point> points) {
- // points多边形顶点，按照逆时针或者顺时针的顺序枚举
+ // points多边形顶点，要求按照逆时针或者顺时针的顺序枚举，否则鞋带公式无法给出正确结果
int n = points.size();
double area = 0;
for (int i = 0; i < n; i++) {
@@ -67,7 +67,7 @@ public class Tool {
return new Object[]{false, null};
}
- // 计算总的多边形面积（通过时间序列扫描交点）
+ // 计算总的多边形面积（通过时间序列扫描交点），默认要求点按照时间戳严格递增排列
public static double total_areal_displacement(List<Point> points, List<Point> points2, boolean debug) {
double totalArea = 0;
int i = 0, j = 0; // i for points, j for points2
@@ -122,7 +122,7 @@ public class Tool {
// polygonArray[k] = polygon.get(k);
// }
- // 计算多边形面积
+ // 计算多边形面积，注意polygon里的点一定是按照顺时针/逆时针几何连接顺序枚举的多边形顶点
double area = calculatePolygonArea(polygon);
if (debug) {
System.out.println("Area = " + area);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 12882823356..289ecf0b913 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -29,15 +29,45 @@ import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
public class eBUG {
public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
- Point pa, Point pi, Point pb) {
- // TODO 寻找最近淘汰的e个点，并且是位于pa~pb之间的
+ int pa_idx, int pi_idx, int pb_idx) {
+ // TODO 复杂度分析
+ // 从最近淘汰的e个点里寻找位于pa~pb之间的：e
+ // 把上一步找到的点和pa pi pb一起按照时间戳递增排序：记c=b-a。如果e<c，那么eloge，否则c
+ // 后面鞋带公式计算面积：min(e+3,c)
// 性质：最近淘汰的那一个点一定是位于pa~pb之间，因为正是这个点的淘汰才使得pi的significance要更新！
// pi: the point whose significance needs updating
// pa: left adjacent non-eliminated point of pi
// pb: right adjacent non-eliminated point of pi
// return a list of points, containing pa,p,pb and points between pa&pa from the e most recently eliminated points,
// order by time in ascending order
+
+ Point pa = lineToSimplify.get(pa_idx);
+ Point pi = lineToSimplify.get(pi_idx);
+ Point pb = lineToSimplify.get(pb_idx);
+
List<Point> res = new ArrayList<>();
+
+// if (recentEliminated.length == 0) { // e=0
+// System.out.println("e=0");
+// res.add(pa);
+// res.add(pi);
+// res.add(pb);
+// return res; // pa pi pb已经按照时间戳递增
+// }
+
+ if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
+ System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
+ res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
+ return res; // 已经按照时间戳递增
+ }
+
+ // 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
+ System.out.println("k>=[c=(b-a-2)]>e, search among e");
+ res.add(pa);
+ res.add(pi);
+ res.add(pb);
+
+ // 包括了c>e=0的情况
for (int idx : recentEliminated) { // e points
if (idx == 0) { // init, not filled by real eliminated points yet
break;
@@ -47,10 +77,10 @@ public class eBUG {
res.add(p);
}
}
- res.add(pa);
- res.add(pi);
- res.add(pb);
+
+ // 按照时间戳递增排序，这是为了后面计算total areal displacement需要
res.sort(Comparator.comparingDouble(p -> p.x)); // 按时间戳排序 TODO 这样的话时间复杂度要变成e*log(e)了吧？
+
return res;
}
@@ -146,9 +176,9 @@ public class eBUG {
// e parameter
List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
- lineToSimplify.get(tri.prev.indices[0]),
- lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2])); // TODO complexity ?
+ tri.prev.indices[0],
+ tri.prev.indices[1],
+ tri.prev.indices[2]); // TODO complexity ?
if (debug) {
System.out.println("(2) update point on the left " + lineToSimplify.get(tri.prev.indices[1]));
for (Point point : pointsForSig) {
@@ -195,9 +225,9 @@ public class eBUG {
// e parameter
List<Point> pointsForSig = findEliminated(lineToSimplify, recentEliminated,
- lineToSimplify.get(tri.next.indices[0]),
- lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2])); // TODO complexity
+ tri.next.indices[0],
+ tri.next.indices[1],
+ tri.next.indices[2]); // TODO complexity
if (debug) {
System.out.println("(2) updating point on the right " + lineToSimplify.get(tri.next.indices[1]));
for (Point point : pointsForSig) {
@@ -234,7 +264,7 @@ public class eBUG {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 10;
+ int n = 100;
int p = 10;
for (int i = 0; i < n; i += p) {
@@ -266,7 +296,7 @@ public class eBUG {
System.out.println("---------------------------------");
// List<Point> results = new ArrayList<>();
// 计算运行时间
- int eParam = 1;
+ int eParam = 10;
long startTime = System.currentTimeMillis();
List<Point> results = buildEffectiveArea(polyline, eParam, false);
// 输出结果
(iotdb) 08/44: fix
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 997e8c0687fe1d02e359985b5b5812190cd3cf55
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jan 17 20:52:57 2025 +0800
fix
---
.../iotdb/db/query/simpiece/Visval_standard.java | 115 +++++----
.../db/query/simpiece/Visval_standard_slow.java | 279 ---------------------
2 files changed, 60 insertions(+), 334 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
index 28f0f26618a..369c0fec918 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.query.simpiece;
import java.io.FileWriter;
import java.io.IOException;
import java.util.*;
+import java.util.stream.Collectors;
// adapted from the open source C++ code
// https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
@@ -159,13 +160,15 @@ public class Visval_standard {
results.get(tri.indices[1]).z = previousEA;
// System.out.println(tri.indices[1] + "," + previousEA);
- System.out.println(Arrays.toString(tri.indices) + "," + previousEA);
+// System.out.println(Arrays.toString(tri.indices) + "," + previousEA);
// 更新相邻三角形
if (tri.prev != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
// 1. 处理旧的tri.prev被标记删除的事情（角色替代）
// triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
- tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只要heap poll到它就跳过就可以
Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
// tri.prev = newPre; // replace TODO can omit
@@ -183,6 +186,8 @@ public class Visval_standard {
}
if (tri.next != null) {
+ // 标记为失效点，同时new一个新的对象接管它的一切数据和前后连接关系，然后更新前后连接关系、更新significance、加入heap使其排好序
+
// 1. 处理旧的tri.next被标记删除的事情（角色替代）
// triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
@@ -211,9 +216,9 @@ public class Visval_standard {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 100000; //1000_000;
+ int n = 1000_000;
- int p = 100;
+ int p = 1000;
for (int i = 0; i < n; i += p) {
Polyline polylineBatch = new Polyline();
for (int j = i; j < Math.min(i + p, n); j++) {
@@ -226,19 +231,19 @@ public class Visval_standard {
polylineList.add(polylineBatch);
}
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- vPoint point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+// try (FileWriter writer = new FileWriter("raw.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
+//
+// // 写入每个点的数据
+// for (int i = 0; i < polyline.size(); i++) {
+// vPoint point = polyline.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// }
+// System.out.println("Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
+// }
System.out.println("---------------------------------");
List<vPoint> results = new ArrayList<>();
@@ -258,47 +263,47 @@ public class Visval_standard {
}
}
- try (FileWriter writer = new FileWriter("fast.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < results.size(); i++) {
- vPoint point = results.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
-
-// System.out.println("---------------------------------");
-// List<List<vPoint>> resultsBatchList = new ArrayList<>();
-// // 计算运行时间
-// int cnt = 0;
-// startTime = System.currentTimeMillis();
-// for (Polyline polylineBatch : polylineList) {
-// List<vPoint> resultsBatch = new ArrayList<>();
-// buildEffectiveArea(polylineBatch, resultsBatch);
-// cnt += resultsBatch.size();
-// resultsBatchList.add(resultsBatch);
-// }
-// // 输出结果
-// endTime = System.currentTimeMillis();
-// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
-// System.out.println(cnt);
+// try (FileWriter writer = new FileWriter("fast.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
//
-// System.out.println("---------------------------------");
-// // 使用 Stream API 合并所有列表
-// List<vPoint> mergedList =
-// resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
-// int sameCnt = 0;
-// for (int i = 0; i < mergedList.size(); i++) {
-// if (mergedList.get(i).z == results.get(i).z) {
-// sameCnt += 1;
+// // 写入每个点的数据
+// for (int i = 0; i < results.size(); i++) {
+// vPoint point = results.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
// }
+// System.out.println("Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
// }
-// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
+
+ System.out.println("---------------------------------");
+ List<List<vPoint>> resultsBatchList = new ArrayList<>();
+ // 计算运行时间
+ int cnt = 0;
+ startTime = System.currentTimeMillis();
+ for (Polyline polylineBatch : polylineList) {
+ List<vPoint> resultsBatch = new ArrayList<>();
+ buildEffectiveArea(polylineBatch, resultsBatch);
+ cnt += resultsBatch.size();
+ resultsBatchList.add(resultsBatch);
+ }
+ // 输出结果
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println(cnt);
+
+ System.out.println("---------------------------------");
+ // 使用 Stream API 合并所有列表
+ List<vPoint> mergedList =
+ resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
+ int sameCnt = 0;
+ for (int i = 0; i < mergedList.size(); i++) {
+ if (mergedList.get(i).z == results.get(i).z) {
+ sameCnt += 1;
+ }
+ }
+ System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
}
// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard_slow.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard_slow.java
deleted file mode 100644
index 33259cc4cff..00000000000
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard_slow.java
+++ /dev/null
@@ -1,279 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing,
-// * software distributed under the License is distributed on an
-// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// * KIND, either express or implied. See the License for the
-// * specific language governing permissions and limitations
-// * under the License.
-// */
-//
-//package org.apache.iotdb.db.query.simpiece;
-//
-//import java.io.FileWriter;
-//import java.io.IOException;
-//import java.util.*;
-//import java.util.stream.Collectors;
-//
-//// adapted from the open source C++ code
-//// https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
-//class Triangle {
-//
-// int[] indices = new int[3];
-// double area;
-// Triangle prev;
-// Triangle next;
-//
-// public Triangle(int index1, int index2, int index3, double area) {
-// this.indices[0] = index1;
-// this.indices[1] = index2;
-// this.indices[2] = index3;
-// this.area = area;
-// }
-//}
-//
-//class vPoint {
-//
-// double x, y, z;
-//
-// public vPoint(double x, double y) {
-// this.x = x;
-// this.y = y;
-// this.z = Double.POSITIVE_INFINITY; // effective area
-// }
-//}
-//
-//class Polyline {
-//
-// private List<vPoint> vertices = new ArrayList<>();
-//
-// public void addVertex(vPoint point) {
-// vertices.add(point);
-// }
-//
-// public List<vPoint> getVertices() {
-// return new ArrayList<>(vertices);
-// }
-//
-// public int size() {
-// return vertices.size();
-// }
-//
-// public vPoint get(int index) {
-// return vertices.get(index);
-// }
-//}
-//
-//public class Visval_standard_slow {
-//
-// // 计算三角形面积
-// private static double triArea(vPoint d0, vPoint d1, vPoint d2) {
-// double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
-// return (double) ((dArea > 0.0) ? dArea : -dArea); // abs
-// }
-//
-// public static void buildEffectiveArea(Polyline lineToSimplify, List<vPoint> results) {
-// results.clear();
-// results.addAll(lineToSimplify.getVertices());
-//
-// int total = lineToSimplify.size();
-// if (total < 3) {
-// return; // 不足 3 个点无法形成三角形
-// }
-//
-// int nTriangles = total - 2;
-// Triangle[] triangles = new Triangle[nTriangles];
-//
-// // 创建所有三角形并计算初始面积
-// for (int i = 1; i < total - 1; i++) {
-// int index1 = i - 1, index2 = i, index3 = i + 1;
-// double area =
-// triArea(
-// lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
-// triangles[i - 1] = new Triangle(index1, index2, index3, area);
-// }
-//
-// // 设置三角形的前后关系
-// for (int i = 0; i < nTriangles; i++) {
-// triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
-// triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
-// }
-//
-// // 使用优先队列构建 minHeap
-// PriorityQueue<Triangle> triangleHeap =
-// new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
-// Collections.addAll(triangleHeap, triangles);
-//
-// double previousEA = -1;
-// while (!triangleHeap.isEmpty()) {
-// // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
-// // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
-// Triangle tri = triangleHeap.poll(); // O(logn)
-//
-//// System.out.println(Arrays.toString(tri.indices));
-//
-// // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
-// if (tri.area > previousEA) {
-// previousEA = tri.area;
-// }
-// results.get(tri.indices[1]).z = previousEA;
-// // System.out.println(tri.indices[1] + "," + previousEA);
-//
-// System.out.println(Arrays.toString(tri.indices) + "," + previousEA);
-//
-// // 更新相邻三角形
-// if (tri.prev != null) {
-// // 前一个三角形连到后一个三角形
-// tri.prev.next = tri.next;
-// tri.prev.indices[2] = tri.indices[2];
-// tri.prev.area =
-// triArea(
-// lineToSimplify.get(tri.prev.indices[0]),
-// lineToSimplify.get(tri.prev.indices[1]),
-// lineToSimplify.get(tri.prev.indices[2]));
-//
-// // 重新加入堆
-// // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
-// // 所以必须通过 remove 和 add 来显式重新插入修改后的元素
-// triangleHeap.remove(tri.prev); // TODO 但是这个操作复杂度是O(n)!!!!
-// triangleHeap.add(tri.prev); // O(logn)
-// }
-//
-// if (tri.next != null) {
-// // 后一个三角形连到前一个三角形
-// tri.next.prev = tri.prev;
-// tri.next.indices[0] = tri.indices[0];
-// tri.next.area =
-// triArea(
-// lineToSimplify.get(tri.next.indices[0]),
-// lineToSimplify.get(tri.next.indices[1]),
-// lineToSimplify.get(tri.next.indices[2]));
-//
-// // 重新加入堆
-// triangleHeap.remove(tri.next);
-// triangleHeap.add(tri.next);
-// }
-// }
-// }
-//
-// public static void main(String[] args) {
-// Polyline polyline = new Polyline();
-// List<Polyline> polylineList = new ArrayList<>();
-// Random rand = new Random(1);
-//// int n = 1000_000;
-// int n = 100000;
-//
-// int p = 100;
-// for (int i = 0; i < n; i += p) {
-// Polyline polylineBatch = new Polyline();
-// for (int j = i; j < Math.min(i + p, n); j++) {
-// double v = rand.nextInt(1000000);
-//
-// polyline.addVertex(new vPoint(j, v));
-//
-// polylineBatch.addVertex(new vPoint(j, v));
-// }
-// polylineList.add(polylineBatch);
-// }
-//
-// System.out.println("---------------------------------");
-// List<vPoint> results = new ArrayList<>();
-// // 计算运行时间
-// long startTime = System.currentTimeMillis();
-// buildEffectiveArea(polyline, results);
-// // 输出结果
-// long endTime = System.currentTimeMillis();
-// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
-// System.out.println(results.size());
-//
-// if (results.size() <= 100) {
-// System.out.println("+++++++++++++++++++");
-// for (int i = 0; i < results.size(); i++) {
-// vPoint point = results.get(i);
-// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
-// }
-// }
-//
-// try (FileWriter writer = new FileWriter("slow.csv")) {
-// // 写入CSV头部
-// writer.append("x,y,z\n");
-//
-// // 写入每个点的数据
-// for (int i = 0; i < results.size(); i++) {
-// vPoint point = results.get(i);
-// writer.append(point.x + "," + point.y + "," + point.z + "\n");
-// }
-// System.out.println("Data has been written");
-// } catch (IOException e) {
-// System.out.println("Error writing to CSV file: " + e.getMessage());
-// }
-//
-//// System.out.println("---------------------------------");
-//// List<List<vPoint>> resultsBatchList = new ArrayList<>();
-//// // 计算运行时间
-//// int cnt = 0;
-//// startTime = System.currentTimeMillis();
-//// for (Polyline polylineBatch : polylineList) {
-//// List<vPoint> resultsBatch = new ArrayList<>();
-//// buildEffectiveArea(polylineBatch, resultsBatch);
-//// cnt += resultsBatch.size();
-//// resultsBatchList.add(resultsBatch);
-//// }
-//// // 输出结果
-//// endTime = System.currentTimeMillis();
-//// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
-//// System.out.println(cnt);
-////
-//// System.out.println("---------------------------------");
-//// // 使用 Stream API 合并所有列表
-//// List<vPoint> mergedList =
-//// resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
-//// int sameCnt = 0;
-//// for (int i = 0; i < mergedList.size(); i++) {
-//// if (mergedList.get(i).z == results.get(i).z) {
-//// sameCnt += 1;
-//// }
-//// }
-//// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
-//
-//
-//
-//
-//
-//
-//
-//// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
-//// 23089};
-//// for (int i = 0; i < vlist.length; i++) {
-//// polyline.addVertex(new vPoint(i, vlist[i]));
-//// }
-////
-//// ArrayList<Double> v = new ArrayList<>();
-//// String filePath = "D://datasets//regular//New York Stock Exchange//merged_prices.csv";
-//// try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
-//// String line;
-//// int count = 0;
-//// while ((line = br.readLine()) != null && count < 1000) {
-//// String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
-//// v.add(Double.parseDouble(columns[0])); // 读取第一列数据
-//// count++;
-//// }
-//// } catch (Exception e) {
-//// e.printStackTrace();
-//// }
-//// // 打印数据长度
-//// System.out.println("Data length: " + v.size());
-//// for (int i = 0; i < v.size(); i++) {
-//// polyline.addVertex(new vPoint(i, v.get(i)));
-//// }
-// }
-//}
(iotdb) 06/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 308affe76884dde28382322accec60b8ad35b6a3
Author: Lei Rui <10...@qq.com>
AuthorDate: Fri Jan 17 18:14:12 2025 +0800
add
---
.../iotdb/db/query/simpiece/Visval_standard.java | 469 ++++++++++++---------
.../db/query/simpiece/Visval_standard_slow.java | 275 ++++++++++++
2 files changed, 546 insertions(+), 198 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
index 5471f62f65f..80aa0995809 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard.java
@@ -19,233 +19,306 @@
package org.apache.iotdb.db.query.simpiece;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Random;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.*;
import java.util.stream.Collectors;
// adapted from the open source C++ code
// https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
class Triangle {
- int[] indices = new int[3];
- double area;
- Triangle prev;
- Triangle next;
-
- public Triangle(int index1, int index2, int index3, double area) {
- this.indices[0] = index1;
- this.indices[1] = index2;
- this.indices[2] = index3;
- this.area = area;
- }
+ int[] indices = new int[3];
+ double area;
+ Triangle prev;
+ Triangle next;
+ boolean isDeleted;
+
+ public Triangle(int index1, int index2, int index3, double area) {
+ this.indices[0] = index1;
+ this.indices[1] = index2;
+ this.indices[2] = index3;
+ this.area = area;
+ this.isDeleted = false; // flag for removal. Avoid using heap.remove(x) as it is O(n) complexity
+ }
+
+ public Triangle(Triangle oldTri) {
+ // deep copy and inherit connection
+
+ this.indices[0] = oldTri.indices[0];
+ this.indices[1] = oldTri.indices[1];
+ this.indices[2] = oldTri.indices[2];
+ this.area = oldTri.area;
+ this.prev = oldTri.prev;
+ this.next = oldTri.next;
+
+ // TODO important! inherit connection relationship to this new point
+ if (this.prev != null) { // previous point to this new point
+ this.prev.next = this;
+ }
+ if (this.next != null) { // next point to this new point
+ this.next.prev = this;
+ }
+
+ this.isDeleted = false; // this new triangle is not deleted
+ }
+
+ public boolean isValid() {
+ return !isDeleted;
+ }
+
+ public void markDeleted() {
+ this.isDeleted = true;
+ }
}
class vPoint {
- double x, y, z;
+ double x, y, z;
- public vPoint(double x, double y) {
- this.x = x;
- this.y = y;
- this.z = Double.POSITIVE_INFINITY; // effective area
- }
+ public vPoint(double x, double y) {
+ this.x = x;
+ this.y = y;
+ this.z = Double.POSITIVE_INFINITY; // effective area
+ }
}
class Polyline {
- private List<vPoint> vertices = new ArrayList<>();
+ private List<vPoint> vertices = new ArrayList<>();
- public void addVertex(vPoint point) {
- vertices.add(point);
- }
+ public void addVertex(vPoint point) {
+ vertices.add(point);
+ }
- public List<vPoint> getVertices() {
- return new ArrayList<>(vertices);
- }
+ public List<vPoint> getVertices() {
+ return new ArrayList<>(vertices);
+ }
- public int size() {
- return vertices.size();
- }
+ public int size() {
+ return vertices.size();
+ }
- public vPoint get(int index) {
- return vertices.get(index);
- }
+ public vPoint get(int index) {
+ return vertices.get(index);
+ }
}
public class Visval_standard {
- // 计算三角形面积
- private static double triArea(vPoint d0, vPoint d1, vPoint d2) {
- double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
- return (double) ((dArea > 0.0) ? dArea : -dArea); // abs
- }
-
- public static void buildEffectiveArea(Polyline lineToSimplify, List<vPoint> results) {
- results.clear();
- results.addAll(lineToSimplify.getVertices());
-
- int total = lineToSimplify.size();
- if (total < 3) {
- return; // 不足 3 个点无法形成三角形
+ // 计算三角形面积
+ private static double triArea(vPoint d0, vPoint d1, vPoint d2) {
+ double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
+ return (double) ((dArea > 0.0) ? dArea : -dArea); // abs
}
- int nTriangles = total - 2;
- Triangle[] triangles = new Triangle[nTriangles];
-
- // 创建所有三角形并计算初始面积
- for (int i = 1; i < total - 1; i++) {
- int index1 = i - 1, index2 = i, index3 = i + 1;
- double area =
- triArea(
- lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
- triangles[i - 1] = new Triangle(index1, index2, index3, area);
+ public static void buildEffectiveArea(Polyline lineToSimplify, List<vPoint> results) {
+ results.clear();
+ results.addAll(lineToSimplify.getVertices()); // TODO debug
+
+ int total = lineToSimplify.size();
+ if (total < 3) {
+ return; // 不足 3 个点无法形成三角形
+ }
+
+ int nTriangles = total - 2;
+ Triangle[] triangles = new Triangle[nTriangles];
+
+ // 创建所有三角形并计算初始面积
+ for (int i = 1; i < total - 1; i++) {
+ int index1 = i - 1, index2 = i, index3 = i + 1;
+ double area =
+ triArea(
+ lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
+ triangles[i - 1] = new Triangle(index1, index2, index3, area);
+ }
+
+ // 设置三角形的前后关系
+ for (int i = 0; i < nTriangles; i++) { // TODO 这个可以和上一个循环合并吗？？好像不可以
+ triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+ triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ }
+
+ // 使用优先队列构建 minHeap
+ PriorityQueue<Triangle> triangleHeap =
+ new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+ Collections.addAll(triangleHeap, triangles); // complexity TODO
+
+ double previousEA = -1;
+ while (!triangleHeap.isEmpty()) {
+ // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+ // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+ Triangle tri = triangleHeap.poll(); // O(logn)
+
+ // 如果该三角形已经被删除，跳过. Avoid using heap.remove(x) as it is O(n) complexity
+ // 而且除了heap里，已经没有其他节点会和它关联了，所有的connections关系已经迁移到新的角色替代节点上
+ if (tri.isDeleted) {
+ continue;
+ }
+
+ // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
+ if (tri.area > previousEA) {
+ previousEA = tri.area;
+ }
+ results.get(tri.indices[1]).z = previousEA;
+ // System.out.println(tri.indices[1] + "," + previousEA);
+
+ // 更新相邻三角形
+ if (tri.prev != null) {
+ // 1. 处理旧的tri.prev被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.prev); // Avoid using heap.remove(x) as it is O(n) complexity!
+ tri.prev.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+
+ Triangle newPre = new Triangle(tri.prev); // deep copy and inherit connection
+ tri.prev = newPre; // replace
+
+ // 2. 处理pi被淘汰引起tri.prev被更新的事情
+ // 前一个三角形连到后一个三角形
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next后面可能会因为处理旧的tri.next被标记删除的事情被换掉！到时候要重新赋值！
+ tri.prev.indices[2] = tri.indices[2];
+ tri.prev.area =
+ triArea(
+ lineToSimplify.get(tri.prev.indices[0]),
+ lineToSimplify.get(tri.prev.indices[1]),
+ lineToSimplify.get(tri.prev.indices[2]));
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.prev); // O(logn) 注意加入的是一个新的对象isDeleted=false
+ }
+
+ if (tri.next != null) {
+ // 1. 处理旧的tri.next被标记删除的事情（角色替代）
+ // triangleHeap.remove(tri.next); // Avoid using heap.remove(x) as it is O(n) complexity
+ tri.next.markDeleted(); // O(1) 这个点标记为废掉，前后关联都砍断，但是不remove因为那太耗时，只有poll到它就跳过就可以
+
+ Triangle newNext = new Triangle(tri.next); // deep copy and inherit connection
+ tri.next = newNext; // replace
+
+ if (tri.prev != null) {
+ tri.prev.next = tri.next; // ATTENTION!!!: 这里的tri.next已经被换掉！所以之前的要重新赋值！
+ }
+
+ // 2. 处理pi被淘汰引起tri.next被更新的事情
+ tri.next.prev = tri.prev; // 注意此时tri.prev已经是替代后的节点，tri.next也是，从而被标记为废点的前后关联真正砍断
+ tri.next.indices[0] = tri.indices[0];
+ tri.next.area =
+ triArea(
+ lineToSimplify.get(tri.next.indices[0]),
+ lineToSimplify.get(tri.next.indices[1]),
+ lineToSimplify.get(tri.next.indices[2]));
+
+ // 重新加入堆
+ // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+ // 所以必须通过add 来显式重新插入修改后的元素
+ triangleHeap.add(tri.next); // 注意加入的是一个新的对象isDeleted=false
+ }
+ }
}
- // 设置三角形的前后关系
- for (int i = 0; i < nTriangles; i++) {
- triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
- triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+ public static void main(String[] args) {
+ Polyline polyline = new Polyline();
+ List<Polyline> polylineList = new ArrayList<>();
+ Random rand = new Random(1);
+ int n = 10000; //1000_000;
+
+ int p = 100;
+ for (int i = 0; i < n; i += p) {
+ Polyline polylineBatch = new Polyline();
+ for (int j = i; j < Math.min(i + p, n); j++) {
+ double v = rand.nextInt(1000);
+
+ polyline.addVertex(new vPoint(j, v));
+
+ polylineBatch.addVertex(new vPoint(j, v));
+ }
+ polylineList.add(polylineBatch);
+ }
+
+ System.out.println("---------------------------------");
+ List<vPoint> results = new ArrayList<>();
+ // 计算运行时间
+ long startTime = System.currentTimeMillis();
+ buildEffectiveArea(polyline, results);
+ // 输出结果
+ long endTime = System.currentTimeMillis();
+ System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println(results.size());
+
+ if (results.size() <= 100) {
+ System.out.println("+++++++++++++++++++");
+ for (int i = 0; i < results.size(); i++) {
+ vPoint point = results.get(i);
+ System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ }
+ }
+
+ try (FileWriter writer = new FileWriter("fast.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (int i = 0; i < results.size(); i++) {
+ vPoint point = results.get(i);
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println("Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
+
+ System.out.println("---------------------------------");
+ List<List<vPoint>> resultsBatchList = new ArrayList<>();
+ // 计算运行时间
+ int cnt = 0;
+ startTime = System.currentTimeMillis();
+ for (Polyline polylineBatch : polylineList) {
+ List<vPoint> resultsBatch = new ArrayList<>();
+ buildEffectiveArea(polylineBatch, resultsBatch);
+ cnt += resultsBatch.size();
+ resultsBatchList.add(resultsBatch);
+ }
+ // 输出结果
+ endTime = System.currentTimeMillis();
+ System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println(cnt);
+
+ System.out.println("---------------------------------");
+ // 使用 Stream API 合并所有列表
+ List<vPoint> mergedList =
+ resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
+ int sameCnt = 0;
+ for (int i = 0; i < mergedList.size(); i++) {
+ if (mergedList.get(i).z == results.get(i).z) {
+ sameCnt += 1;
+ }
+ }
+ System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
}
- // 使用优先队列构建 minHeap
- PriorityQueue<Triangle> triangleHeap =
- new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
- Collections.addAll(triangleHeap, triangles);
-
- double previousEA = -1;
- while (!triangleHeap.isEmpty()) {
- // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
- // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
- Triangle tri = triangleHeap.poll();
-
- // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
- if (tri.area > previousEA) {
- previousEA = tri.area;
- }
- results.get(tri.indices[1]).z = previousEA;
- // System.out.println(tri.indices[1] + "," + previousEA);
-
- // 更新相邻三角形
- if (tri.prev != null) {
- // 前一个三角形连到后一个三角形
- tri.prev.next = tri.next;
- tri.prev.indices[2] = tri.indices[2];
- tri.prev.area =
- triArea(
- lineToSimplify.get(tri.prev.indices[0]),
- lineToSimplify.get(tri.prev.indices[1]),
- lineToSimplify.get(tri.prev.indices[2]));
-
- // 重新加入堆
- // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
- // 所以必须通过 remove 和 add 来显式重新插入修改后的元素
- triangleHeap.remove(tri.prev);
- triangleHeap.add(tri.prev);
- }
-
- if (tri.next != null) {
- // 后一个三角形连到前一个三角形
- tri.next.prev = tri.prev;
- tri.next.indices[0] = tri.indices[0];
- tri.next.area =
- triArea(
- lineToSimplify.get(tri.next.indices[0]),
- lineToSimplify.get(tri.next.indices[1]),
- lineToSimplify.get(tri.next.indices[2]));
-
- // 重新加入堆
- triangleHeap.remove(tri.next);
- triangleHeap.add(tri.next);
- }
- }
- }
-
- public static void main(String[] args) {
- Polyline polyline = new Polyline();
- List<Polyline> polylineList = new ArrayList<>();
- Random rand = new Random();
- int n = 1000_000;
- int p = 10;
- for (int i = 0; i < n; i += p) {
- Polyline polylineBatch = new Polyline();
- for (int j = i; j < Math.min(i + p, n); j++) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new vPoint(j, v));
- polylineBatch.addVertex(new vPoint(j, v));
- }
- polylineList.add(polylineBatch);
- }
-
- System.out.println("---------------------------------");
- List<vPoint> results = new ArrayList<>();
- // 计算运行时间
- long startTime = System.currentTimeMillis();
- buildEffectiveArea(polyline, results);
- // 输出结果
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(results.size());
-
- // for (int i = 0; i < results.size(); i++) {
- // vPoint point = results.get(i);
- // System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+ // float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
+ // 23089};
+ // for (int i = 0; i < vlist.length; i++) {
+ // polyline.addVertex(new vPoint(i, vlist[i]));
// }
- System.out.println("---------------------------------");
- List<List<vPoint>> resultsBatchList = new ArrayList<>();
- // 计算运行时间
- int cnt = 0;
- startTime = System.currentTimeMillis();
- for (Polyline polylineBatch : polylineList) {
- List<vPoint> resultsBatch = new ArrayList<>();
- buildEffectiveArea(polylineBatch, resultsBatch);
- cnt += resultsBatch.size();
- resultsBatchList.add(resultsBatch);
- }
- // 输出结果
- endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(cnt);
-
- System.out.println("---------------------------------");
- // 使用 Stream API 合并所有列表
- List<vPoint> mergedList =
- resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
- int sameCnt = 0;
- for (int i = 0; i < mergedList.size(); i++) {
- if (mergedList.get(i).z == results.get(i).z) {
- sameCnt += 1;
- }
- }
- System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
- }
-
- // float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
- // 23089};
- // for (int i = 0; i < vlist.length; i++) {
- // polyline.addVertex(new vPoint(i, vlist[i]));
- // }
-
- // ArrayList<Double> v = new ArrayList<>();
- // String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
- // try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
- // String line;
- // int count = 0;
- // while ((line = br.readLine()) != null && count < 1000) {
- // String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
- // v.add(Double.parseDouble(columns[0])); // 读取第一列数据
- // count++;
- // }
- // } catch (Exception e) {
- // e.printStackTrace();
- // }
- // // 打印数据长度
- // System.out.println("Data length: " + v.size());
- // for (int i = 0; i < v.size(); i++) {
- // polyline.addVertex(new vPoint(i, v.get(i)));
- // }
+ // ArrayList<Double> v = new ArrayList<>();
+ // String filePath = "D://desktop//数据集//New York Stock Exchange//merged_prices.csv";
+ // try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+ // String line;
+ // int count = 0;
+ // while ((line = br.readLine()) != null && count < 1000) {
+ // String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
+ // v.add(Double.parseDouble(columns[0])); // 读取第一列数据
+ // count++;
+ // }
+ // } catch (Exception e) {
+ // e.printStackTrace();
+ // }
+ // // 打印数据长度
+ // System.out.println("Data length: " + v.size());
+ // for (int i = 0; i < v.size(); i++) {
+ // polyline.addVertex(new vPoint(i, v.get(i)));
+ // }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard_slow.java b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard_slow.java
new file mode 100644
index 00000000000..929fa304568
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/simpiece/Visval_standard_slow.java
@@ -0,0 +1,275 @@
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+//
+//package org.apache.iotdb.db.query.simpiece;
+//
+//import java.io.FileWriter;
+//import java.io.IOException;
+//import java.util.*;
+//import java.util.stream.Collectors;
+//
+//// adapted from the open source C++ code
+//// https://github.com/ofZach/Visvalingam-Whyatt/blob/master/src/testApp.cpp
+//class Triangle {
+//
+// int[] indices = new int[3];
+// double area;
+// Triangle prev;
+// Triangle next;
+//
+// public Triangle(int index1, int index2, int index3, double area) {
+// this.indices[0] = index1;
+// this.indices[1] = index2;
+// this.indices[2] = index3;
+// this.area = area;
+// }
+//}
+//
+//class vPoint {
+//
+// double x, y, z;
+//
+// public vPoint(double x, double y) {
+// this.x = x;
+// this.y = y;
+// this.z = Double.POSITIVE_INFINITY; // effective area
+// }
+//}
+//
+//class Polyline {
+//
+// private List<vPoint> vertices = new ArrayList<>();
+//
+// public void addVertex(vPoint point) {
+// vertices.add(point);
+// }
+//
+// public List<vPoint> getVertices() {
+// return new ArrayList<>(vertices);
+// }
+//
+// public int size() {
+// return vertices.size();
+// }
+//
+// public vPoint get(int index) {
+// return vertices.get(index);
+// }
+//}
+//
+//public class Visval_standard_slow {
+//
+// // 计算三角形面积
+// private static double triArea(vPoint d0, vPoint d1, vPoint d2) {
+// double dArea = ((d1.x - d0.x) * (d2.y - d0.y) - (d2.x - d0.x) * (d1.y - d0.y)) / 2.0;
+// return (double) ((dArea > 0.0) ? dArea : -dArea); // abs
+// }
+//
+// public static void buildEffectiveArea(Polyline lineToSimplify, List<vPoint> results) {
+// results.clear();
+// results.addAll(lineToSimplify.getVertices());
+//
+// int total = lineToSimplify.size();
+// if (total < 3) {
+// return; // 不足 3 个点无法形成三角形
+// }
+//
+// int nTriangles = total - 2;
+// Triangle[] triangles = new Triangle[nTriangles];
+//
+// // 创建所有三角形并计算初始面积
+// for (int i = 1; i < total - 1; i++) {
+// int index1 = i - 1, index2 = i, index3 = i + 1;
+// double area =
+// triArea(
+// lineToSimplify.get(index1), lineToSimplify.get(index2), lineToSimplify.get(index3));
+// triangles[i - 1] = new Triangle(index1, index2, index3, area);
+// }
+//
+// // 设置三角形的前后关系
+// for (int i = 0; i < nTriangles; i++) {
+// triangles[i].prev = (i == 0 ? null : triangles[i - 1]);
+// triangles[i].next = (i == nTriangles - 1 ? null : triangles[i + 1]);
+// }
+//
+// // 使用优先队列构建 minHeap
+// PriorityQueue<Triangle> triangleHeap =
+// new PriorityQueue<>(Comparator.comparingDouble(t -> t.area));
+// Collections.addAll(triangleHeap, triangles);
+//
+// double previousEA = -1;
+// while (!triangleHeap.isEmpty()) {
+// // 注意peek只需要直接访问该位置的元素，不涉及任何重排或堆化操作
+// // 而poll是删除堆顶元素，需要重新堆化以维护堆的性质，复杂度是O(logk),k是当前堆的大小
+// Triangle tri = triangleHeap.poll(); // O(logn)
+//
+// // 更新当前点的重要性（z 轴存储effective area,这是一个单调增的指标）
+// if (tri.area > previousEA) {
+// previousEA = tri.area;
+// }
+// results.get(tri.indices[1]).z = previousEA;
+// // System.out.println(tri.indices[1] + "," + previousEA);
+//
+// // 更新相邻三角形
+// if (tri.prev != null) {
+// // 前一个三角形连到后一个三角形
+// tri.prev.next = tri.next;
+// tri.prev.indices[2] = tri.indices[2];
+// tri.prev.area =
+// triArea(
+// lineToSimplify.get(tri.prev.indices[0]),
+// lineToSimplify.get(tri.prev.indices[1]),
+// lineToSimplify.get(tri.prev.indices[2]));
+//
+// // 重新加入堆
+// // 在 Java 的 PriorityQueue 中，修改元素的属性不会自动更新堆的顺序
+// // 所以必须通过 remove 和 add 来显式重新插入修改后的元素
+// triangleHeap.remove(tri.prev); // TODO 但是这个操作复杂度是O(n)!!!!
+// triangleHeap.add(tri.prev); // O(logn)
+// }
+//
+// if (tri.next != null) {
+// // 后一个三角形连到前一个三角形
+// tri.next.prev = tri.prev;
+// tri.next.indices[0] = tri.indices[0];
+// tri.next.area =
+// triArea(
+// lineToSimplify.get(tri.next.indices[0]),
+// lineToSimplify.get(tri.next.indices[1]),
+// lineToSimplify.get(tri.next.indices[2]));
+//
+// // 重新加入堆
+// triangleHeap.remove(tri.next);
+// triangleHeap.add(tri.next);
+// }
+// }
+// }
+//
+// public static void main(String[] args) {
+// Polyline polyline = new Polyline();
+// List<Polyline> polylineList = new ArrayList<>();
+// Random rand = new Random(1);
+//// int n = 1000_000;
+// int n = 10000;
+//
+// int p = 100;
+// for (int i = 0; i < n; i += p) {
+// Polyline polylineBatch = new Polyline();
+// for (int j = i; j < Math.min(i + p, n); j++) {
+// double v = rand.nextInt(1000);
+//
+// polyline.addVertex(new vPoint(j, v));
+//
+// polylineBatch.addVertex(new vPoint(j, v));
+// }
+// polylineList.add(polylineBatch);
+// }
+//
+// System.out.println("---------------------------------");
+// List<vPoint> results = new ArrayList<>();
+// // 计算运行时间
+// long startTime = System.currentTimeMillis();
+// buildEffectiveArea(polyline, results);
+// // 输出结果
+// long endTime = System.currentTimeMillis();
+// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+// System.out.println(results.size());
+//
+// if (results.size() <= 100) {
+// System.out.println("+++++++++++++++++++");
+// for (int i = 0; i < results.size(); i++) {
+// vPoint point = results.get(i);
+// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+// }
+// }
+//
+// try (FileWriter writer = new FileWriter("slow.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
+//
+// // 写入每个点的数据
+// for (int i = 0; i < results.size(); i++) {
+// vPoint point = results.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// }
+// System.out.println("Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
+// }
+//
+// System.out.println("---------------------------------");
+// List<List<vPoint>> resultsBatchList = new ArrayList<>();
+// // 计算运行时间
+// int cnt = 0;
+// startTime = System.currentTimeMillis();
+// for (Polyline polylineBatch : polylineList) {
+// List<vPoint> resultsBatch = new ArrayList<>();
+// buildEffectiveArea(polylineBatch, resultsBatch);
+// cnt += resultsBatch.size();
+// resultsBatchList.add(resultsBatch);
+// }
+// // 输出结果
+// endTime = System.currentTimeMillis();
+// System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
+// System.out.println(cnt);
+//
+// System.out.println("---------------------------------");
+// // 使用 Stream API 合并所有列表
+// List<vPoint> mergedList =
+// resultsBatchList.stream().flatMap(List::stream).collect(Collectors.toList());
+// int sameCnt = 0;
+// for (int i = 0; i < mergedList.size(); i++) {
+// if (mergedList.get(i).z == results.get(i).z) {
+// sameCnt += 1;
+// }
+// }
+// System.out.println("sameCnt=" + sameCnt + ", percent=" + sameCnt * 1.0 / mergedList.size());
+//
+//
+//
+//
+//
+//
+//
+//// float[] vlist = new float[]{11346, 33839, 35469, 23108, 22812, 5519, 5526, 4865, 5842,
+//// 23089};
+//// for (int i = 0; i < vlist.length; i++) {
+//// polyline.addVertex(new vPoint(i, vlist[i]));
+//// }
+////
+//// ArrayList<Double> v = new ArrayList<>();
+//// String filePath = "D://datasets//regular//New York Stock Exchange//merged_prices.csv";
+//// try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+//// String line;
+//// int count = 0;
+//// while ((line = br.readLine()) != null && count < 1000) {
+//// String[] columns = line.split(","); // 假设 CSV 文件以逗号分隔
+//// v.add(Double.parseDouble(columns[0])); // 读取第一列数据
+//// count++;
+//// }
+//// } catch (Exception e) {
+//// e.printStackTrace();
+//// }
+//// // 打印数据长度
+//// System.out.println("Data length: " + v.size());
+//// for (int i = 0; i < v.size(); i++) {
+//// polyline.addVertex(new vPoint(i, v.get(i)));
+//// }
+// }
+//}
(iotdb) 13/44: f
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8dfbcb233e1d31c73cf0ea39e9286df714d484b1
Author: Lei Rui <10...@qq.com>
AuthorDate: Wed Jan 22 00:11:17 2025 +0800
f
---
.../db/query/eBUG/TimestampPriorityQueue.java | 117 +++++++++++++++++++++
.../java/org/apache/iotdb/db/query/eBUG/eBUG.java | 74 +++++++------
.../db/query/eBUG/{eBUG.java => eBUG_old.java} | 98 +++++++++--------
3 files changed, 207 insertions(+), 82 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java
new file mode 100644
index 00000000000..222790aed42
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/TimestampPriorityQueue.java
@@ -0,0 +1,117 @@
+package org.apache.iotdb.db.query.eBUG;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+import java.util.*;
+
+public class TimestampPriorityQueue {
+ private final int e; // 队列的最大大小
+ private final LinkedList<Point> insertionOrder; // 保持插入顺序
+ public final PriorityQueue<Point> queue; // 按时间戳排序的优先队列
+
+ // 构造函数：初始化队列大小、插入顺序队列和优先队列
+ public TimestampPriorityQueue(int e) {
+ this.e = e;
+ this.insertionOrder = new LinkedList<>();
+ this.queue = new PriorityQueue<>(Comparator.comparingDouble(p -> p.timestamp));
+ }
+
+ // 插入新的元素
+ public void insert(Point newPoint) {
+ // 如果队列已满，移除最早插入的点
+ if (queue.size() == e) {
+ Point removedPoint = insertionOrder.removeFirst(); // 移除最早加入的点
+ queue.remove(removedPoint); // 从优先队列中移除该点
+ }
+
+ // 将新点添加到插入顺序队列中
+ insertionOrder.addLast(newPoint);
+
+ // 插入到优先队列，保持时间戳顺序
+ queue.offer(newPoint);
+ }
+
+ // 获取队列中的所有元素（按时间戳排序）
+ public List<Point> getQueue() {
+ return new ArrayList<>(queue);
+ }
+
+ // Point 类，代表一个带时间戳的点
+ public static class Point {
+ int value;
+ long timestamp;
+
+ public Point(int value, long timestamp) {
+ this.value = value;
+ this.timestamp = timestamp;
+ }
+
+ @Override
+ public String toString() {
+ return "Value: " + value + ", Timestamp: " + timestamp;
+ }
+ }
+
+ public static void main(String[] args) {
+ TimestampPriorityQueue tq = new TimestampPriorityQueue(3);
+
+ tq.insert(new Point(10, 1000));
+ tq.insert(new Point(20, 2000));
+ tq.insert(new Point(15, 1500));
+ tq.insert(new Point(25, 2500));
+
+ // 打印队列内容
+ for (Point point : tq.getQueue()) {
+ System.out.println(point);
+ }
+ }
+}
+
+
+//public class TimestampPriorityQueue { // for managing the e most recently eliminated points, sorted by time in ascending order
+// private final int e;
+// public final PriorityQueue<Point> queue;
+//
+// // 构造函数：初始化优先队列的最大大小
+// public TimestampPriorityQueue(int e) {
+// this.e = e;
+// this.queue = new PriorityQueue<>(Comparator.comparingDouble(p -> p.x));
+// }
+//
+// // 插入新的元素
+// public void insert(Point newPoint) {
+// // 如果队列已满，移除最小（最早）的元素
+// if (queue.size() == e) {
+// queue.poll();
+// }
+//
+// // 插入元素，队列会根据时间戳自动排序
+// queue.offer(newPoint);
+// }
+//
+// // 获取队列中的所有元素
+// public List<Point> getQueue() {
+// return new ArrayList<>(queue);
+// }
+//
+// public static void main(String[] args) {
+// TimestampPriorityQueue tq = new TimestampPriorityQueue(3);
+//
+// tq.insert(new Point(10, 1000));
+// tq.insert(new Point(20, 2000));
+// tq.insert(new Point(15, 1500));
+// tq.insert(new Point(25, 2500));
+//
+// // 打印队列内容
+// long tmp = 2000;
+// for (Point point : tq.getQueue()) {
+// point.x = tmp--;
+// System.out.println("Value: " + point.x + ", Timestamp: " + point.y);
+// }
+//
+// System.out.println(tq.queue.poll());
+// }
+//}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
index 289ecf0b913..62e67a0f1f9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.query.eBUG;
-import java.io.FileWriter;
-import java.io.IOException;
import java.util.*;
import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
@@ -56,13 +54,13 @@ public class eBUG {
// }
if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
- System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
+// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
return res; // 已经按照时间戳递增
}
// 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
- System.out.println("k>=[c=(b-a-2)]>e, search among e");
+// System.out.println("k>=[c=(b-a-2)]>e, search among e");
res.add(pa);
res.add(pi);
res.add(pb);
@@ -264,7 +262,7 @@ public class eBUG {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 100;
+ int n = 1000_0000;
int p = 10;
for (int i = 0; i < n; i += p) {
@@ -279,19 +277,19 @@ public class eBUG {
polylineList.add(polylineBatch);
}
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+// try (FileWriter writer = new FileWriter("raw.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
+//
+// // 写入每个点的数据
+// for (int i = 0; i < polyline.size(); i++) {
+// Point point = polyline.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// }
+// System.out.println("Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
+// }
System.out.println("---------------------------------");
// List<Point> results = new ArrayList<>();
@@ -304,27 +302,27 @@ public class eBUG {
System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
System.out.println(results.size());
- if (results.size() <= 100) {
- System.out.println("+++++++++++++++++++");
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
- }
- }
-
- try (FileWriter writer = new FileWriter("fast.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+// if (results.size() <= 100) {
+// System.out.println("+++++++++++++++++++");
+// for (int i = 0; i < results.size(); i++) {
+// Point point = results.get(i);
+// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+// }
+// }
- // 写入每个点的数据
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+// try (FileWriter writer = new FileWriter("fast.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
+//
+// // 写入每个点的数据
+// for (int i = 0; i < results.size(); i++) {
+// Point point = results.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// }
+// System.out.println("Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
+// }
// System.out.println("---------------------------------");
// List<List<Point>> resultsBatchList = new ArrayList<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old.java
similarity index 86%
copy from server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
copy to server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old.java
index 289ecf0b913..81825ff1b79 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/eBUG_old.java
@@ -19,15 +19,16 @@
package org.apache.iotdb.db.query.eBUG;
-import java.io.FileWriter;
-import java.io.IOException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
import java.util.*;
import static org.apache.iotdb.db.query.eBUG.Tool.total_areal_displacement;
import static org.apache.iotdb.db.query.eBUG.Tool.triArea;
-public class eBUG {
+public class eBUG_old {
public static List<Point> findEliminated(Polyline lineToSimplify, int[] recentEliminated,
int pa_idx, int pi_idx, int pb_idx) {
// TODO 复杂度分析
@@ -56,13 +57,13 @@ public class eBUG {
// }
if (recentEliminated.length >= (pb_idx - pa_idx - 2)) { // e >= the total number of eliminated points in this region
- System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
+// System.out.println("[c=(b-a-2)]<=e, use T directly"); // worst case下, k=c
res = lineToSimplify.getVertices().subList(pa_idx, pb_idx + 1); // 左闭右开
return res; // 已经按照时间戳递增
}
// 从最近淘汰的e个点里寻找位于pa~pb之间的点，找到的有可能小于e个点
- System.out.println("k>=[c=(b-a-2)]>e, search among e");
+// System.out.println("k>=[c=(b-a-2)]>e, search among e");
res.add(pa);
res.add(pi);
res.add(pb);
@@ -264,7 +265,7 @@ public class eBUG {
Polyline polyline = new Polyline();
List<Polyline> polylineList = new ArrayList<>();
Random rand = new Random(1);
- int n = 100;
+ int n = 100_0000;
int p = 10;
for (int i = 0; i < n; i += p) {
@@ -279,52 +280,61 @@ public class eBUG {
polylineList.add(polylineBatch);
}
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
-
- // 写入每个点的数据
- for (int i = 0; i < polyline.size(); i++) {
- Point point = polyline.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+// try (FileWriter writer = new FileWriter("raw.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
+//
+// // 写入每个点的数据
+// for (int i = 0; i < polyline.size(); i++) {
+// Point point = polyline.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// }
+// System.out.println("Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
+// }
System.out.println("---------------------------------");
// List<Point> results = new ArrayList<>();
// 计算运行时间
- int eParam = 10;
- long startTime = System.currentTimeMillis();
- List<Point> results = buildEffectiveArea(polyline, eParam, false);
- // 输出结果
- long endTime = System.currentTimeMillis();
- System.out.println("Time taken to reduce points: " + (endTime - startTime) + "ms");
- System.out.println(results.size());
-
- if (results.size() <= 100) {
- System.out.println("+++++++++++++++++++");
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+// int eParam = 10;
+ try (PrintWriter writer = new PrintWriter(new File("exp.csv"))) {
+ int[] eParamList = {0, 1, 100, 500, 1000, 5000, 10000, 50000, 10_0000, 50_0000, 100_0000, 200_0000};
+// for (int eParam = 0; eParam < 2 * n; eParam += 1000) {
+ for (int eParam : eParamList) {
+ long startTime = System.currentTimeMillis();
+ List<Point> results = buildEffectiveArea(polyline, eParam, false);
+ // 输出结果
+ long endTime = System.currentTimeMillis();
+ System.out.println("n=" + n + ", e=" + eParam + ", Time taken to reduce points: " + (endTime - startTime) + "ms");
+ System.out.println(results.size());
+ writer.println(n + "," + eParam + "," + (endTime - startTime));
}
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
}
- try (FileWriter writer = new FileWriter("fast.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
+// if (results.size() <= 100) {
+// System.out.println("+++++++++++++++++++");
+// for (int i = 0; i < results.size(); i++) {
+// Point point = results.get(i);
+// System.out.println("Point: (" + point.x + ", " + point.y + ", " + point.z + ")");
+// }
+// }
- // 写入每个点的数据
- for (int i = 0; i < results.size(); i++) {
- Point point = results.get(i);
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println("Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+// try (FileWriter writer = new FileWriter("fast.csv")) {
+// // 写入CSV头部
+// writer.append("x,y,z\n");
+//
+// // 写入每个点的数据
+// for (int i = 0; i < results.size(); i++) {
+// Point point = results.get(i);
+// writer.append(point.x + "," + point.y + "," + point.z + "\n");
+// }
+// System.out.println("Data has been written");
+// } catch (IOException e) {
+// System.out.println("Error writing to CSV file: " + e.getMessage());
+// }
// System.out.println("---------------------------------");
// List<List<Point>> resultsBatchList = new ArrayList<>();
(iotdb) 43/44: add
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leirui pushed a commit to branch research/area-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 35b3b2c7a37b3ad034513f990eca770b75cf3b80
Author: Lei Rui <10...@qq.com>
AuthorDate: Sun Feb 23 18:14:05 2025 +0800
add
---
.../java/org/apache/iotdb/db/query/eBUG/DNSL1.java | 159 +++++++++++----------
1 file changed, 81 insertions(+), 78 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
index be4ffe2d6f1..28f26a3f619 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/eBUG/DNSL1.java
@@ -9,91 +9,94 @@ import java.util.Random;
import static org.apache.iotdb.db.query.eBUG.DP.dynamic_programming;
public class DNSL1 {
- // 默认使用linear interpolation连接分段首尾点来近似
- // 默认使用L1 error
- // 使用divide and conquer算法
- public static List<Point> reducePoints(List<Point> points, int m, boolean debug) {
- int n = points.size();
- int k = m - 1; // 分段数
- int intervalPts = (int) Math.pow((double) n / k, (double) 2 / 3); // divide batch点数
+ // 默认使用linear interpolation连接分段首尾点来近似
+ // 默认使用L1 error
+ // 使用divide and conquer算法
+ public static List<Point> reducePoints(List<Point> points, int m, boolean debug) {
+ int n = points.size();
+ int k = m - 1; // 分段数
+ int intervalNum = (int) Math.pow((double) n / k, (double) 2 / 3); // divide batch点数
+ int intervalPts = n / intervalNum;
- if (debug) {
- System.out.println("interval point length=" + intervalPts);
- }
+ if (debug) {
+ System.out.println("interval point length=" + intervalPts);
+ }
- // divide into intervalPts equal-length intervals
- List<Point> allSampledPoints = new ArrayList<>();
- for (int start = 0; start < n; start += intervalPts) {
- int end = Math.min(start + intervalPts, n);
- List<Point> batch = points.subList(start, end); // 左闭右开
- if (debug) {
- System.out.println("Processing batch: [" + start + "," + end + ")");
- }
- List<Point> sampledForBatch = dynamic_programming(batch, k, DP.ERRORtype.L1, false);
- allSampledPoints.addAll(sampledForBatch); // 把上一步的结果加到大的容器里
- }
+ // divide into intervalPts equal-length intervals
+ List<Point> allSampledPoints = new ArrayList<>();
+ for (int start = 0; start < n; start += intervalPts) {
+ int end = Math.min(start + intervalPts, n);
+ List<Point> batch = points.subList(start, end); // 左闭右开
+ if (debug) {
+ System.out.println("Processing batch: [" + start + "," + end + ")");
+ }
+ List<Point> sampledForBatch = dynamic_programming(batch, k, DP.ERRORtype.L1, false);
+ allSampledPoints.addAll(sampledForBatch); // 把上一步的结果加到大的容器里
+ }
- // 在大的容器上最后执行DP.dynamic_programming得到最后的m个采样点
- if (debug) {
- System.out.println("number of points from batch sampling: " + allSampledPoints.size());
- }
- List<Point> finalSampledPoints =
- dynamic_programming(allSampledPoints, k, DP.ERRORtype.L1, false);
- if (debug) {
- System.out.println("result point number: " + finalSampledPoints.size());
- }
- return finalSampledPoints;
- }
+ // 在大的容器上最后执行DP.dynamic_programming得到最后的m个采样点
+ if (debug) {
+ System.out.println("number of points from batch sampling: " + allSampledPoints.size());
+ }
- public static void main(String[] args) {
- Random rand = new Random(10);
- String input = "raw.csv";
- boolean hasHeader = true;
- int timeIdx = 0;
- int valueIdx = 1;
- int N = 1000;
- // List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
- Polyline polyline = new Polyline();
- for (int i = 0; i < N; i += 1) {
- double v = rand.nextInt(1000);
- polyline.addVertex(new Point(i, v));
+ // TODO 但是注意这里用的是linear interpolation近似，而不是DNS原文的constant
+ List<Point> finalSampledPoints =
+ dynamic_programming(allSampledPoints, k, DP.ERRORtype.L1, false);
+ if (debug) {
+ System.out.println("result point number: " + finalSampledPoints.size());
+ }
+ return finalSampledPoints;
}
- List<Point> points = polyline.getVertices();
- try (FileWriter writer = new FileWriter("raw.csv")) {
- // 写入CSV头部
- writer.append("x,y,z\n");
- // 写入每个点的数据
- for (Point point : points) {
- writer.append(point.x + "," + point.y + "," + point.z + "\n");
- }
- System.out.println(points.size() + " Data has been written");
- } catch (IOException e) {
- System.out.println("Error writing to CSV file: " + e.getMessage());
- }
+ public static void main(String[] args) {
+ Random rand = new Random(10);
+ String input = "raw.csv";
+ boolean hasHeader = true;
+ int timeIdx = 0;
+ int valueIdx = 1;
+ int N = 100_0000;
+// List<Point> points = Tool.readFromFile(input, hasHeader, timeIdx, valueIdx, N);
+ Polyline polyline = new Polyline();
+ for (int i = 0; i < N; i += 1) {
+ double v = rand.nextInt(1000);
+ polyline.addVertex(new Point(i, v));
+ }
+ List<Point> points = polyline.getVertices();
+ try (FileWriter writer = new FileWriter("raw.csv")) {
+ // 写入CSV头部
+ writer.append("x,y,z\n");
+
+ // 写入每个点的数据
+ for (Point point : points) {
+ writer.append(point.x + "," + point.y + "," + point.z + "\n");
+ }
+ System.out.println(points.size() + " Data has been written");
+ } catch (IOException e) {
+ System.out.println("Error writing to CSV file: " + e.getMessage());
+ }
- // int m = (int) Math.pow(points.size(), 0.5);
- int m = 10;
+ // int m = (int) Math.pow(points.size(), 0.5);
+ int m = 1_000;
- // long startTime = System.currentTimeMillis();
- // List<Point> sampled = dynamic_programming(points, m - 1, DP.ERROR.area, false);
- // long endTime = System.currentTimeMillis();
- // System.out.println("Time taken: " + (endTime - startTime) + "ms");
- // for (Point p : sampled) {
- // System.out.println(p);
- // }
- // System.out.println(sampled.size());
+ // long startTime = System.currentTimeMillis();
+ // List<Point> sampled = dynamic_programming(points, m - 1, DP.ERROR.area, false);
+ // long endTime = System.currentTimeMillis();
+ // System.out.println("Time taken: " + (endTime - startTime) + "ms");
+ // for (Point p : sampled) {
+ // System.out.println(p);
+ // }
+ // System.out.println(sampled.size());
- long startTime2 = System.currentTimeMillis();
- // System.out.println(points.size() * m / (Math.pow(points.size() * 1.0 / (m - 1), 2 /
- // 3.0)));
- // System.out.println(10000000 * (2 / (Math.pow(10000000 * 1.0 / (2 - 1), 2 / 3.0))));
- List<Point> sampled2 = reducePoints(points, m, true);
- long endTime2 = System.currentTimeMillis();
- System.out.println("Time taken: " + (endTime2 - startTime2) + "ms");
- // for (Point p : sampled2) {
- // System.out.println(p);
- // }
- // System.out.println(sampled2.size());
- }
+ long startTime2 = System.currentTimeMillis();
+ // System.out.println(points.size() * m / (Math.pow(points.size() * 1.0 / (m - 1), 2 /
+ // 3.0)));
+ // System.out.println(10000000 * (2 / (Math.pow(10000000 * 1.0 / (2 - 1), 2 / 3.0))));
+ List<Point> sampled2 = reducePoints(points, m, true);
+ long endTime2 = System.currentTimeMillis();
+ System.out.println("Time taken: " + (endTime2 - startTime2) + "ms");
+ // for (Point p : sampled2) {
+ // System.out.println(p);
+ // }
+ // System.out.println(sampled2.size());
+ }
}