You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/04/09 07:01:18 UTC
(iotdb) branch load--1-time-partition created (now ce6c12200ae)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a change to branch load--1-time-partition
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at ce6c12200ae Load: support load tsfile with data in partition whose id = -1
This branch includes the following new commits:
new ce6c12200ae Load: support load tsfile with data in partition whose id = -1
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: Load: support load tsfile with data in partition whose id = -1
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch load--1-time-partition
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ce6c12200aebbb8900882ac45842fda8ece761cc
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Tue Apr 9 15:01:09 2024 +0800
Load: support load tsfile with data in partition whose id = -1
---
.../db/exception/PartitionViolationException.java | 6 +-
.../tsfile/timeindex/DeviceTimeIndex.java | 64 +++++++++++++---------
.../dataregion/tsfile/timeindex/FileTimeIndex.java | 22 +++-----
.../dataregion/tsfile/timeindex/ITimeIndex.java | 2 -
4 files changed, 51 insertions(+), 43 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/PartitionViolationException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/PartitionViolationException.java
index 75d1f954e02..8b4c8dfb8ab 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/PartitionViolationException.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/PartitionViolationException.java
@@ -22,6 +22,10 @@ package org.apache.iotdb.db.exception;
public class PartitionViolationException extends LoadFileException {
public PartitionViolationException(String file) {
- super(String.format("The data of file %s cross partitions", file));
+ super(String.format("The data of file %s crosses partitions", file));
+ }
+
+ public PartitionViolationException() {
+ super("The data of file crosses partitions");
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java
index 024d4154f1e..777d092fee3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java
@@ -261,40 +261,52 @@ public class DeviceTimeIndex implements ITimeIndex {
}
}
- /** @return the time partition id, if spans multi time partitions, return -1. */
- private long getTimePartitionWithCheck() {
- long partitionId = SPANS_MULTI_TIME_PARTITIONS_FLAG_ID;
- for (int index : deviceToIndex.values()) {
- long p = TimePartitionUtils.getTimePartitionId(startTimes[index]);
- if (partitionId == SPANS_MULTI_TIME_PARTITIONS_FLAG_ID) {
- partitionId = p;
- } else {
- if (partitionId != p) {
- return SPANS_MULTI_TIME_PARTITIONS_FLAG_ID;
- }
- }
-
- p = TimePartitionUtils.getTimePartitionId(endTimes[index]);
- if (partitionId != p) {
- return SPANS_MULTI_TIME_PARTITIONS_FLAG_ID;
- }
- }
- return partitionId;
- }
-
@Override
public long getTimePartitionWithCheck(String tsFilePath) throws PartitionViolationException {
- long partitionId = getTimePartitionWithCheck();
- if (partitionId == SPANS_MULTI_TIME_PARTITIONS_FLAG_ID) {
+ try {
+ return getTimePartitionWithCheck();
+ } catch (PartitionViolationException e) {
throw new PartitionViolationException(tsFilePath);
}
- return partitionId;
}
@Override
public boolean isSpanMultiTimePartitions() {
- long partitionId = getTimePartitionWithCheck();
- return partitionId == SPANS_MULTI_TIME_PARTITIONS_FLAG_ID;
+ try {
+ getTimePartitionWithCheck();
+ return false;
+ } catch (PartitionViolationException e) {
+ return true;
+ }
+ }
+
+ private long getTimePartitionWithCheck() throws PartitionViolationException {
+ Long partitionId = null;
+
+ for (final int index : deviceToIndex.values()) {
+ final long startTimePartitionId = TimePartitionUtils.getTimePartitionId(startTimes[index]);
+ final long endTimePartitionId = TimePartitionUtils.getTimePartitionId(endTimes[index]);
+
+ if (startTimePartitionId != endTimePartitionId) {
+ throw new PartitionViolationException();
+ }
+
+ if (partitionId == null) {
+ partitionId = startTimePartitionId;
+ continue;
+ }
+
+ if (partitionId != startTimePartitionId) {
+ throw new PartitionViolationException();
+ }
+ }
+
+ // Just in case
+ if (partitionId == null) {
+ throw new PartitionViolationException();
+ }
+
+ return partitionId;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java
index d397dd384c2..65aa8153a2a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java
@@ -142,28 +142,22 @@ public class FileTimeIndex implements ITimeIndex {
}
}
- private long getTimePartitionWithCheck() {
- long startPartitionId = TimePartitionUtils.getTimePartitionId(startTime);
- long endPartitionId = TimePartitionUtils.getTimePartitionId(endTime);
+ @Override
+ public long getTimePartitionWithCheck(String tsFilePath) throws PartitionViolationException {
+ final long startPartitionId = TimePartitionUtils.getTimePartitionId(startTime);
+ final long endPartitionId = TimePartitionUtils.getTimePartitionId(endTime);
+
if (startPartitionId == endPartitionId) {
return startPartitionId;
}
- return SPANS_MULTI_TIME_PARTITIONS_FLAG_ID;
- }
- @Override
- public long getTimePartitionWithCheck(String tsFilePath) throws PartitionViolationException {
- long partitionId = getTimePartitionWithCheck();
- if (partitionId == SPANS_MULTI_TIME_PARTITIONS_FLAG_ID) {
- throw new PartitionViolationException(tsFilePath);
- }
- return partitionId;
+ throw new PartitionViolationException(tsFilePath);
}
@Override
public boolean isSpanMultiTimePartitions() {
- long partitionId = getTimePartitionWithCheck();
- return partitionId == SPANS_MULTI_TIME_PARTITIONS_FLAG_ID;
+ return TimePartitionUtils.getTimePartitionId(startTime)
+ != TimePartitionUtils.getTimePartitionId(endTime);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java
index 84169b1087b..0edbe81745d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java
@@ -37,8 +37,6 @@ public interface ITimeIndex {
byte DEVICE_TIME_INDEX_TYPE = 1;
byte FILE_TIME_INDEX_TYPE = 2;
- int SPANS_MULTI_TIME_PARTITIONS_FLAG_ID = -1;
-
/**
* serialize to outputStream
*