You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2024/12/12 03:37:19 UTC
(iotdb) 01/03: Add table model auto-cast
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch table_model_auto_cast
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b8b9928296decdfc76504413ebe4db513e7c222c
Author: Tian Jiang <jt...@163.com>
AuthorDate: Thu Dec 12 10:56:51 2024 +0800
Add table model auto-cast
---
.../it/session}/IoTDBSessionRelationalIT.java | 182 ++++++++++++++++++++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 +
.../relational/sql/ast/WrappedInsertStatement.java | 20 ++-
.../plan/statement/crud/InsertBaseStatement.java | 2 +-
.../plan/statement/crud/InsertRowStatement.java | 5 +-
.../plan/statement/crud/InsertTabletStatement.java | 5 +-
.../dataregion/memtable/AbstractMemTable.java | 4 +-
.../memtable/AlignedWritableMemChunk.java | 4 +
.../memtable/AlignedWritableMemChunkGroup.java | 8 +-
.../memtable/IWritableMemChunkGroup.java | 2 +
.../dataregion/memtable/WritableMemChunkGroup.java | 5 +
.../org/apache/iotdb/db/utils/CommonUtils.java | 86 ----------
.../db/utils/datastructure/AlignedTVList.java | 26 ++-
.../resources/conf/iotdb-system.properties | 118 +++++++++++++
.../conf/iotdb-system.properties.template | 2 +-
pom.xml | 2 +-
16 files changed, 371 insertions(+), 108 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java
similarity index 86%
rename from integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java
rename to integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java
index 438c5844ca5..fdffb7c9312 100644
--- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.session.it;
+package org.apache.iotdb.relational.it.session;
import org.apache.iotdb.isession.ISession;
import org.apache.iotdb.isession.ITableSession;
@@ -30,6 +30,7 @@ import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.RowRecord;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.record.Tablet.ColumnCategory;
import org.apache.tsfile.write.schema.IMeasurementSchema;
@@ -42,11 +43,17 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+import java.nio.charset.StandardCharsets;
+import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.fail;
@@ -473,8 +480,25 @@ public class IoTDBSessionRelationalIT {
cnt++;
}
assertEquals(30, cnt);
+
+ // partial insert is disabled
+ session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"false\"");
+ int rowIndex = 0;
+ tablet.addTimestamp(rowIndex, timestamp + rowIndex);
+ tablet.addValue("id1", rowIndex, "id:" + rowIndex);
+ tablet.addValue("attr1", rowIndex, "attr:" + rowIndex);
+ tablet.addValue("m1", rowIndex, rowIndex * 1.0);
+ tablet.addValue("m2", rowIndex, rowIndex * 1.0);
+ try {
+ session.insert(tablet);
+ fail("Exception expected");
+ } catch (StatementExecutionException e) {
+ assertEquals("", e.getMessage());
+ }
+
} finally {
try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"true\"");
session.executeNonQueryStatement(
"SET CONFIGURATION \"enable_auto_create_schema\"=\"true\"");
}
@@ -1104,4 +1128,160 @@ public class IoTDBSessionRelationalIT {
assertEquals(1, cnt);
}
}
+
+ private void testOneCast(TSDataType from, TSDataType to, int testNum, boolean partialInsert)
+ throws IoTDBConnectionException, StatementExecutionException {
+ try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
+ session.executeNonQueryStatement("USE \"db1\"");
+ // create a column with type of "to"
+ session.executeNonQueryStatement(
+ "CREATE TABLE table"
+ + testNum
+ + " (id1 string id,"
+ + "m1 "
+ + to.toString()
+ + " measurement)");
+ if (partialInsert) {
+ session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='true' ON 1");
+ } else {
+ session.executeNonQueryStatement("SET CONFIGURATION enable_partial_insert='false' ON 1");
+ }
+
+ // insert a tablet with type of "from"
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("id1", TSDataType.STRING));
+ schemaList.add(new MeasurementSchema("m1", from));
+ final List<ColumnCategory> columnTypes =
+ Arrays.asList(ColumnCategory.ID, ColumnCategory.MEASUREMENT);
+ Tablet tablet =
+ new Tablet(
+ "table" + testNum,
+ IMeasurementSchema.getMeasurementNameList(schemaList),
+ IMeasurementSchema.getDataTypeList(schemaList),
+ columnTypes,
+ 15);
+
+ tablet.addTimestamp(0, 0);
+ tablet.addTimestamp(1, 1);
+ tablet.addValue(0, 0, "d1");
+ tablet.addValue(1, 0, "d1");
+ // the measurement in the first row is null
+ tablet.addValue("m1", 1, genValue(from, 1));
+ System.out.println("from=" + from + ", to=" + to);
+ if (to.isCompatible(from)) {
+ // can cast, insert and check the result
+ session.insert(tablet);
+ // time, id1, m1
+ SessionDataSet dataSet =
+ session.executeQueryStatement("select * from table" + testNum + " order by time");
+ RowRecord rec = dataSet.next();
+ System.out.println(rec);
+ assertEquals(0, rec.getFields().get(0).getLongV());
+ assertEquals("d1", rec.getFields().get(1).toString());
+ assertNull(rec.getFields().get(2).getDataType());
+ rec = dataSet.next();
+ System.out.println(rec);
+ assertEquals(1, rec.getFields().get(0).getLongV());
+ assertEquals("d1", rec.getFields().get(1).toString());
+ if (to == TSDataType.BLOB) {
+ assertEquals(genValue(to, 1), rec.getFields().get(2).getBinaryV());
+ } else if (to == TSDataType.DATE) {
+ assertEquals(genValue(to, 1), rec.getFields().get(2).getDateV());
+ } else {
+ assertEquals(genValue(to, 1).toString(), rec.getFields().get(2).toString());
+ }
+ assertFalse(dataSet.hasNext());
+ } else {
+ if (partialInsert) {
+ // cannot cast, but partial insert
+ try {
+ session.insert(tablet);
+ fail("Exception expected");
+ } catch (StatementExecutionException e) {
+ assertEquals(
+ "507: Fail to insert measurements [m1] caused by [Incompatible data type of column m1: "
+ + from
+ + "/"
+ + to
+ + "]",
+ e.getMessage());
+ }
+ // time, id1, m1
+ SessionDataSet dataSet =
+ session.executeQueryStatement("select * from table" + testNum + " order by time");
+ RowRecord rec = dataSet.next();
+ System.out.println(rec);
+ assertEquals(0, rec.getFields().get(0).getLongV());
+ assertEquals("d1", rec.getFields().get(1).toString());
+ assertNull(rec.getFields().get(2).getDataType());
+ rec = dataSet.next();
+ System.out.println(rec);
+ assertEquals(1, rec.getFields().get(0).getLongV());
+ assertEquals("d1", rec.getFields().get(1).toString());
+ assertNull(rec.getFields().get(2).getDataType());
+ assertFalse(dataSet.hasNext());
+ } else {
+ // cannot cast, expect an exception
+ try {
+ session.insert(tablet);
+ fail("Exception expected");
+ } catch (StatementExecutionException e) {
+ assertEquals(
+ "614: Incompatible data type of column m1: " + from + "/" + to, e.getMessage());
+ }
+ }
+ }
+
+ session.executeNonQueryStatement("DROP TABLE table" + testNum);
+ }
+ }
+
+ @SuppressWarnings("SameParameterValue")
+ private Object genValue(TSDataType dataType, int i) {
+ switch (dataType) {
+ case INT32:
+ return i;
+ case DATE:
+ return LocalDate.ofEpochDay(i);
+ case TIMESTAMP:
+ case INT64:
+ return (long) i;
+ case BOOLEAN:
+ return i % 2 == 0;
+ case FLOAT:
+ return i * 1.0f;
+ case DOUBLE:
+ return i * 1.0;
+ case STRING:
+ case TEXT:
+ case BLOB:
+ return new Binary(Integer.toString(i), StandardCharsets.UTF_8);
+ case UNKNOWN:
+ case VECTOR:
+ default:
+ throw new IllegalArgumentException("Unsupported data type: " + dataType);
+ }
+ }
+
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void insertRelationalTabletWithAutoCastTest()
+ throws IoTDBConnectionException, StatementExecutionException {
+ int testNum = 14;
+ Set<TSDataType> dataTypes = new HashSet<>();
+ Collections.addAll(dataTypes, TSDataType.values());
+ dataTypes.remove(TSDataType.VECTOR);
+ dataTypes.remove(TSDataType.UNKNOWN);
+
+ for (TSDataType from : dataTypes) {
+ for (TSDataType to : dataTypes) {
+ testOneCast(from, to, testNum, false);
+ testOneCast(from, to, testNum, true);
+ }
+ }
+
+ try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ session.executeNonQueryStatement("SET CONFIGURATION \"enable_partial_insert\"=\"true\" ON 1");
+ }
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index b652acf7329..e112e5efb96 100755
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -2870,6 +2870,14 @@ public class IoTDBDescriptor {
} else {
BinaryAllocator.getInstance().close(true);
}
+
+ conf.setEnablePartialInsert(
+ Boolean.parseBoolean(
+ Optional.ofNullable(
+ properties.getProperty(
+ "enable_partial_insert", String.valueOf(conf.isEnablePartialInsert())))
+ .map(String::trim)
+ .orElse(String.valueOf(conf.isEnablePartialInsert()))));
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java
index 06d0003a9f5..bd53aaf0131 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/WrappedInsertStatement.java
@@ -20,6 +20,9 @@
package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException;
+import org.apache.iotdb.db.exception.metadata.PathNotExistException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
@@ -168,7 +171,8 @@ public abstract class WrappedInsertStatement extends WrappedStatement
new SemanticException(
"Column " + incoming.getName() + " does not exists or fails to be " + "created",
TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode());
- if (incoming.getColumnCategory() != TsTableColumnCategory.MEASUREMENT) {
+ if (incoming.getColumnCategory() != TsTableColumnCategory.MEASUREMENT
+ || !IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) {
// non-measurement columns cannot be partially inserted
throw semanticException;
} else {
@@ -182,19 +186,22 @@ public abstract class WrappedInsertStatement extends WrappedStatement
// sql insertion does not provide type
// the type is inferred and can be inconsistent with the existing one
innerTreeStatement.setDataType(InternalTypeManager.getTSDataType(real.getType()), i);
- } else if (!incoming.getType().equals(real.getType())) {
+ } else if (!InternalTypeManager.getTSDataType(real.getType())
+ .isCompatible(InternalTypeManager.getTSDataType(incoming.getType()))) {
SemanticException semanticException =
new SemanticException(
String.format(
- "Inconsistent data type of column %s: %s/%s",
+ "Incompatible data type of column %s: %s/%s",
incoming.getName(), incoming.getType(), real.getType()),
TSStatusCode.DATA_TYPE_MISMATCH.getStatusCode());
- if (incoming.getColumnCategory() != TsTableColumnCategory.MEASUREMENT) {
+ if (incoming.getColumnCategory() != TsTableColumnCategory.MEASUREMENT
+ || !IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) {
// non-measurement columns cannot be partially inserted
throw semanticException;
} else {
// partial insertion
innerTreeStatement.markFailedMeasurement(i, semanticException);
+ return;
}
}
if (incoming.getColumnCategory() == null) {
@@ -215,6 +222,11 @@ public abstract class WrappedInsertStatement extends WrappedStatement
getDefaultEncoding(tsDataType),
TSFileDescriptor.getInstance().getConfig().getCompressor());
innerTreeStatement.setMeasurementSchema(measurementSchema, i);
+ try {
+ innerTreeStatement.selfCheckDataTypes(i);
+ } catch (DataTypeMismatchException | PathNotExistException e) {
+ throw new SemanticException(e);
+ }
}
public void validateDeviceSchema(Metadata metadata, MPPQueryContext context) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java
index 1856d8968ae..7df56bbce88 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertBaseStatement.java
@@ -192,7 +192,7 @@ public abstract class InsertBaseStatement extends Statement {
public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryProcessException {}
/** Check whether data types are matched with measurement schemas */
- protected void selfCheckDataTypes(int index)
+ public void selfCheckDataTypes(int index)
throws DataTypeMismatchException, PathNotExistException {
if (IoTDBDescriptor.getInstance().getConfig().isEnablePartialInsert()) {
// if enable partial insert, mark failed measurements with exception
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java
index 8c832a4e6f3..00be61c6b80 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertRowStatement.java
@@ -190,9 +190,8 @@ public class InsertRowStatement extends InsertBaseStatement implements ISchemaVa
@Override
protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) {
- if (CommonUtils.checkCanCastType(dataTypes[columnIndex], dataType)) {
- values[columnIndex] =
- CommonUtils.castValue(dataTypes[columnIndex], dataType, values[columnIndex]);
+ if (dataType.isCompatible(dataTypes[columnIndex])) {
+ values[columnIndex] = dataType.castFromSingleValue(dataType, values[columnIndex]);
dataTypes[columnIndex] = dataType;
return true;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
index f61db241355..e6f6f5f1a20 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java
@@ -191,9 +191,8 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem
@Override
protected boolean checkAndCastDataType(int columnIndex, TSDataType dataType) {
- if (CommonUtils.checkCanCastType(dataTypes[columnIndex], dataType)) {
- columns[columnIndex] =
- CommonUtils.castArray(dataTypes[columnIndex], dataType, columns[columnIndex]);
+ if (dataType.isCompatible(dataTypes[columnIndex])) {
+ columns[columnIndex] = dataType.castFromArray(dataTypes[columnIndex], columns[columnIndex]);
dataTypes[columnIndex] = dataType;
return true;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index fff93680c9c..b46f1a7612a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -796,8 +796,8 @@ public abstract class AbstractMemTable implements IMemTable {
} else {
pointDeleted += pair.right.delete(modEntry);
}
- if (pair.right.getMemChunkMap().isEmpty()) {
- memTableMap.remove(pair.left);
+ if (pair.right.isEmpty()) {
+ memTableMap.remove(pair.left).release();
}
}
return pointDeleted;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index 74c4682de6a..81105bbc686 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -554,4 +554,8 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
public List<IMeasurementSchema> getSchemaList() {
return schemaList;
}
+
+ public boolean isAllDeleted() {
+ return list.isAllDeleted();
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
index 4e079b61787..bcba1b41ad2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java
@@ -98,6 +98,11 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup {
return Collections.singletonMap("", memChunk);
}
+ @Override
+ public boolean isEmpty() {
+ return memChunk.isEmpty() || memChunk.isAllDeleted();
+ }
+
@Override
public long delete(ModEntry modEntry) {
int deletedPointsNumber = 0;
@@ -124,7 +129,8 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup {
}
public long deleteTime(ModEntry modEntry) {
- return memChunk.deleteTime(modEntry.getStartTime(), modEntry.getEndTime());
+ int pointNumDeleted = memChunk.deleteTime(modEntry.getStartTime(), modEntry.getEndTime());
+ return pointNumDeleted;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
index 353da05bf28..a2d23dc9db0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java
@@ -51,6 +51,8 @@ public interface IWritableMemChunkGroup extends WALEntryValue {
Map<String, IWritableMemChunk> getMemChunkMap();
+ boolean isEmpty();
+
long delete(ModEntry modEntry);
long deleteTime(ModEntry modEntry);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
index b3fade015a3..ac5d588eb4f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java
@@ -116,6 +116,11 @@ public class WritableMemChunkGroup implements IWritableMemChunkGroup {
return memChunkMap;
}
+ @Override
+ public boolean isEmpty() {
+ return memChunkMap.isEmpty();
+ }
+
@Override
public long delete(ModEntry modEntry) {
Iterator<Entry<String, IWritableMemChunk>> iter = memChunkMap.entrySet().iterator();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
index 3b3e6311164..b54838b4735 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
@@ -57,9 +57,7 @@ import org.apache.tsfile.write.UnSupportedDataTypeException;
import java.time.LocalDate;
import java.time.ZoneId;
-import java.util.Arrays;
import java.util.List;
-import java.util.Objects;
import java.util.concurrent.TimeUnit;
@SuppressWarnings("java:S106") // for console outputs
@@ -190,90 +188,6 @@ public class CommonUtils {
}
}
- public static boolean checkCanCastType(TSDataType src, TSDataType dest) {
- if (Objects.isNull(src)) {
- return true;
- }
- switch (src) {
- case INT32:
- if (dest == TSDataType.INT64 || dest == TSDataType.FLOAT || dest == TSDataType.DOUBLE) {
- return true;
- }
- case INT64:
- if (dest == TSDataType.DOUBLE) {
- return true;
- }
- case FLOAT:
- if (dest == TSDataType.DOUBLE) {
- return true;
- }
- }
- return false;
- }
-
- public static Object castValue(TSDataType srcDataType, TSDataType destDataType, Object value) {
- if (Objects.isNull(value)) {
- return null;
- }
- switch (srcDataType) {
- case INT32:
- if (destDataType == TSDataType.INT64) {
- value = (long) ((int) value);
- } else if (destDataType == TSDataType.FLOAT) {
- value = (float) ((int) value);
- } else if (destDataType == TSDataType.DOUBLE) {
- value = (double) ((int) value);
- }
- break;
- case INT64:
- if (destDataType == TSDataType.DOUBLE) {
- value = (double) ((long) value);
- }
- break;
- case FLOAT:
- if (destDataType == TSDataType.DOUBLE) {
- value = (double) ((float) value);
- }
- break;
- }
- return value;
- }
-
- public static Object castArray(TSDataType srcDataType, TSDataType destDataType, Object value) {
- switch (srcDataType) {
- case INT32:
- if (destDataType == TSDataType.INT64) {
- value = Arrays.stream((int[]) value).mapToLong(Long::valueOf).toArray();
- } else if (destDataType == TSDataType.FLOAT) {
- int[] tmp = (int[]) value;
- float[] result = new float[tmp.length];
- for (int i = 0; i < tmp.length; i++) {
- result[i] = (float) tmp[i];
- }
- value = result;
- } else if (destDataType == TSDataType.DOUBLE) {
- value = Arrays.stream((int[]) value).mapToDouble(Double::valueOf).toArray();
- }
- break;
- case INT64:
- if (destDataType == TSDataType.DOUBLE) {
- value = Arrays.stream((long[]) value).mapToDouble(Double::valueOf).toArray();
- }
- break;
- case FLOAT:
- if (destDataType == TSDataType.DOUBLE) {
- float[] tmp = (float[]) value;
- double[] result = new double[tmp.length];
- for (int i = 0; i < tmp.length; i++) {
- result[i] = tmp[i];
- }
- value = result;
- }
- break;
- }
- return value;
- }
-
private static boolean parseBoolean(String value) throws QueryProcessException {
value = value.toLowerCase();
if (SqlConstant.BOOLEAN_FALSE_NUM.equals(value) || SqlConstant.BOOLEAN_FALSE.equals(value)) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index f0a645bf3ec..5d5a6e2b41c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -86,6 +86,8 @@ public abstract class AlignedTVList extends TVList {
// constructed after deletion
BitMap timeColDeletedMap;
+ protected int timeDeletedCnt = 0;
+
AlignedTVList(List<TSDataType> types) {
super();
indices = new ArrayList<>(types.size());
@@ -141,6 +143,7 @@ public abstract class AlignedTVList extends TVList {
// for table model, we won't discard any row even if all value columns are null
alignedTvList.allValueColDeletedMap = ignoreAllNullRows ? getAllValueColDeletedMap() : null;
alignedTvList.timeColDeletedMap = this.timeColDeletedMap;
+ alignedTvList.timeDeletedCnt = this.timeDeletedCnt;
return alignedTvList;
}
@@ -149,6 +152,7 @@ public abstract class AlignedTVList extends TVList {
public AlignedTVList clone() {
AlignedTVList cloneList = AlignedTVList.newAlignedList(dataTypes);
cloneAs(cloneList);
+ cloneList.timeDeletedCnt = this.timeDeletedCnt;
System.arraycopy(
memoryBinaryChunkSize, 0, cloneList.memoryBinaryChunkSize, 0, dataTypes.size());
for (int[] indicesArray : indices) {
@@ -538,19 +542,18 @@ public abstract class AlignedTVList extends TVList {
public int deleteTime(long lowerBound, long upperBound) {
delete(lowerBound, upperBound);
- int deletedNumber = 0;
+ int prevDeletedCnt = this.timeDeletedCnt;
for (int i = 0; i < rowCount; i++) {
long time = getTime(i);
if (time >= lowerBound && time <= upperBound) {
markRowNull(i);
- deletedNumber++;
}
}
boolean needUpdateMaxTime = lowerBound <= maxTime && maxTime <= upperBound;
if (needUpdateMaxTime) {
updateMaxTime();
}
- return deletedNumber;
+ return timeDeletedCnt - prevDeletedCnt;
}
private void updateMaxTime() {
@@ -707,7 +710,10 @@ public abstract class AlignedTVList extends TVList {
}
}
- private void markRowNull(int i) {
+ /**
+ * @return true if the row is marked, false if it is already marked
+ */
+ private boolean markRowNull(int i) {
if (timeColDeletedMap == null) {
timeColDeletedMap = new BitMap(rowCount);
} else if (timeColDeletedMap.getSize() < rowCount) {
@@ -717,7 +723,13 @@ public abstract class AlignedTVList extends TVList {
timeColDeletedMap = new BitMap(rowCount, newBytes);
}
// use value index so that sorts will not change the nullability
- timeColDeletedMap.mark(getValueIndex(i));
+ if (timeColDeletedMap.isMarked(i)) {
+ return false;
+ } else {
+ timeColDeletedMap.mark(getValueIndex(i));
+ timeDeletedCnt++;
+ return true;
+ }
}
/**
@@ -1478,4 +1490,8 @@ public abstract class AlignedTVList extends TVList {
public List<List<BitMap>> getBitMaps() {
return bitMaps;
}
+
+ public boolean isAllDeleted() {
+ return timeDeletedCnt == rowCount;
+ }
}
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties
index c51f948ad25..f99015dbe25 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties
@@ -70,3 +70,121 @@ cn_metric_prometheus_reporter_port=9091
# dn_metric_reporter_list=
dn_metric_prometheus_reporter_port=9092
+#Thu Dec 12 10:40:29 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:40:33 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:40:33 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:40:37 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:40:38 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:42:19 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:42:19 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:42:19 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:42:19 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:42:19 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:42:19 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:42:19 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:42:19 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:42:19 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:42:20 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:29 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:29 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:29 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:29 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:29 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:29 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:29 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:30 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:31 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:31 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:31 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:31 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:43:31 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:43:31 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:45:59 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:46:00 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:46:00 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:46:00 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:46:02 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:46:02 HKT 2024
+#enable_partial_insert=false
+#Thu Dec 12 10:46:02 HKT 2024
+#enable_partial_insert=true
+#Thu Dec 12 10:46:02 HKT 2024
+enable_partial_insert=false
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 2efae33aae9..fa0ab8407c5 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1137,7 +1137,7 @@ flush_thread_count=0
# In one insert (one device, one timestamp, multiple measurements),
# if enable partial insert, one measurement failure will not impact other measurements
-# effectiveMode: restart
+# effectiveMode: hot_reload
# Datatype: boolean
enable_partial_insert=true
diff --git a/pom.xml b/pom.xml
index ad0eaea6bcf..112468fa607 100644
--- a/pom.xml
+++ b/pom.xml
@@ -166,7 +166,7 @@
<thrift.version>0.14.1</thrift.version>
<xz.version>1.9</xz.version>
<zstd-jni.version>1.5.6-3</zstd-jni.version>
- <tsfile.version>1.2.0-241129-SNAPSHOT</tsfile.version>
+ <tsfile.version>1.2.0-241211-SNAPSHOT</tsfile.version>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim