You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2025/12/18 03:55:23 UTC
(iotdb) 01/33: Add Object Type
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch force_ci/object_type
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c82230cb440dd2c6deefb82f010a658307a22a85
Author: Jackie Tien <ja...@gmail.com>
AuthorDate: Fri Jul 11 14:11:18 2025 +0800
Add Object Type
---
.../main/java/org/apache/iotdb/ObjectExample.java | 148 +++++++++
.../java/org/apache/iotdb/ObjectReadExample.java | 114 +++++++
.../apache/iotdb/TableModelSessionPoolExample.java | 22 +-
.../java/org/apache/iotdb/pipe/api/type/Type.java | 5 +-
.../java/org/apache/iotdb/udf/api/type/Type.java | 10 +-
.../java/org/apache/iotdb/cli/AbstractCli.java | 1 +
.../org/apache/iotdb/isession/SessionDataSet.java | 1 +
.../iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java | 1 +
.../org/apache/iotdb/rpc/IoTDBJDBCDataSet.java | 6 +
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 6 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 3 +
.../java/org/apache/iotdb/session/Session.java | 1 +
.../payload/SubscriptionSessionDataSet.java | 1 +
.../apache/iotdb/session/util/SessionUtils.java | 3 +
iotdb-core/datanode/pom.xml | 5 +
.../apache/iotdb/db/conf/DataNodeMemoryConfig.java | 5 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 9 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 12 +
.../dataregion/DataExecutionVisitor.java | 13 +
.../processor/aggregate/AggregateProcessor.java | 3 +
.../resource/memory/InsertNodeMemoryEstimator.java | 1 +
.../sink/protocol/opcda/OpcDaServerHandle.java | 2 +
.../sink/util/sorter/PipeInsertEventSorter.java | 1 +
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 2 +-
.../execution/aggregation/AccumulatorFactory.java | 1 +
.../execution/aggregation/AvgAccumulator.java | 1 +
.../execution/aggregation/ExtremeAccumulator.java | 6 +
.../aggregation/FirstValueAccumulator.java | 6 +
.../aggregation/LastValueAccumulator.java | 6 +
.../aggregation/MaxMinByBaseAccumulator.java | 6 +
.../execution/aggregation/MaxValueAccumulator.java | 6 +
.../execution/aggregation/MinValueAccumulator.java | 6 +
.../execution/aggregation/SumAccumulator.java | 1 +
.../execution/aggregation/VarianceAccumulator.java | 1 +
.../execution/executor/RegionWriteExecutor.java | 14 +
.../execution/memory/LocalMemoryManager.java | 1 -
.../execution/operator/AggregationUtil.java | 1 +
.../execution/operator/process/TopKOperator.java | 2 +
.../operator/process/TransformOperator.java | 1 +
.../process/join/merge/MergeSortComparator.java | 1 +
.../process/window/function/value/LagFunction.java | 1 +
.../window/function/value/LeadFunction.java | 1 +
.../process/window/utils/RowComparator.java | 3 +
.../execution/operator/source/SeriesScanUtil.java | 1 +
.../relational/LastQueryAggTableScanOperator.java | 1 +
.../relational/aggregation/AccumulatorFactory.java | 2 +
.../ApproxCountDistinctAccumulator.java | 1 +
.../relational/aggregation/AvgAccumulator.java | 2 +
.../relational/aggregation/ExtremeAccumulator.java | 5 +
.../relational/aggregation/FirstAccumulator.java | 4 +
.../relational/aggregation/FirstByAccumulator.java | 4 +
.../relational/aggregation/LastAccumulator.java | 4 +
.../relational/aggregation/LastByAccumulator.java | 4 +
.../relational/aggregation/SumAccumulator.java | 2 +
.../aggregation/TableMaxMinByBaseAccumulator.java | 5 +
.../aggregation/TableVarianceAccumulator.java | 2 +
.../source/relational/aggregation/Utils.java | 2 +
.../GroupedApproxCountDistinctAccumulator.java | 1 +
.../aggregation/grouped/GroupedAvgAccumulator.java | 1 +
.../grouped/GroupedExtremeAccumulator.java | 8 +
.../grouped/GroupedFirstAccumulator.java | 8 +
.../grouped/GroupedFirstByAccumulator.java | 9 +
.../grouped/GroupedLastAccumulator.java | 8 +
.../grouped/GroupedLastByAccumulator.java | 9 +
.../aggregation/grouped/GroupedMaxAccumulator.java | 8 +
.../grouped/GroupedMaxMinByBaseAccumulator.java | 15 +
.../aggregation/grouped/GroupedMinAccumulator.java | 8 +
.../grouped/GroupedModeAccumulator.java | 4 +
.../aggregation/grouped/GroupedSumAccumulator.java | 1 +
.../grouped/GroupedVarianceAccumulator.java | 1 +
.../operator/window/WindowManagerFactory.java | 2 +
.../relational/ColumnTransformerBuilder.java | 34 +-
.../plan/planner/TableOperatorGenerator.java | 23 +-
.../plan/planner/plan/node/PlanNodeType.java | 8 +
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../planner/plan/node/write/InsertTabletNode.java | 29 +-
.../plan/planner/plan/node/write/ObjectNode.java | 346 +++++++++++++++++++++
.../node/write/RelationalInsertTabletNode.java | 74 +++++
.../relational/metadata/TableMetadataImpl.java | 19 ++
.../plan/relational/sql/ast/DeleteDevice.java | 3 +-
.../plan/relational/type/InternalTypeManager.java | 6 +
.../plan/relational/utils/TypeUtil.java | 13 +-
.../unary/scalar/ReadObjectColumnTransformer.java | 129 ++++++++
.../dag/input/ConstantInputReader.java | 1 +
.../unary/ArithmeticNegationTransformer.java | 1 +
.../dag/transformer/unary/InTransformer.java | 2 +
.../unary/scalar/CastFunctionTransformer.java | 7 +
.../unary/scalar/DiffFunctionTransformer.java | 1 +
.../unary/scalar/RoundFunctionTransformer.java | 1 +
.../transformation/dag/util/TransformUtils.java | 2 +
.../transformation/dag/util/TypeUtils.java | 1 +
.../datastructure/row/SerializableRowList.java | 2 +
.../datastructure/tv/SerializableTVList.java | 1 +
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 6 +-
.../iotdb/db/storageengine/StorageEngine.java | 38 +++
.../db/storageengine/dataregion/DataRegion.java | 63 +++-
.../performer/impl/FastCompactionPerformer.java | 3 +-
.../execute/task/CrossSpaceCompactionTask.java | 6 +
.../execute/task/InnerSpaceCompactionTask.java | 6 +
.../execute/task/SettleCompactionTask.java | 7 +-
.../compaction/execute/utils/CompactionUtils.java | 168 ++++++++++
.../execute/utils/MultiTsFileDeviceIterator.java | 11 +-
.../fast/FastAlignedSeriesCompactionExecutor.java | 25 +-
.../FastNonAlignedSeriesCompactionExecutor.java | 6 +-
.../fast/element/ChunkMetadataElement.java | 10 +-
.../ReadChunkAlignedSeriesCompactionExecutor.java | 1 +
.../readchunk/SingleSeriesCompactionExecutor.java | 1 +
.../utils/writer/AbstractCompactionWriter.java | 1 +
.../memtable/AlignedReadOnlyMemChunk.java | 2 +
.../memtable/AlignedWritableMemChunk.java | 1 +
.../dataregion/memtable/WritableMemChunk.java | 1 +
.../read/reader/chunk/MemAlignedPageReader.java | 1 +
.../read/reader/chunk/MemPageReader.java | 2 +
.../tsfile/generator/TsFileNameGenerator.java | 15 +
.../dataregion/wal/buffer/WALBuffer.java | 3 +
.../dataregion/wal/buffer/WALEntry.java | 6 +
.../dataregion/wal/buffer/WALEntryType.java | 4 +-
.../dataregion/wal/buffer/WALInfoEntry.java | 4 +
.../dataregion/wal/node/IWALNode.java | 3 +
.../dataregion/wal/node/WALFakeNode.java | 6 +
.../storageengine/dataregion/wal/node/WALNode.java | 52 +++-
.../file/UnsealedTsFileRecoverPerformer.java | 2 +
.../splitter/BatchedAlignedValueChunkData.java | 2 +
.../db/storageengine/rescon/disk/TierManager.java | 51 +++
.../rescon/memory/PrimitiveArrayManager.java | 1 +
.../iotdb/db/tools/TsFileSplitByPartitionTool.java | 1 +
.../org/apache/iotdb/db/utils/CommonUtils.java | 1 +
.../iotdb/db/utils/EncodingInferenceUtils.java | 1 +
.../java/org/apache/iotdb/db/utils/MemUtils.java | 4 +-
.../org/apache/iotdb/db/utils/ObjectTypeUtils.java | 85 +++++
.../org/apache/iotdb/db/utils/ObjectWriter.java | 82 +++++
.../apache/iotdb/db/utils/QueryDataSetUtils.java | 4 +
.../apache/iotdb/db/utils/TimeValuePairUtils.java | 2 +
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 1 +
.../db/utils/datastructure/AlignedTVList.java | 12 +
.../MergeSortMultiAlignedTVListIterator.java | 1 +
.../MergeSortMultiTVListIterator.java | 1 +
.../datastructure/MultiAlignedTVListIterator.java | 1 +
.../utils/datastructure/MultiTVListIterator.java | 1 +
.../iotdb/db/utils/datastructure/TVList.java | 5 +
.../db/utils/windowing/window/WindowImpl.java | 1 +
.../compaction/CompactionDeleteObjectFileTest.java | 122 ++++++++
.../datastructure/PrimitiveArrayManagerTest.java | 1 +
.../conf/iotdb-system.properties.template | 5 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 +
.../commons/exception/ObjectFileNotExist.java | 44 +--
.../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 3 +
.../iotdb/commons/udf/builtin/UDTFBottomK.java | 1 +
.../commons/udf/builtin/UDTFCommonDerivative.java | 1 +
.../udf/builtin/UDTFCommonValueDifference.java | 1 +
.../iotdb/commons/udf/builtin/UDTFConst.java | 5 +
.../udf/builtin/UDTFContinuouslySatisfy.java | 2 +
.../udf/builtin/UDTFEqualSizeBucketAggSample.java | 1 +
.../udf/builtin/UDTFEqualSizeBucketM4Sample.java | 1 +
.../builtin/UDTFEqualSizeBucketOutlierSample.java | 1 +
.../builtin/UDTFEqualSizeBucketRandomSample.java | 1 +
.../iotdb/commons/udf/builtin/UDTFInRange.java | 3 +
.../apache/iotdb/commons/udf/builtin/UDTFJexl.java | 3 +
.../apache/iotdb/commons/udf/builtin/UDTFM4.java | 1 +
.../apache/iotdb/commons/udf/builtin/UDTFMath.java | 3 +
.../udf/builtin/UDTFNonNegativeDerivative.java | 1 +
.../builtin/UDTFNonNegativeValueDifference.java | 1 +
.../iotdb/commons/udf/builtin/UDTFOnOff.java | 1 +
.../iotdb/commons/udf/builtin/UDTFSelectK.java | 2 +
.../apache/iotdb/commons/udf/builtin/UDTFTopK.java | 1 +
.../iotdb/commons/udf/builtin/UDTFValueTrend.java | 1 +
.../relational/TableBuiltinScalarFunction.java | 1 +
.../commons/udf/utils/UDFDataTypeTransformer.java | 7 +
.../apache/iotdb/commons/utils/SerializeUtils.java | 6 +
.../thrift-datanode/src/main/thrift/client.thrift | 2 +-
170 files changed, 2125 insertions(+), 79 deletions(-)
diff --git a/example/session/src/main/java/org/apache/iotdb/ObjectExample.java b/example/session/src/main/java/org/apache/iotdb/ObjectExample.java
new file mode 100644
index 00000000000..3339d3aae5a
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/ObjectExample.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.isession.ITableSession;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.TableSessionBuilder;
+
+import org.apache.tsfile.enums.ColumnCategory;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.write.record.Tablet;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class ObjectExample {
+ private static final String LOCAL_URL = "127.0.0.1:6667";
+
+ public static void main(String[] args) {
+
+ // don't specify database in constructor
+ try (ITableSession session =
+ new TableSessionBuilder()
+ .nodeUrls(Collections.singletonList(LOCAL_URL))
+ .username("root")
+ .password("root")
+ .build()) {
+ session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS test1");
+ session.executeNonQueryStatement("use test1");
+
+ // insert table data by tablet
+ List<String> columnNameList =
+ Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file");
+ List<TSDataType> dataTypeList =
+ Arrays.asList(
+ TSDataType.STRING,
+ TSDataType.STRING,
+ TSDataType.STRING,
+ TSDataType.FLOAT,
+ TSDataType.OBJECT);
+ List<ColumnCategory> columnTypeList =
+ new ArrayList<>(
+ Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.TAG,
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD));
+ Tablet tablet = new Tablet("tsfile_table", columnNameList, dataTypeList, columnTypeList, 1);
+ int rowIndex = tablet.getRowSize();
+ tablet.addTimestamp(rowIndex, 1);
+ tablet.addValue(rowIndex, 0, "1");
+ tablet.addValue(rowIndex, 1, "5");
+ tablet.addValue(rowIndex, 2, "3");
+ tablet.addValue(rowIndex, 3, 37.6F);
+ tablet.addValue(
+ rowIndex,
+ 4,
+ true,
+ 0,
+ Files.readAllBytes(
+ Paths.get(
+ "/Users/jackietien/Downloads/2_1746622362350_fa24aa15233f4e76bcda789a5771f43f")));
+ session.insert(tablet);
+ tablet.reset();
+
+ tablet = new Tablet("tsfile_table", columnNameList, dataTypeList, columnTypeList, 1);
+ rowIndex = tablet.getRowSize();
+ tablet.addTimestamp(rowIndex, 2);
+ tablet.addValue(rowIndex, 0, "1");
+ tablet.addValue(rowIndex, 1, "5");
+ tablet.addValue(rowIndex, 2, "3");
+ tablet.addValue(rowIndex, 3, 37.6F);
+ tablet.addValue(
+ rowIndex,
+ 4,
+ true,
+ 0,
+ Files.readAllBytes(
+ Paths.get(
+ "/Users/jackietien/Downloads/2_1746622367063_8fb5ac8e21724140874195b60b878664")));
+ session.insert(tablet);
+ tablet.reset();
+
+ tablet = new Tablet("tiff_table", columnNameList, dataTypeList, columnTypeList, 1);
+ rowIndex = tablet.getRowSize();
+ tablet.addTimestamp(rowIndex, 1);
+ tablet.addValue(rowIndex, 0, "1");
+ tablet.addValue(rowIndex, 1, "5");
+ tablet.addValue(rowIndex, 2, "3");
+ tablet.addValue(rowIndex, 3, 37.6F);
+ tablet.addValue(
+ rowIndex,
+ 4,
+ true,
+ 0,
+ Files.readAllBytes(Paths.get("/Users/jackietien/Downloads/1751891240130.tiff")));
+ session.insert(tablet);
+ tablet.reset();
+
+ tablet = new Tablet("tiff_table", columnNameList, dataTypeList, columnTypeList, 1);
+ rowIndex = tablet.getRowSize();
+ tablet.addTimestamp(rowIndex, 2);
+ tablet.addValue(rowIndex, 0, "1");
+ tablet.addValue(rowIndex, 1, "5");
+ tablet.addValue(rowIndex, 2, "4");
+ tablet.addValue(rowIndex, 3, 37.6F);
+ tablet.addValue(
+ rowIndex,
+ 4,
+ true,
+ 0,
+ Files.readAllBytes(Paths.get("/Users/jackietien/Downloads/1751891242743.tiff")));
+ session.insert(tablet);
+ tablet.reset();
+
+ } catch (IoTDBConnectionException e) {
+ e.printStackTrace();
+ } catch (StatementExecutionException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/example/session/src/main/java/org/apache/iotdb/ObjectReadExample.java b/example/session/src/main/java/org/apache/iotdb/ObjectReadExample.java
new file mode 100644
index 00000000000..344939175db
--- /dev/null
+++ b/example/session/src/main/java/org/apache/iotdb/ObjectReadExample.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb;
+
+import org.apache.iotdb.isession.ITableSession;
+import org.apache.iotdb.isession.SessionDataSet;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.session.TableSessionBuilder;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.tsfile.utils.Binary;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+
+public class ObjectReadExample {
+ private static final String LOCAL_URL = "127.0.0.1:6667";
+
+ public static void main(String[] args) {
+
+ // don't specify database in constructor
+ try (ITableSession session =
+ new TableSessionBuilder()
+ .nodeUrls(Collections.singletonList(LOCAL_URL))
+ .username("root")
+ .password("root")
+ .database("test1")
+ .thriftMaxFrameSize(256 * 1024 * 1024)
+ .build()) {
+ try (SessionDataSet dataSet =
+ session.executeQueryStatement(
+ "select READ_OBJECT(file) from tsfile_table where time = 1")) {
+ SessionDataSet.DataIterator iterator = dataSet.iterator();
+ while (iterator.next()) {
+ Binary binary = iterator.getBlob(1);
+ System.out.println(DigestUtils.md5Hex(binary.getValues()));
+ }
+ }
+
+ try (SessionDataSet dataSet =
+ session.executeQueryStatement(
+ "select READ_OBJECT(file) from tsfile_table where time = 2")) {
+ SessionDataSet.DataIterator iterator = dataSet.iterator();
+ while (iterator.next()) {
+ Binary binary = iterator.getBlob(1);
+ System.out.println(DigestUtils.md5Hex(binary.getValues()));
+ }
+ }
+
+ try (SessionDataSet dataSet =
+ session.executeQueryStatement("select READ_OBJECT(file) from tsfile_table")) {
+ SessionDataSet.DataIterator iterator = dataSet.iterator();
+ while (iterator.next()) {
+ Binary binary = iterator.getBlob(1);
+ System.out.println(DigestUtils.md5Hex(binary.getValues()));
+ }
+ }
+
+ try (SessionDataSet dataSet =
+ session.executeQueryStatement(
+ "select geo_penetrate(file, '0,3,7501,7504') from tsfile_table")) {
+ SessionDataSet.DataIterator iterator = dataSet.iterator();
+ while (iterator.next()) {
+ Binary binary = iterator.getBlob(1);
+ ByteBuffer byteBuffer = ByteBuffer.wrap(binary.getValues());
+ float[] res = new float[byteBuffer.limit() / Float.BYTES];
+ for (int i = 0; i < res.length; i++) {
+ res[i] = byteBuffer.getFloat();
+ }
+ System.out.println(Arrays.toString(res));
+ }
+ }
+
+ try (SessionDataSet dataSet =
+ session.executeQueryStatement(
+ "select geo_penetrate(file, '0,3,7501,7504', 'UNCOMPRESSED_TIFF') from tiff_table")) {
+ SessionDataSet.DataIterator iterator = dataSet.iterator();
+ while (iterator.next()) {
+ Binary binary = iterator.getBlob(1);
+ ByteBuffer byteBuffer = ByteBuffer.wrap(binary.getValues());
+ float[] res = new float[byteBuffer.limit() / Float.BYTES];
+ for (int i = 0; i < res.length; i++) {
+ res[i] = byteBuffer.getFloat();
+ }
+ System.out.println(Arrays.toString(res));
+ }
+ }
+
+ } catch (IoTDBConnectionException e) {
+ e.printStackTrace();
+ } catch (StatementExecutionException e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java b/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java
index 0a335f02c96..19b245653ba 100644
--- a/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java
+++ b/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java
@@ -113,7 +113,7 @@ public class TableModelSessionPoolExample {
int rowIndex = tablet.getRowSize();
tablet.addTimestamp(rowIndex, timestamp);
tablet.addValue("region_id", rowIndex, "1");
- tablet.addValue("plant_id", rowIndex, "5");
+ tablet.addValue("plant_id", rowIndex, null);
tablet.addValue("device_id", rowIndex, "3");
tablet.addValue("model", rowIndex, "A");
tablet.addValue("temperature", rowIndex, 37.6F);
@@ -128,6 +128,26 @@ public class TableModelSessionPoolExample {
tablet.reset();
}
+ // query device leader
+ List<Boolean> isSetTag = Arrays.asList(true, true, false, true);
+ String correctURL =
+ session.getDeviceLeaderURL("test2", Arrays.asList("test1", "1", "3"), isSetTag, 66);
+ System.out.println("Correct device leader URL: " + correctURL);
+ String errorDbURL =
+ session.getDeviceLeaderURL("test3", Arrays.asList("test1", "1", "3"), isSetTag, 66);
+ System.out.println("Error dbName device leader URL: " + errorDbURL);
+ String errorDeviceURL =
+ session.getDeviceLeaderURL("test2", Arrays.asList("test1", "3", "1"), isSetTag, 66);
+ System.out.println("Error deviceId device leader URL: " + errorDeviceURL);
+ List<Boolean> falseTagList = Arrays.asList(false, true, true, true);
+ String errorTagURL =
+ session.getDeviceLeaderURL("test2", Arrays.asList("test1", "1", "3"), falseTagList, 66);
+ System.out.println("Error tag device leader URL: " + errorTagURL);
+ String errorTimeURL =
+ session.getDeviceLeaderURL(
+ "test2", Arrays.asList("test1", "1", "3"), isSetTag, 6666666666666666L);
+ System.out.println("Error time device leader URL: " + errorTimeURL);
+
// query table data
try (SessionDataSet dataSet =
session.executeQueryStatement(
diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java
index 9093133631a..59819ee418e 100644
--- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java
+++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java
@@ -49,7 +49,10 @@ public enum Type {
BLOB((byte) 10),
/* STRING */
- STRING((byte) 11);
+ STRING((byte) 11),
+
+ /* OBJECT */
+ OBJECT((byte) 12);
private final byte dataType;
diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java
index a5c2852ac1b..c4c38c285dc 100644
--- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java
+++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java
@@ -56,7 +56,11 @@ public enum Type {
BLOB((byte) 10),
/* STRING */
- STRING((byte) 11);
+ STRING((byte) 11),
+
+ /* OBJECT */
+ OBJECT((byte) 12);
+
private final byte dataType;
Type(byte type) {
@@ -92,6 +96,7 @@ public enum Type {
case DATE:
return o instanceof LocalDate;
case BLOB:
+ case OBJECT:
return o instanceof Binary;
case STRING:
case TEXT:
@@ -102,7 +107,8 @@ public enum Type {
}
public static List<Type> allTypes() {
- return Arrays.asList(BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, TIMESTAMP, DATE, BLOB, STRING);
+ return Arrays.asList(
+ BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, TIMESTAMP, DATE, BLOB, STRING, OBJECT);
}
public static List<Type> numericTypes() {
diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java
index aee48bd9927..17587cf8811 100644
--- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java
+++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java
@@ -742,6 +742,7 @@ public abstract class AbstractCli {
case DOUBLE:
case TEXT:
case STRING:
+ case OBJECT:
return resultSet.getString(columnIndex);
case BLOB:
byte[] v = resultSet.getBytes(columnIndex);
diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java
index fa67708fed0..bb113680d8e 100644
--- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java
+++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java
@@ -189,6 +189,7 @@ public class SessionDataSet implements ISessionDataSet {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
field.setBinaryV(ioTDBRpcDataSet.getBinary(columnName));
break;
default:
diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java
index 8baf1f9b6f2..f14e2ee6b32 100644
--- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java
+++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java
@@ -396,6 +396,7 @@ public abstract class IoTDBAbstractDatabaseMetadata implements DatabaseMetaData
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
tsBlockBuilder
.getColumnBuilder(j)
.writeBinary(
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java
index 5bba6d0ea71..43064ecdaa9 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java
@@ -184,6 +184,7 @@ public class IoTDBJDBCDataSet {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
values[i] = null;
break;
default:
@@ -307,6 +308,7 @@ public class IoTDBJDBCDataSet {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
values[i] = null;
break;
default:
@@ -418,6 +420,7 @@ public class IoTDBJDBCDataSet {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
int length = valueBuffer.getInt();
values[i] = ReadWriteIOUtils.readBytes(valueBuffer, length);
break;
@@ -596,6 +599,8 @@ public class IoTDBJDBCDataSet {
case TEXT:
case STRING:
return new String(values[index], StandardCharsets.UTF_8);
+ case OBJECT:
+ return BytesUtils.parseObjectByteArrayToString(values[index]);
case BLOB:
return BytesUtils.parseBlobByteArrayToString(values[index]);
case DATE:
@@ -634,6 +639,7 @@ public class IoTDBJDBCDataSet {
case TEXT:
case STRING:
return new String(values[index], StandardCharsets.UTF_8);
+ case OBJECT:
case BLOB:
return new Binary(values[index]);
case TIMESTAMP:
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java
index a40197fac79..940b3460d92 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java
@@ -494,6 +494,9 @@ public class IoTDBRpcDataSet {
.getColumn(tsBlockColumnIndex)
.getBinary(tsBlockIndex)
.getStringValue(TSFileConfig.STRING_CHARSET);
+ case OBJECT:
+ return BytesUtils.parseObjectByteArrayToString(
+ curTsBlock.getColumn(tsBlockColumnIndex).getBinary(tsBlockIndex).getValues());
case BLOB:
return BytesUtils.parseBlobByteArrayToString(
curTsBlock.getColumn(tsBlockColumnIndex).getBinary(tsBlockIndex).getValues());
@@ -554,6 +557,9 @@ public class IoTDBRpcDataSet {
.getColumn(index)
.getBinary(tsBlockIndex)
.getStringValue(TSFileConfig.STRING_CHARSET);
+ case OBJECT:
+ return BytesUtils.parseObjectByteArrayToString(
+ curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues());
case BLOB:
return BytesUtils.parseBlobByteArrayToString(
curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues());
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
index 6bd0da063b0..ddf178889b4 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
@@ -145,6 +145,9 @@ public enum TSStatusCode {
PLAN_FAILED_NETWORK_PARTITION(721),
CANNOT_FETCH_FI_STATE(722),
+ // OBJECT
+ OBJECT_NOT_EXISTS(740),
+
// Arithmetic
NUMERIC_VALUE_OUT_OF_RANGE(750),
DIVISION_BY_ZERO(751),
diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
index 8844f36c247..ab28bd10ad9 100644
--- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
+++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
@@ -3745,6 +3745,7 @@ public class Session implements ISession {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary[] binaryValues = (Binary[]) valueList;
Binary[] sortedBinaryValues = new Binary[binaryValues.length];
for (int i = 0; i < index.length; i++) {
diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java
index a3f7f8e3f26..a3f01b3e11e 100644
--- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java
+++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java
@@ -242,6 +242,7 @@ public class SubscriptionSessionDataSet implements ISessionDataSet {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
final Binary binaryValue = new Binary((((Binary[]) value)[index]).getValues());
field.setBinaryV(binaryValue);
break;
diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
index dd30496090c..1b5ad290678 100644
--- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
+++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
@@ -135,6 +135,7 @@ public class SessionUtils {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueOccupation += rowSize * 4;
Binary[] binaries = (Binary[]) values[columnIndex];
for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) {
@@ -185,6 +186,7 @@ public class SessionUtils {
break;
case TEXT:
case STRING:
+ case OBJECT:
res += Integer.BYTES;
if (values.get(i) instanceof Binary) {
res += ((Binary) values.get(i)).getValues().length;
@@ -336,6 +338,7 @@ public class SessionUtils {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
Binary[] binaryValues = (Binary[]) tablet.getValues()[i];
for (int index = 0; index < tablet.getRowSize(); index++) {
if (!tablet.isNull(index, i) && binaryValues[index] != null) {
diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml
index 80bebe90f77..c660b3aa853 100644
--- a/iotdb-core/datanode/pom.xml
+++ b/iotdb-core/datanode/pom.xml
@@ -371,6 +371,11 @@
<version>1.3.0</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.gdal</groupId>
+ <artifactId>gdal</artifactId>
+ <version>3.11.0</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java
index 4215585b499..373cec94a66 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java
@@ -64,8 +64,7 @@ public class DataNodeMemoryConfig {
private int queryThreadCount = Runtime.getRuntime().availableProcessors();
/** Max bytes of each FragmentInstance for DataExchange */
- private long maxBytesPerFragmentInstance =
- Runtime.getRuntime().maxMemory() * 3 / 10 * 200 / 1001 / queryThreadCount;
+ private long maxBytesPerFragmentInstance = Runtime.getRuntime().maxMemory() * 3 / 10 * 200 / 1001;
/** The memory manager of on heap */
private MemoryManager onHeapMemoryManager;
@@ -487,7 +486,7 @@ public class DataNodeMemoryConfig {
operatorsMemorySize += partForOperators;
}
// set max bytes per fragment instance
- setMaxBytesPerFragmentInstance(dataExchangeMemorySize / getQueryThreadCount());
+ setMaxBytesPerFragmentInstance(dataExchangeMemorySize);
bloomFilterCacheMemoryManager =
queryEngineMemoryManager.getOrCreateMemoryManager(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 9f67ab268c2..3f19a2101a8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -1206,6 +1206,7 @@ public class IoTDBConfig {
private ConcurrentHashMap<String, EncryptParameter> tsFileDBToEncryptMap =
new ConcurrentHashMap<>(
Collections.singletonMap("root.__audit", new EncryptParameter("UNENCRYPTED", null)));
+ private long maxObjectSizeInByte = 4 * 1024 * 1024 * 1024L;
IoTDBConfig() {}
@@ -4326,4 +4327,12 @@ public class IoTDBConfig {
public ConcurrentHashMap<String, EncryptParameter> getTSFileDBToEncryptMap() {
return tsFileDBToEncryptMap;
}
+
+ public long getMaxObjectSizeInByte() {
+ return maxObjectSizeInByte;
+ }
+
+ public void setMaxObjectSizeInByte(long maxObjectSizeInByte) {
+ this.maxObjectSizeInByte = maxObjectSizeInByte;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 2fb7f7efdf0..2915365262d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -900,10 +900,17 @@ public class IoTDBDescriptor {
}
conf.setExtPipeDir(properties.getProperty("ext_pipe_dir", conf.getExtPipeDir()).trim());
+
conf.setPipeTaskThreadCount(
Integer.parseInt(
properties.getProperty(
"pipe_task_thread_count", Integer.toString(conf.getPipeTaskThreadCount()).trim())));
+
+ conf.setMaxObjectSizeInByte(
+ Long.parseLong(
+ properties.getProperty(
+ "max_object_file_size_in_byte", String.valueOf(conf.getMaxObjectSizeInByte()))));
+
// At the same time, set TSFileConfig
List<FSType> fsTypes = new ArrayList<>();
fsTypes.add(FSType.LOCAL);
@@ -2170,6 +2177,10 @@ public class IoTDBDescriptor {
"include_null_value_in_write_throughput_metric",
ConfigurationFileUtils.getConfigurationDefaultValue(
"include_null_value_in_write_throughput_metric"))));
+ conf.setMaxObjectSizeInByte(
+ Long.parseLong(
+ properties.getProperty(
+ "max_object_file_size_in_byte", String.valueOf(conf.getMaxObjectSizeInByte()))));
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
@@ -2729,6 +2740,7 @@ public class IoTDBDescriptor {
return conf.getDefaultDoubleEncoding();
case STRING:
case BLOB:
+ case OBJECT:
case TEXT:
default:
return conf.getDefaultTextEncoding();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
index cdce05b46c0..643f62830c2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode;
@@ -292,4 +293,16 @@ public class DataExecutionVisitor extends PlanVisitor<TSStatus, DataRegion> {
node.getDeleteDataNode().markAsGeneratedByPipe();
return node.getDeleteDataNode().accept(this, context);
}
+
+ @Override
+ public TSStatus visitWriteObjectFile(ObjectNode node, DataRegion dataRegion) {
+ try {
+ dataRegion.writeObject(node);
+ dataRegion.insertSeparatorToWAL();
+ return StatusUtils.OK;
+ } catch (final Exception e) {
+ LOGGER.error("Error in executing plan node: {}", node, e);
+ return new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
index d16c404e422..10a72a0d9c5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
@@ -498,6 +498,7 @@ public class AggregateProcessor implements PipeProcessor {
timestamp, row.getString(index), outputMinReportIntervalMilliseconds);
break;
case BLOB:
+ case OBJECT:
result =
state.updateWindows(
timestamp, row.getBinary(index), outputMinReportIntervalMilliseconds);
@@ -685,6 +686,7 @@ public class AggregateProcessor implements PipeProcessor {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
valueColumns[columnIndex] = new Binary[distinctOutputs.size()];
break;
@@ -734,6 +736,7 @@ public class AggregateProcessor implements PipeProcessor {
TSFileConfig.STRING_CHARSET);
break;
case BLOB:
+ case OBJECT:
((Binary[]) valueColumns[columnIndex])[rowIndex] =
(Binary) aggregatedResults.get(columnNameStringList[columnIndex]).getRight();
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java
index 6e17d637681..7f1d7357b02 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java
@@ -602,6 +602,7 @@ public class InsertNodeMemoryEstimator {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
{
size += RamUsageEstimator.sizeOf((Binary[]) columns[i]);
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java
index 8ca78a2da55..1ba0a8841d2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java
@@ -318,6 +318,7 @@ public class OpcDaServerHandle implements Closeable {
// Note that "Variant" does not support "VT_BLOB" data, and not all the DA server
// support this, thus we use "VT_BSTR" to substitute
case BLOB:
+ case OBJECT:
return Variant.VT_BSTR;
default:
throw new UnSupportedDataTypeException("UnSupported dataType " + dataType);
@@ -354,6 +355,7 @@ public class OpcDaServerHandle implements Closeable {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
bstr = OleAuto.INSTANCE.SysAllocString(((Binary[]) column)[rowIndex].toString());
value.setValue(Variant.VT_BSTR, bstr);
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java
index 817df1c06e0..46a3fc6df94 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java
@@ -185,6 +185,7 @@ public class PipeInsertEventSorter {
return deDuplicatedDoubleValues;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
final Binary[] binaryValues = (Binary[]) valueList;
final Binary[] deDuplicatedBinaryValues = new Binary[binaryValues.length];
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
index cb5347e51f2..167a1fa914f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
@@ -890,7 +890,7 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler {
IMeasurementSchema measurementSchema = new MeasurementSchema(measurement, dataType);
AbstractSeriesAggregationScanOperator operator;
boolean canUseStatistics =
- !TSDataType.BLOB.equals(dataType)
+ (!TSDataType.BLOB.equals(dataType) && !TSDataType.OBJECT.equals(dataType))
|| (!TAggregationType.LAST_VALUE.equals(aggregationType)
&& !TAggregationType.FIRST_VALUE.equals(aggregationType));
IFullPath path;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java
index c61d769bcb3..fb7938309af 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java
@@ -163,6 +163,7 @@ public class AccumulatorFactory {
return new FloatModeAccumulator();
case DOUBLE:
return new DoubleModeAccumulator();
+ case OBJECT:
default:
throw new IllegalArgumentException("Unknown data type: " + tsDataType);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java
index c6d1baa3383..6549a7a6748 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java
@@ -57,6 +57,7 @@ public class AvgAccumulator implements Accumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
case BOOLEAN:
case DATE:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java
index 76a42b41c71..ff54a48ce5e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java
@@ -58,6 +58,7 @@ public class ExtremeAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -90,6 +91,7 @@ public class ExtremeAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -124,6 +126,7 @@ public class ExtremeAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -155,6 +158,7 @@ public class ExtremeAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -188,6 +192,7 @@ public class ExtremeAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -219,6 +224,7 @@ public class ExtremeAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java
index 3bc2e0f9c89..640008f0a5f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java
@@ -63,6 +63,7 @@ public class FirstValueAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(columns, bitMap);
return;
case BOOLEAN:
@@ -98,6 +99,7 @@ public class FirstValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
updateBinaryFirstValue(partialResult[0].getBinary(0), partialResult[1].getLong(0));
break;
@@ -132,6 +134,7 @@ public class FirstValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
updateBinaryFirstValue((Binary) statistics.getFirstValue(), statistics.getStartTime());
break;
@@ -167,6 +170,7 @@ public class FirstValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
firstValue.setBinary(finalResult.getBinary(0));
break;
@@ -206,6 +210,7 @@ public class FirstValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
columnBuilders[0].writeBinary(firstValue.getBinary());
break;
@@ -242,6 +247,7 @@ public class FirstValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
columnBuilder.writeBinary(firstValue.getBinary());
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java
index 603f3427860..36c6ed5f88d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java
@@ -63,6 +63,7 @@ public class LastValueAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(columns, bitMap);
return;
case BOOLEAN:
@@ -99,6 +100,7 @@ public class LastValueAccumulator implements Accumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
updateBinaryLastValue(partialResult[0].getBinary(0), partialResult[1].getLong(0));
break;
case BOOLEAN:
@@ -133,6 +135,7 @@ public class LastValueAccumulator implements Accumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
updateBinaryLastValue((Binary) statistics.getLastValue(), statistics.getEndTime());
break;
case BOOLEAN:
@@ -168,6 +171,7 @@ public class LastValueAccumulator implements Accumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
lastValue.setBinary(finalResult.getBinary(0));
break;
case BOOLEAN:
@@ -207,6 +211,7 @@ public class LastValueAccumulator implements Accumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
columnBuilders[0].writeBinary(lastValue.getBinary());
break;
case BOOLEAN:
@@ -243,6 +248,7 @@ public class LastValueAccumulator implements Accumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
columnBuilder.writeBinary(lastValue.getBinary());
break;
case BOOLEAN:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java
index a0e4f80120c..ebbd1aa6dc1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java
@@ -88,6 +88,7 @@ public abstract class MaxMinByBaseAccumulator implements Accumulator {
case TEXT:
case BLOB:
case BOOLEAN:
+ case OBJECT:
default:
throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, yDataType));
}
@@ -302,6 +303,7 @@ public abstract class MaxMinByBaseAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
columnBuilder.writeBinary(xResult.getBinary());
break;
case BOOLEAN:
@@ -335,6 +337,7 @@ public abstract class MaxMinByBaseAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
xResult.setBinary(xColumn.getBinary(xIndex));
break;
case BOOLEAN:
@@ -385,6 +388,7 @@ public abstract class MaxMinByBaseAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
String content = value.getBinary().toString();
dataOutputStream.writeInt(content.length());
dataOutputStream.writeBytes(content);
@@ -441,6 +445,7 @@ public abstract class MaxMinByBaseAccumulator implements Accumulator {
case TEXT:
case BLOB:
case BOOLEAN:
+ case OBJECT:
default:
throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, yDataType));
}
@@ -471,6 +476,7 @@ public abstract class MaxMinByBaseAccumulator implements Accumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length)));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java
index 0d58de8064f..d02678bd622 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java
@@ -64,6 +64,7 @@ public class MaxValueAccumulator implements Accumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -98,6 +99,7 @@ public class MaxValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -130,6 +132,7 @@ public class MaxValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -164,6 +167,7 @@ public class MaxValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -199,6 +203,7 @@ public class MaxValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -232,6 +237,7 @@ public class MaxValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java
index 1d9cc59aa17..fd55ea0f1bc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java
@@ -64,6 +64,7 @@ public class MinValueAccumulator implements Accumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -98,6 +99,7 @@ public class MinValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -130,6 +132,7 @@ public class MinValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -164,6 +167,7 @@ public class MinValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -199,6 +203,7 @@ public class MinValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -232,6 +237,7 @@ public class MinValueAccumulator implements Accumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java
index 37daf1a84b1..066d8cd4fc5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java
@@ -57,6 +57,7 @@ public class SumAccumulator implements Accumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case TIMESTAMP:
case DATE:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java
index 3242518c3dc..8d27f1497e0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java
@@ -70,6 +70,7 @@ public class VarianceAccumulator implements Accumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case STRING:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java
index f45e71110d5..b5a831b90b2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java
@@ -66,6 +66,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode;
@@ -370,6 +371,19 @@ public class RegionWriteExecutor {
}
}
+ @Override
+ public RegionExecutionResult visitWriteObjectFile(
+ final ObjectNode node, final WritePlanNodeExecutionContext context) {
+ // data deletion don't need to block data insertion, but there are some creation operation
+ // require write lock on data region.
+ context.getRegionWriteValidationRWLock().writeLock().lock();
+ try {
+ return super.visitWriteObjectFile(node, context);
+ } finally {
+ context.getRegionWriteValidationRWLock().writeLock().unlock();
+ }
+ }
+
@Override
public RegionExecutionResult visitDeleteTimeseries(
final DeleteTimeSeriesNode node, final WritePlanNodeExecutionContext context) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java
index 03766ab5276..b6ce8b52110 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java
@@ -30,7 +30,6 @@ public class LocalMemoryManager {
private final MemoryPool queryPool;
public LocalMemoryManager() {
- // TODO @spricoder: why this pool is only used for query data exchange
queryPool =
new MemoryPool(
"read",
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java
index 7c671e7b274..2df0be4c339 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java
@@ -262,6 +262,7 @@ public class AggregationUtil {
return BooleanColumn.SIZE_IN_BYTES_PER_POSITION;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
return StatisticsManager.getInstance().getMaxBinarySizeInBytes();
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java
index 4668da6413e..bf021e5b4f4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java
@@ -306,6 +306,7 @@ public abstract class TopKOperator implements ProcessOperator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
columns[i] =
new BinaryColumn(
positionCount,
@@ -382,6 +383,7 @@ public abstract class TopKOperator implements ProcessOperator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
memory += 16;
break;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java
index 6dd24e3795a..28a80462ea7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java
@@ -344,6 +344,7 @@ public class TransformOperator implements ProcessOperator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
writer.writeBinary(valueColumn.getBinary(currentIndex));
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java
index 205808fd120..468e340200a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java
@@ -113,6 +113,7 @@ public class MergeSortComparator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
comparator =
Comparator.comparing(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java
index 90b519d52cd..91e39753344 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java
@@ -105,6 +105,7 @@ public class LagFunction extends ValueWindowFunction {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
builder.writeBinary(partition.getBinary(defaultValChannel, index));
return;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java
index e21ff13a861..c51eac15635 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java
@@ -106,6 +106,7 @@ public class LeadFunction extends ValueWindowFunction {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
builder.writeBinary(partition.getBinary(defaultValChannel, index));
return;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java
index ac6f2258809..73ff0fe60e6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java
@@ -99,6 +99,7 @@ public class RowComparator {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
Binary bin1 = column.getBinary(offset1);
Binary bin2 = column.getBinary(offset2);
if (!bin1.equals(bin2)) {
@@ -178,6 +179,7 @@ public class RowComparator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
Binary bin1 = column.getBinary(offset1);
Binary bin2 = column.getBinary(offset2);
if (!bin1.equals(bin2)) {
@@ -242,6 +244,7 @@ public class RowComparator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
Binary bin1 = column1.getBinary(offset1);
Binary bin2 = column2.getBinary(offset2);
if (!bin1.equals(bin2)) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
index 233d24bf097..e2fbb4be4ec 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
@@ -1068,6 +1068,7 @@ public class SeriesScanUtil implements Accountable {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
builder.getColumnBuilder(0).writeBinary(timeValuePair.getValue().getBinary());
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java
index fffd43043d3..999842cece3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java
@@ -726,6 +726,7 @@ public class LastQueryAggTableScanOperator extends AbstractAggTableScanOperator
return new TsPrimitiveType.TsDouble(originalValue.getDouble());
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
return new TsPrimitiveType.TsBinary(originalValue.getBinary());
case VECTOR:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java
index d7195cc6dfb..45617de31d1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java
@@ -391,6 +391,7 @@ public class AccumulatorFactory {
return new BinaryGroupedApproxMostFrequentAccumulator();
case BLOB:
return new BlobGroupedApproxMostFrequentAccumulator();
+ case OBJECT:
default:
throw new UnSupportedDataTypeException(
String.format("Unsupported data type in APPROX_COUNT_DISTINCT Aggregation: %s", type));
@@ -416,6 +417,7 @@ public class AccumulatorFactory {
return new BinaryApproxMostFrequentAccumulator();
case BLOB:
return new BlobApproxMostFrequentAccumulator();
+ case OBJECT:
default:
throw new UnSupportedDataTypeException(
String.format("Unsupported data type in APPROX_COUNT_DISTINCT Aggregation: %s", type));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java
index b78aa1b0177..ef31759ee67 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java
@@ -78,6 +78,7 @@ public class ApproxCountDistinctAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(arguments[0], mask, hll);
return;
case BOOLEAN:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java
index 464820e22f4..7bb03faadeb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java
@@ -73,6 +73,7 @@ public class AvgAccumulator implements TableAccumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
case BOOLEAN:
case DATE:
@@ -101,6 +102,7 @@ public class AvgAccumulator implements TableAccumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
case BOOLEAN:
case DATE:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java
index db50a3cb7d7..cc9d3e3354c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java
@@ -68,6 +68,7 @@ public class ExtremeAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -100,6 +101,7 @@ public class ExtremeAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -136,6 +138,7 @@ public class ExtremeAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -168,6 +171,7 @@ public class ExtremeAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -200,6 +204,7 @@ public class ExtremeAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java
index 3e4111fd454..2ca9b7785cc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java
@@ -82,6 +82,7 @@ public class FirstAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(arguments[0], arguments[1], mask);
return;
case BOOLEAN:
@@ -132,6 +133,7 @@ public class FirstAccumulator implements TableAccumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length));
@@ -184,6 +186,7 @@ public class FirstAccumulator implements TableAccumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
columnBuilder.writeBinary(firstValue.getBinary());
break;
case BOOLEAN:
@@ -225,6 +228,7 @@ public class FirstAccumulator implements TableAccumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
updateBinaryFirstValue(
(Binary) statistics[0].getFirstValue(), statistics[0].getStartTime());
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java
index e53bfd0bfd5..2ca6c201f94 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java
@@ -104,6 +104,7 @@ public class FirstByAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(arguments[0], arguments[1], arguments[2], mask);
return;
case BOOLEAN:
@@ -164,6 +165,7 @@ public class FirstByAccumulator implements TableAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
@@ -219,6 +221,7 @@ public class FirstByAccumulator implements TableAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
columnBuilder.writeBinary(xResult.getBinary());
break;
@@ -278,6 +281,7 @@ public class FirstByAccumulator implements TableAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
xResult.setBinary((Binary) statistics[0].getFirstValue());
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java
index d805500b118..40f8e2ef97a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java
@@ -92,6 +92,7 @@ public class LastAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(arguments[0], arguments[1], mask);
return;
case BOOLEAN:
@@ -141,6 +142,7 @@ public class LastAccumulator implements TableAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
@@ -193,6 +195,7 @@ public class LastAccumulator implements TableAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
columnBuilder.writeBinary(lastValue.getBinary());
break;
@@ -234,6 +237,7 @@ public class LastAccumulator implements TableAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
updateBinaryLastValue((Binary) statistics[0].getLastValue(), statistics[0].getEndTime());
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java
index 70024273157..2ee09380b83 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java
@@ -112,6 +112,7 @@ public class LastByAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(arguments[0], arguments[1], arguments[2], mask);
return;
case BOOLEAN:
@@ -173,6 +174,7 @@ public class LastByAccumulator implements TableAccumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length));
@@ -228,6 +230,7 @@ public class LastByAccumulator implements TableAccumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
columnBuilder.writeBinary(xResult.getBinary());
break;
case BOOLEAN:
@@ -287,6 +290,7 @@ public class LastByAccumulator implements TableAccumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
xResult.setBinary((Binary) statistics[0].getLastValue());
break;
case BOOLEAN:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java
index aaab07aa90b..6f42e1a9995 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java
@@ -67,6 +67,7 @@ public class SumAccumulator implements TableAccumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
case BOOLEAN:
case DATE:
@@ -95,6 +96,7 @@ public class SumAccumulator implements TableAccumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
case BOOLEAN:
case DATE:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java
index c33fd5f703a..c6002304b10 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java
@@ -85,6 +85,7 @@ public abstract class TableMaxMinByBaseAccumulator implements TableAccumulator {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
addBinaryInput(arguments, mask);
return;
case BOOLEAN:
@@ -350,6 +351,7 @@ public abstract class TableMaxMinByBaseAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
columnBuilder.writeBinary(xResult.getBinary());
break;
case BOOLEAN:
@@ -383,6 +385,7 @@ public abstract class TableMaxMinByBaseAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
xResult.setBinary(xColumn.getBinary(xIndex));
break;
case BOOLEAN:
@@ -451,6 +454,7 @@ public abstract class TableMaxMinByBaseAccumulator implements TableAccumulator {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
Binary binaryMaxVal = new Binary(BytesUtils.subBytes(bytes, offset, length));
@@ -494,6 +498,7 @@ public abstract class TableMaxMinByBaseAccumulator implements TableAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length)));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java
index 355acba8aa6..92ea803f2d0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java
@@ -79,6 +79,7 @@ public class TableVarianceAccumulator implements TableAccumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case STRING:
@@ -106,6 +107,7 @@ public class TableVarianceAccumulator implements TableAccumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case STRING:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java
index 37f2f5d912d..99d5fef1208 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java
@@ -54,6 +54,7 @@ public class Utils {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
BytesUtils.intToBytes(value.getBinary().getValues().length, valueBytes, offset);
offset += 4;
System.arraycopy(
@@ -116,6 +117,7 @@ public class Utils {
return 8;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
return 4 + value.getBinary().getValues().length;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java
index 2573350a31d..9fac50c2ad2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java
@@ -78,6 +78,7 @@ public class GroupedApproxCountDistinctAccumulator implements GroupedAccumulator
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(groupIds, arguments[0], mask, hlls, maxStandardError);
return;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java
index 2947c258e79..4e177c352b7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java
@@ -77,6 +77,7 @@ public class GroupedAvgAccumulator implements GroupedAccumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
case BOOLEAN:
case DATE:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java
index 8d46ab274f4..f53ea593142 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java
@@ -62,6 +62,7 @@ public class GroupedExtremeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -92,6 +93,7 @@ public class GroupedExtremeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
break;
case BOOLEAN:
break;
@@ -124,6 +126,7 @@ public class GroupedExtremeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
throw new UnSupportedDataTypeException(
@@ -150,6 +153,7 @@ public class GroupedExtremeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -183,6 +187,7 @@ public class GroupedExtremeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -214,6 +219,7 @@ public class GroupedExtremeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -245,6 +251,7 @@ public class GroupedExtremeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
@@ -277,6 +284,7 @@ public class GroupedExtremeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case TIMESTAMP:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java
index a1884162c49..b8fe8bbe1b2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java
@@ -78,6 +78,7 @@ public class GroupedFirstAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues = new BinaryBigArray();
return;
case BOOLEAN:
@@ -110,6 +111,7 @@ public class GroupedFirstAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
valuesSize += binaryValues.sizeOf();
break;
case BOOLEAN:
@@ -144,6 +146,7 @@ public class GroupedFirstAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues.ensureCapacity(groupCount);
return;
case BOOLEAN:
@@ -176,6 +179,7 @@ public class GroupedFirstAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(groupIds, arguments[0], arguments[1], mask);
return;
case BOOLEAN:
@@ -225,6 +229,7 @@ public class GroupedFirstAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
@@ -276,6 +281,7 @@ public class GroupedFirstAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
columnBuilder.writeBinary(binaryValues.get(groupId));
break;
@@ -313,6 +319,7 @@ public class GroupedFirstAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues.reset();
return;
case BOOLEAN:
@@ -356,6 +363,7 @@ public class GroupedFirstAccumulator implements GroupedAccumulator {
return bytes;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
byte[] values = binaryValues.get(groupId).getValues();
length += Integer.BYTES + values.length;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java
index 33ee0f90873..d56ba59ef94 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java
@@ -87,6 +87,7 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
xBinaryValues = new BinaryBigArray();
break;
@@ -120,6 +121,7 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
valuesSize += xBinaryValues.sizeOf();
break;
case BOOLEAN:
@@ -156,6 +158,7 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
xBinaryValues.ensureCapacity(groupCount);
return;
case BOOLEAN:
@@ -192,6 +195,7 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
xBinaryValues.reset();
break;
@@ -227,6 +231,7 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(groupIds, arguments[0], arguments[1], arguments[2], mask);
return;
case BOOLEAN:
@@ -288,6 +293,7 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
@@ -343,6 +349,7 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator {
return bytes;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
byte[] values = xBinaryValues.get(groupId).getValues();
intToBytes(values.length, bytes, Long.BYTES + 1);
@@ -373,6 +380,7 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator {
return Double.BYTES;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
return Integer.BYTES + xBinaryValues.get(groupId).getValues().length;
case BOOLEAN:
@@ -407,6 +415,7 @@ public class GroupedFirstByAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
columnBuilder.writeBinary(xBinaryValues.get(groupId));
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java
index 75cc674ef96..b21a683ab0a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java
@@ -78,6 +78,7 @@ public class GroupedLastAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues = new BinaryBigArray();
return;
case BOOLEAN:
@@ -110,6 +111,7 @@ public class GroupedLastAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
valuesSize += binaryValues.sizeOf();
break;
case BOOLEAN:
@@ -144,6 +146,7 @@ public class GroupedLastAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues.ensureCapacity(groupCount);
return;
case BOOLEAN:
@@ -176,6 +179,7 @@ public class GroupedLastAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(groupIds, arguments[0], arguments[1], mask);
return;
case BOOLEAN:
@@ -225,6 +229,7 @@ public class GroupedLastAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
@@ -276,6 +281,7 @@ public class GroupedLastAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
columnBuilder.writeBinary(binaryValues.get(groupId));
break;
@@ -313,6 +319,7 @@ public class GroupedLastAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues.reset();
return;
case BOOLEAN:
@@ -356,6 +363,7 @@ public class GroupedLastAccumulator implements GroupedAccumulator {
return bytes;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
byte[] values = binaryValues.get(groupId).getValues();
length += Integer.BYTES + values.length;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java
index 386d4c2f34f..db48d221f18 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java
@@ -87,6 +87,7 @@ public class GroupedLastByAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
xBinaryValues = new BinaryBigArray();
break;
@@ -120,6 +121,7 @@ public class GroupedLastByAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
valuesSize += xBinaryValues.sizeOf();
break;
case BOOLEAN:
@@ -156,6 +158,7 @@ public class GroupedLastByAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
xBinaryValues.ensureCapacity(groupCount);
return;
case BOOLEAN:
@@ -192,6 +195,7 @@ public class GroupedLastByAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
xBinaryValues.reset();
break;
@@ -227,6 +231,7 @@ public class GroupedLastByAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(groupIds, arguments[0], arguments[1], arguments[2], mask);
return;
case BOOLEAN:
@@ -288,6 +293,7 @@ public class GroupedLastByAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
@@ -343,6 +349,7 @@ public class GroupedLastByAccumulator implements GroupedAccumulator {
return bytes;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
byte[] values = xBinaryValues.get(groupId).getValues();
intToBytes(values.length, bytes, Long.BYTES + 1);
@@ -373,6 +380,7 @@ public class GroupedLastByAccumulator implements GroupedAccumulator {
return Double.BYTES;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
return Integer.BYTES + xBinaryValues.get(groupId).getValues().length;
case BOOLEAN:
@@ -407,6 +415,7 @@ public class GroupedLastByAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
columnBuilder.writeBinary(xBinaryValues.get(groupId));
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java
index 3b538f0244b..ed47c96d251 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java
@@ -69,6 +69,7 @@ public class GroupedMaxAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues = new BinaryBigArray();
return;
case BOOLEAN:
@@ -101,6 +102,7 @@ public class GroupedMaxAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
valuesSize += binaryValues.sizeOf();
break;
case BOOLEAN:
@@ -135,6 +137,7 @@ public class GroupedMaxAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues.ensureCapacity(groupCount);
return;
case BOOLEAN:
@@ -167,6 +170,7 @@ public class GroupedMaxAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(groupIds, arguments[0], mask);
return;
case BOOLEAN:
@@ -204,6 +208,7 @@ public class GroupedMaxAccumulator implements GroupedAccumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
updateBinaryValue(groupIds[i], argument.getBinary(i));
break;
case BOOLEAN:
@@ -240,6 +245,7 @@ public class GroupedMaxAccumulator implements GroupedAccumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
columnBuilder.writeBinary(binaryValues.get(groupId));
break;
case BOOLEAN:
@@ -275,6 +281,7 @@ public class GroupedMaxAccumulator implements GroupedAccumulator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
columnBuilder.writeBinary(binaryValues.get(groupId));
break;
case BOOLEAN:
@@ -311,6 +318,7 @@ public class GroupedMaxAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues.reset();
return;
case BOOLEAN:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java
index 7cfe93629ea..b9b25f62715 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java
@@ -96,6 +96,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
xBinaryValues = new BinaryBigArray();
break;
@@ -124,6 +125,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
yBinaryValues = new BinaryBigArray();
break;
@@ -157,6 +159,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
valuesSize += xBinaryValues.sizeOf();
break;
case BOOLEAN:
@@ -185,6 +188,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
valuesSize += yBinaryValues.sizeOf();
break;
case BOOLEAN:
@@ -220,6 +224,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
xBinaryValues.ensureCapacity(groupCount);
break;
case BOOLEAN:
@@ -247,6 +252,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
yBinaryValues.ensureCapacity(groupCount);
break;
case BOOLEAN:
@@ -282,6 +288,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
xBinaryValues.reset();
break;
@@ -310,6 +317,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
yBinaryValues.reset();
break;
@@ -341,6 +349,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
return;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
addBinaryInput(groupIds, arguments, mask);
return;
@@ -596,6 +605,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
columnBuilder.writeBinary(xBinaryValues.get(groupId));
break;
case BOOLEAN:
@@ -630,6 +640,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
xBinaryValues.set(groupId, xColumn.getBinary(xIndex));
break;
case BOOLEAN:
@@ -692,6 +703,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
byte[] values =
isX ? xBinaryValues.get(groupId).getValues() : yBinaryValues.get(groupId).getValues();
intToBytes(values.length, bytes, offset);
@@ -725,6 +737,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
return Double.BYTES;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
return Integer.BYTES
+ (isX
@@ -774,6 +787,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
Binary binaryMaxVal = new Binary(BytesUtils.subBytes(bytes, offset, length));
@@ -818,6 +832,7 @@ public abstract class GroupedMaxMinByBaseAccumulator implements GroupedAccumulat
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length)));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java
index 1c7206928ff..8fa9b4d8fa0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java
@@ -69,6 +69,7 @@ public class GroupedMinAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues = new BinaryBigArray();
return;
case BOOLEAN:
@@ -101,6 +102,7 @@ public class GroupedMinAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
valuesSize += binaryValues.sizeOf();
break;
case BOOLEAN:
@@ -135,6 +137,7 @@ public class GroupedMinAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues.ensureCapacity(groupCount);
return;
case BOOLEAN:
@@ -167,6 +170,7 @@ public class GroupedMinAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(groupIds, arguments[0], mask);
return;
case BOOLEAN:
@@ -203,6 +207,7 @@ public class GroupedMinAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
updateBinaryValue(groupIds[i], argument.getBinary(i));
break;
@@ -240,6 +245,7 @@ public class GroupedMinAccumulator implements GroupedAccumulator {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
columnBuilder.writeBinary(binaryValues.get(groupId));
break;
case BOOLEAN:
@@ -274,6 +280,7 @@ public class GroupedMinAccumulator implements GroupedAccumulator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
columnBuilder.writeBinary(binaryValues.get(groupId));
break;
@@ -311,6 +318,7 @@ public class GroupedMinAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
binaryValues.reset();
return;
case BOOLEAN:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java
index 697a7d3b44a..a5bf82c5f40 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java
@@ -95,6 +95,7 @@ public class GroupedModeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
addBinaryInput(groupIds, arguments[0], mask);
break;
default:
@@ -166,6 +167,7 @@ public class GroupedModeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
columnBuilder.writeBinary(maxEntry.getKey().getBinary());
break;
default:
@@ -270,6 +272,7 @@ public class GroupedModeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
bytes =
new byte
[offset
@@ -366,6 +369,7 @@ public class GroupedModeAccumulator implements GroupedAccumulator {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
for (int i = 0; i < size; i++) {
int length = BytesUtils.bytesToInt(bytes, offset);
offset += Integer.BYTES;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java
index 4734e7c3671..0c0325ebd0b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java
@@ -70,6 +70,7 @@ public class GroupedSumAccumulator implements GroupedAccumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
case BOOLEAN:
case DATE:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java
index ea8dead07bd..cb6123b636f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java
@@ -83,6 +83,7 @@ public class GroupedVarianceAccumulator implements GroupedAccumulator {
return;
case TEXT:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case DATE:
case STRING:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java
index 3c3fc6e616a..ae4975dff9a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java
@@ -70,6 +70,7 @@ public class WindowManagerFactory {
case BOOLEAN:
return new EqualBooleanWindowManager(eventWindowParameter, ascending);
case BLOB:
+ case OBJECT:
case STRING:
case TIMESTAMP:
case DATE:
@@ -97,6 +98,7 @@ public class WindowManagerFactory {
case STRING:
case BOOLEAN:
case BLOB:
+ case OBJECT:
case TEXT:
default:
throw new UnSupportedDataTypeException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java
index 9580f440e04..508c2dac2ae 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.SessionInfo;
+import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation;
import org.apache.iotdb.db.queryengine.plan.relational.function.arithmetic.AdditionResolver;
@@ -166,6 +167,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.Lo
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RTrim2ColumnTransformer;
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RTrimColumnTransformer;
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RadiansColumnTransformer;
+import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ReadObjectColumnTransformer;
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RegexpLike2ColumnTransformer;
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RegexpLikeColumnTransformer;
import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.Replace2ColumnTransformer;
@@ -212,6 +214,8 @@ import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.read.common.type.TypeEnum;
import org.apache.tsfile.utils.Binary;
+import javax.annotation.Nullable;
+
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
@@ -240,6 +244,7 @@ import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE;
import static org.apache.tsfile.read.common.type.FloatType.FLOAT;
import static org.apache.tsfile.read.common.type.IntType.INT32;
import static org.apache.tsfile.read.common.type.LongType.INT64;
+import static org.apache.tsfile.read.common.type.ObjectType.OBJECT;
import static org.apache.tsfile.read.common.type.StringType.STRING;
public class ColumnTransformerBuilder
@@ -1452,6 +1457,29 @@ public class ColumnTransformerBuilder
this.process(children.get(0), context),
this.process(children.get(1), context),
this.process(children.get(2), context));
+ } else if (TableBuiltinScalarFunction.READ_OBJECT
+ .getFunctionName()
+ .equalsIgnoreCase(functionName)) {
+ ColumnTransformer first = this.process(children.get(0), context);
+ if (children.size() == 1) {
+ return new ReadObjectColumnTransformer(OBJECT, first, context.fragmentInstanceContext);
+ } else if (children.size() == 2) {
+ return new ReadObjectColumnTransformer(
+ OBJECT,
+ ((LongLiteral) children.get(1)).getParsedValue(),
+ first,
+ context.fragmentInstanceContext);
+ } else {
+ long offset = ((LongLiteral) children.get(1)).getParsedValue();
+ long length = ((LongLiteral) children.get(2)).getParsedValue();
+ checkArgument(offset >= 0 && length >= 0);
+ return new ReadObjectColumnTransformer(
+ OBJECT,
+ ((LongLiteral) children.get(1)).getParsedValue(),
+ ((LongLiteral) children.get(2)).getParsedValue(),
+ first,
+ context.fragmentInstanceContext);
+ }
} else {
// user defined function
if (TableUDFUtils.isScalarFunction(functionName)) {
@@ -1916,6 +1944,8 @@ public class ColumnTransformerBuilder
private final Metadata metadata;
+ private final Optional<FragmentInstanceContext> fragmentInstanceContext;
+
public Context(
SessionInfo sessionInfo,
List<LeafColumnTransformer> leafList,
@@ -1926,7 +1956,8 @@ public class ColumnTransformerBuilder
List<TSDataType> inputDataTypes,
int originSize,
TypeProvider typeProvider,
- Metadata metadata) {
+ Metadata metadata,
+ @Nullable FragmentInstanceContext fragmentInstanceContext) {
this.sessionInfo = sessionInfo;
this.leafList = leafList;
this.inputLocations = inputLocations;
@@ -1937,6 +1968,7 @@ public class ColumnTransformerBuilder
this.originSize = originSize;
this.typeProvider = typeProvider;
this.metadata = metadata;
+ this.fragmentInstanceContext = Optional.ofNullable(fragmentInstanceContext);
}
public Type getType(SymbolReference symbolReference) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
index a2bd0dc4815..adf4efd8c5d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
@@ -41,6 +41,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.sink.DownStreamChannel
import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISinkHandle;
import org.apache.iotdb.db.queryengine.execution.exchange.sink.ShuffleSinkHandle;
import org.apache.iotdb.db.queryengine.execution.exchange.source.ISourceHandle;
+import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceManager;
import org.apache.iotdb.db.queryengine.execution.operator.EmptyDataOperator;
import org.apache.iotdb.db.queryengine.execution.operator.ExplainAnalyzeOperator;
@@ -1339,13 +1340,15 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
ColumnTransformerBuilder visitor = new ColumnTransformerBuilder();
+ FragmentInstanceContext fragmentInstanceContext =
+ context.getDriverContext().getFragmentInstanceContext();
ColumnTransformer filterOutputTransformer =
predicate
.map(
p -> {
ColumnTransformerBuilder.Context filterColumnTransformerContext =
new ColumnTransformerBuilder.Context(
- context.getDriverContext().getFragmentInstanceContext().getSessionInfo(),
+ fragmentInstanceContext.getSessionInfo(),
filterLeafColumnTransformerList,
inputLocations,
filterExpressionColumnTransformerMap,
@@ -1354,7 +1357,8 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
ImmutableList.of(),
0,
context.getTypeProvider(),
- metadata);
+ metadata,
+ fragmentInstanceContext);
return visitor.process(p, filterColumnTransformerContext);
})
@@ -1372,7 +1376,7 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
ColumnTransformerBuilder.Context projectColumnTransformerContext =
new ColumnTransformerBuilder.Context(
- context.getDriverContext().getFragmentInstanceContext().getSessionInfo(),
+ fragmentInstanceContext.getSessionInfo(),
projectLeafColumnTransformerList,
inputLocations,
projectExpressionColumnTransformerMap,
@@ -1381,7 +1385,8 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
filterOutputDataTypes,
inputLocations.size(),
context.getTypeProvider(),
- metadata);
+ metadata,
+ fragmentInstanceContext);
for (Expression expression : projectExpressions) {
projectOutputTransformerList.add(
@@ -2461,6 +2466,8 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
// In "count" we have to reuse filter operator per "next"
final List<LeafColumnTransformer> filterLeafColumnTransformerList = new ArrayList<>();
+ FragmentInstanceContext fragmentInstanceContext =
+ context.getDriverContext().getFragmentInstanceContext();
return new SchemaCountOperator<>(
node.getPlanNodeId(),
context
@@ -2482,10 +2489,7 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
.process(
node.getTagFuzzyPredicate(),
new ColumnTransformerBuilder.Context(
- context
- .getDriverContext()
- .getFragmentInstanceContext()
- .getSessionInfo(),
+ fragmentInstanceContext.getSessionInfo(),
filterLeafColumnTransformerList,
makeLayout(Collections.singletonList(node)),
new HashMap<>(),
@@ -2494,7 +2498,8 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution
ImmutableList.of(),
0,
context.getTypeProvider(),
- metadata)),
+ metadata,
+ fragmentInstanceContext)),
columnSchemaList,
database,
table)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
index 56ad6d59ba0..6ebfa6dcfdc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
@@ -112,6 +112,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode;
@@ -321,6 +322,7 @@ public enum PlanNodeType {
RELATIONAL_INSERT_ROW((short) 2001),
RELATIONAL_INSERT_ROWS((short) 2002),
RELATIONAL_DELETE_DATA((short) 2003),
+ OBJECT_FILE_NODE((short) 2004),
;
public static final int BYTES = Short.BYTES;
@@ -364,6 +366,8 @@ public enum PlanNodeType {
return RelationalInsertRowsNode.deserializeFromWAL(stream);
case 2003:
return RelationalDeleteDataNode.deserializeFromWAL(stream);
+ case 2004:
+ return ObjectNode.deserializeFromWAL(stream);
default:
throw new IllegalArgumentException("Invalid node type: " + nodeType);
}
@@ -390,6 +394,8 @@ public enum PlanNodeType {
return RelationalInsertRowsNode.deserializeFromWAL(buffer);
case 2003:
return RelationalDeleteDataNode.deserializeFromWAL(buffer);
+ case 2004:
+ return ObjectNode.deserialize(buffer);
default:
throw new IllegalArgumentException("Invalid node type: " + nodeType);
}
@@ -717,6 +723,8 @@ public enum PlanNodeType {
return RelationalInsertRowsNode.deserialize(buffer);
case 2003:
return RelationalDeleteDataNode.deserialize(buffer);
+ case 2004:
+ return ObjectNode.deserialize(buffer);
default:
throw new IllegalArgumentException("Invalid node type: " + nodeType);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
index ea669491a5f..4bebb692254 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
@@ -116,6 +116,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode;
@@ -633,6 +634,10 @@ public abstract class PlanVisitor<R, C> {
return visitPlan(node, context);
}
+ public R visitWriteObjectFile(ObjectNode node, C context) {
+ return visitPlan(node, context);
+ }
+
/////////////////////////////////////////////////////////////////////////////////////////////////
// Pipe Related Node
/////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
index edbc262971a..52386b7e077 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
@@ -94,10 +94,26 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
// proper positions.
protected List<Integer> range;
+ private Boolean shouldCheckTTL;
+
public InsertTabletNode(PlanNodeId id) {
super(id);
}
+ public boolean shouldCheckTTL() {
+ if (shouldCheckTTL != null) {
+ return shouldCheckTTL;
+ }
+ shouldCheckTTL = true;
+ for (MeasurementSchema measurementSchema : measurementSchemas) {
+ if (measurementSchema.getType() == TSDataType.OBJECT) {
+ shouldCheckTTL = false;
+ break;
+ }
+ }
+ return shouldCheckTTL;
+ }
+
@Override
public InsertNode mergeInsertNode(List<InsertNode> insertNodes) {
List<Integer> index = new ArrayList<>();
@@ -219,7 +235,6 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
final Map<IDeviceID, PartitionSplitInfo> deviceIDSplitInfoMap = collectSplitRanges();
final Map<TRegionReplicaSet, List<Integer>> splitMap =
splitByReplicaSet(deviceIDSplitInfoMap, analysis);
-
return doSplit(splitMap);
}
@@ -289,7 +304,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
return splitMap;
}
- private List<WritePlanNode> doSplit(Map<TRegionReplicaSet, List<Integer>> splitMap) {
+ protected List<WritePlanNode> doSplit(Map<TRegionReplicaSet, List<Integer>> splitMap) {
List<WritePlanNode> result = new ArrayList<>();
if (splitMap.size() == 1) {
@@ -326,7 +341,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
subTimes.length);
}
- private WritePlanNode generateOneSplit(Map.Entry<TRegionReplicaSet, List<Integer>> entry) {
+ protected WritePlanNode generateOneSplit(Map.Entry<TRegionReplicaSet, List<Integer>> entry) {
List<Integer> locs;
// generate a new times and values
locs = entry.getValue();
@@ -387,6 +402,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
values[i] = new Binary[rowSize];
break;
case FLOAT:
@@ -640,6 +656,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary[] binaryValues = (Binary[]) column;
for (int j = 0; j < rowCount; j++) {
if (binaryValues[j] != null && binaryValues[j].getValues() != null) {
@@ -692,6 +709,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
Binary[] binaryValues = (Binary[]) column;
for (int j = 0; j < rowCount; j++) {
if (binaryValues[j] != null && binaryValues[j].getValues() != null) {
@@ -833,6 +851,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary[] binaryValues = (Binary[]) column;
for (int j = start; j < end; j++) {
size += ReadWriteIOUtils.sizeToWrite(binaryValues[j]);
@@ -964,6 +983,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
Binary[] binaryValues = (Binary[]) column;
for (int j = start; j < end; j++) {
if (binaryValues[j] != null && binaryValues[j].getValues() != null) {
@@ -1127,6 +1147,7 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
if (!Arrays.equals((Binary[]) this.columns[i], (Binary[]) columns[i])) {
return false;
}
@@ -1199,6 +1220,8 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue {
Binary[] binaryValues = (Binary[]) columns[measurementIndex];
value = new TsPrimitiveType.TsBinary(binaryValues[lastIdx]);
break;
+ case OBJECT:
+ return null;
default:
throw new UnSupportedDataTypeException(
String.format(DATATYPE_UNSUPPORTED, dataTypes[measurementIndex]));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java
new file mode 100644
index 00000000000..ebe839bbc0c
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write;
+
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.commons.consensus.index.ProgressIndex;
+import org.apache.iotdb.commons.exception.ObjectFileNotExist;
+import org.apache.iotdb.commons.exception.runtime.SerializationRunTimeException;
+import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
+import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
+import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
+import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryType;
+import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
+import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils;
+import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
+
+import org.apache.tsfile.utils.PublicBAOS;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Optional;
+
+public class ObjectNode extends SearchNode implements WALEntryValue {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ObjectNode.class);
+
+ private final boolean isEOF;
+
+ private final long offset;
+
+ private byte[] content;
+
+ private String filePath;
+
+ private final int contentLength;
+
+ private TRegionReplicaSet dataRegionReplicaSet;
+
+ private boolean isGeneratedByRemoteConsensusLeader;
+
+ private Long time = null;
+
+ private String table = null;
+
+ public ObjectNode(boolean isEOF, long offset, byte[] content, String filePath) {
+ super(new PlanNodeId(""));
+ this.isEOF = isEOF;
+ this.offset = offset;
+ this.filePath = filePath;
+ this.content = content;
+ this.contentLength = content.length;
+ }
+
+ public ObjectNode(boolean isEOF, long offset, int contentLength, String filePath) {
+ super(new PlanNodeId(""));
+ this.isEOF = isEOF;
+ this.offset = offset;
+ this.filePath = filePath;
+ this.contentLength = contentLength;
+ }
+
+ public long getTimestamp() {
+ calculateTimeAndTableName();
+ return time;
+ }
+
+ public String getTable() {
+ calculateTimeAndTableName();
+ return table;
+ }
+
+ private void calculateTimeAndTableName() {
+ if (time != null && table != null) {
+ return;
+ }
+ File file = new File(filePath);
+ String fileName = new File(filePath).getName();
+ String timeStr = fileName.substring(0, fileName.length() - ".bin".length());
+ time = Long.parseLong(timeStr);
+ table = file.getParentFile().getParentFile().getParentFile().getParentFile().getName();
+ }
+
+ public boolean isEOF() {
+ return isEOF;
+ }
+
+ public byte[] getContent() {
+ return content;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ @Override
+ public void serializeToWAL(IWALByteBufferView buffer) {
+ buffer.putShort(getType().getNodeType());
+ buffer.putLong(searchIndex);
+ buffer.put((byte) (isEOF ? 1 : 0));
+ buffer.putLong(offset);
+ WALWriteUtils.write(filePath, buffer);
+ buffer.putInt(content.length);
+ }
+
+ @Override
+ public int serializedSize() {
+ return Short.BYTES
+ + Long.BYTES
+ + Byte.BYTES
+ + Long.BYTES
+ + Integer.BYTES
+ + ReadWriteIOUtils.sizeToWrite(filePath);
+ }
+
+ public static ObjectNode deserializeFromWAL(DataInputStream stream) throws IOException {
+ long searchIndex = stream.readLong();
+ boolean isEOF = stream.readByte() == 1;
+ long offset = stream.readLong();
+ String filePath = ReadWriteIOUtils.readString(stream);
+ int contentLength = stream.readInt();
+ ObjectNode objectNode = new ObjectNode(isEOF, offset, contentLength, filePath);
+ objectNode.setSearchIndex(searchIndex);
+ return objectNode;
+ }
+
+ public static ObjectNode deserializeFromWAL(ByteBuffer buffer) {
+ long searchIndex = buffer.getLong();
+ boolean isEOF = buffer.get() == 1;
+ long offset = buffer.getLong();
+ String filePath = ReadWriteIOUtils.readString(buffer);
+ Optional<File> objectFile = TierManager.getInstance().getAbsoluteObjectFilePath(filePath);
+ int contentLength = buffer.getInt();
+ byte[] contents = new byte[contentLength];
+ if (objectFile.isPresent()) {
+ try (RandomAccessFile raf = new RandomAccessFile(objectFile.get(), "r")) {
+ raf.seek(offset);
+ raf.read(contents);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ throw new ObjectFileNotExist(filePath);
+ }
+
+ ObjectNode objectNode = new ObjectNode(isEOF, offset, contents, filePath);
+ objectNode.setSearchIndex(searchIndex);
+ return objectNode;
+ }
+
+ public static ObjectNode deserialize(ByteBuffer byteBuffer) {
+ boolean isEoF = ReadWriteIOUtils.readBool(byteBuffer);
+ long offset = ReadWriteIOUtils.readLong(byteBuffer);
+ String filePath = ReadWriteIOUtils.readString(byteBuffer);
+ int contentLength = ReadWriteIOUtils.readInt(byteBuffer);
+ byte[] content = ReadWriteIOUtils.readBytes(byteBuffer, contentLength);
+ return new ObjectNode(isEoF, offset, content, filePath);
+ }
+
+ @Override
+ public SearchNode merge(List<SearchNode> searchNodes) {
+ if (searchNodes.size() == 1) {
+ return searchNodes.get(0);
+ }
+ throw new UnsupportedOperationException("Merge is not supported");
+ }
+
+ @Override
+ public ProgressIndex getProgressIndex() {
+ return null;
+ }
+
+ @Override
+ public void setProgressIndex(ProgressIndex progressIndex) {}
+
+ @Override
+ public List<WritePlanNode> splitByPartition(IAnalysis analysis) {
+ return null;
+ }
+
+ @Override
+ public TRegionReplicaSet getRegionReplicaSet() {
+ return dataRegionReplicaSet;
+ }
+
+ public void setDataRegionReplicaSet(TRegionReplicaSet dataRegionReplicaSet) {
+ this.dataRegionReplicaSet = dataRegionReplicaSet;
+ }
+
+ @Override
+ public List<PlanNode> getChildren() {
+ return null;
+ }
+
+ @Override
+ public void addChild(PlanNode child) {}
+
+ @Override
+ public PlanNode clone() {
+ return null;
+ }
+
+ @Override
+ public int allowedChildCount() {
+ return NO_CHILD_ALLOWED;
+ }
+
+ @Override
+ public List<String> getOutputColumnNames() {
+ return null;
+ }
+
+ @Override
+ protected void serializeAttributes(ByteBuffer byteBuffer) {
+ getType().serialize(byteBuffer);
+ ReadWriteIOUtils.write(isEOF, byteBuffer);
+ ReadWriteIOUtils.write(offset, byteBuffer);
+ ReadWriteIOUtils.write(filePath, byteBuffer);
+ ReadWriteIOUtils.write(contentLength, byteBuffer);
+ byteBuffer.put(content);
+ }
+
+ @Override
+ protected void serializeAttributes(DataOutputStream stream) throws IOException {
+ getType().serialize(stream);
+ ReadWriteIOUtils.write(isEOF, stream);
+ ReadWriteIOUtils.write(offset, stream);
+ ReadWriteIOUtils.write(filePath, stream);
+ ReadWriteIOUtils.write(contentLength, stream);
+ stream.write(content);
+ }
+
+ public ByteBuffer serialize() {
+ try (PublicBAOS byteArrayOutputStream = new PublicBAOS();
+ DataOutputStream stream = new DataOutputStream(byteArrayOutputStream)) {
+ ReadWriteIOUtils.write(WALEntryType.OBJECT_FILE_NODE.getCode(), stream);
+ ReadWriteIOUtils.write((long) TsFileProcessor.MEMTABLE_NOT_EXIST, stream);
+ ReadWriteIOUtils.write(getType().getNodeType(), stream);
+ byte[] contents = new byte[contentLength];
+ boolean readSuccess = false;
+ for (int i = 0; i < 2; i++) {
+ Optional<File> objectFile = TierManager.getInstance().getAbsoluteObjectFilePath(filePath);
+ if (objectFile.isPresent()) {
+ try {
+ readContentFromFile(objectFile.get(), contents);
+ readSuccess = true;
+ } catch (IOException e) {
+ LOGGER.error("Error when read object file {}.", objectFile.get(), e);
+ }
+ if (readSuccess) {
+ break;
+ }
+ }
+ Optional<File> objectTmpFile =
+ TierManager.getInstance().getAbsoluteObjectFilePath(filePath + ".tmp");
+ if (objectTmpFile.isPresent()) {
+ try {
+ readContentFromFile(objectTmpFile.get(), contents);
+ readSuccess = true;
+ } catch (IOException e) {
+ LOGGER.error("Error when read tmp object file {}.", objectTmpFile.get(), e);
+ }
+ if (readSuccess) {
+ break;
+ }
+ }
+ }
+ ReadWriteIOUtils.write(readSuccess && isEOF, stream);
+ ReadWriteIOUtils.write(offset, stream);
+ ReadWriteIOUtils.write(filePath, stream);
+ ReadWriteIOUtils.write(contentLength, stream);
+ stream.write(contents);
+ return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
+ } catch (IOException e) {
+ throw new SerializationRunTimeException(e);
+ }
+ }
+
+ private void readContentFromFile(File file, byte[] contents) throws IOException {
+ try (RandomAccessFile raf = new RandomAccessFile(file, "r")) {
+ raf.seek(offset);
+ raf.read(contents);
+ }
+ }
+
+ @Override
+ public PlanNodeType getType() {
+ return PlanNodeType.OBJECT_FILE_NODE;
+ }
+
+ @Override
+ public long getMemorySize() {
+ return contentLength;
+ }
+
+ @Override
+ public void markAsGeneratedByRemoteConsensusLeader() {
+ isGeneratedByRemoteConsensusLeader = true;
+ }
+
+ public boolean isGeneratedByRemoteConsensusLeader() {
+ return isGeneratedByRemoteConsensusLeader;
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitWriteObjectFile(this, context);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
index 6e5e8eeb598..63ca8d385ff 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
@@ -30,14 +30,18 @@ import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.IDeviceID.Factory;
import org.apache.tsfile.read.TimeValuePair;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.utils.BytesUtils;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.schema.MeasurementSchema;
@@ -46,10 +50,12 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
public class RelationalInsertTabletNode extends InsertTabletNode {
@@ -58,6 +64,8 @@ public class RelationalInsertTabletNode extends InsertTabletNode {
private boolean singleDevice;
+ private Object[] convertedColumns;
+
public RelationalInsertTabletNode(
PlanNodeId id,
PartialPath devicePath,
@@ -108,6 +116,16 @@ public class RelationalInsertTabletNode extends InsertTabletNode {
this.singleDevice = true;
}
+ public List<Binary[]> getObjectColumns() {
+ List<Binary[]> objectColumns = new ArrayList<>();
+ for (int i = 0; i < columns.length; i++) {
+ if (dataTypes[i] == TSDataType.OBJECT) {
+ objectColumns.add((Binary[]) columns[i]);
+ }
+ }
+ return objectColumns;
+ }
+
@Override
public IDeviceID getDeviceID(int rowIdx) {
if (singleDevice) {
@@ -372,4 +390,60 @@ public class RelationalInsertTabletNode extends InsertTabletNode {
startOffset = endOffset;
}
}
+
+ @Override
+ protected List<WritePlanNode> doSplit(Map<TRegionReplicaSet, List<Integer>> splitMap) {
+ List<WritePlanNode> result = new ArrayList<>();
+
+ if (splitMap.size() == 1) {
+ final Entry<TRegionReplicaSet, List<Integer>> entry = splitMap.entrySet().iterator().next();
+ if (entry.getValue().size() == 2) {
+ // Avoid using system arraycopy when there is no need to split
+ setRange(entry.getValue());
+ setDataRegionReplicaSet(entry.getKey());
+ for (int i = 0; i < columns.length; i++) {
+ if (dataTypes[i] == TSDataType.OBJECT) {
+ for (int j = 0; j < times.length; j++) {
+ byte[] binary = ((Binary[]) columns[i])[j].getValues();
+ ByteBuffer buffer = ByteBuffer.wrap(binary);
+ boolean isEoF = buffer.get() == 1;
+ long offset = buffer.getLong();
+ byte[] content = ReadWriteIOUtils.readBytes(buffer, buffer.remaining());
+ String relativePath =
+ TsFileNameGenerator.generateObjectFilePath(
+ dataRegionReplicaSet.regionId.getId(), times[j], getDeviceID(j));
+ ObjectNode objectNode = new ObjectNode(isEoF, offset, content, relativePath);
+ objectNode.setDataRegionReplicaSet(entry.getKey());
+ result.add(objectNode);
+ if (isEoF) {
+ byte[] filePathBytes = relativePath.getBytes(StandardCharsets.UTF_8);
+ byte[] valueBytes = new byte[filePathBytes.length + Long.BYTES];
+ System.arraycopy(
+ BytesUtils.longToBytes(offset + content.length), 0, valueBytes, 0, Long.BYTES);
+ System.arraycopy(filePathBytes, 0, valueBytes, Long.BYTES, filePathBytes.length);
+ ((Binary[]) columns[i])[j] = new Binary(valueBytes);
+ } else {
+ ((Binary[]) columns[i])[j] = null;
+ if (bitMaps == null) {
+ bitMaps = new BitMap[columns.length];
+ }
+ if (bitMaps[i] == null) {
+ bitMaps[i] = new BitMap(rowCount);
+ }
+ bitMaps[i].mark(j);
+ }
+ }
+ }
+ }
+ result.add(this);
+ return result;
+ }
+ }
+
+ for (Map.Entry<TRegionReplicaSet, List<Integer>> entry : splitMap.entrySet()) {
+ // TODO: add ObjectNode for split
+ result.add(generateOneSplit(entry));
+ }
+ return result;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
index c15071be697..ec41311635e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
@@ -66,6 +66,7 @@ import org.apache.iotdb.udf.api.relational.ScalarFunction;
import org.apache.iotdb.udf.api.relational.TableFunction;
import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.read.common.type.ObjectType;
import org.apache.tsfile.read.common.type.StringType;
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.read.common.type.TypeFactory;
@@ -1078,6 +1079,20 @@ public class TableMetadataImpl implements Metadata {
functionName));
}
return BLOB;
+ } else if (TableBuiltinScalarFunction.READ_OBJECT
+ .getFunctionName()
+ .equalsIgnoreCase(functionName)) {
+ if (argumentTypes.isEmpty()
+ || argumentTypes.size() > 3
+ || !isObjectType(argumentTypes.get(0))
+ || (argumentTypes.size() >= 2 && !isIntegerNumber(argumentTypes.get(1)))
+ || (argumentTypes.size() >= 3 && !isIntegerNumber(argumentTypes.get(2)))) {
+ throw new SemanticException(
+ "Scalar function "
+ + functionName.toLowerCase(Locale.ENGLISH)
+ + " must have at 1~3 arguments, and first argument must be OBJECT type, other arguments must be int32 or int64 type");
+ }
+ return BLOB;
}
// builtin aggregation function
@@ -1516,6 +1531,10 @@ public class TableMetadataImpl implements Metadata {
return TEXT.equals(type) || StringType.STRING.equals(type);
}
+ public static boolean isObjectType(Type type) {
+ return ObjectType.OBJECT.equals(type);
+ }
+
public static boolean isBlobType(Type type) {
return BLOB.equals(type);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java
index 9b1487b0a48..ed5bc0d3719 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java
@@ -226,7 +226,8 @@ public class DeleteDevice extends AbstractTraverseDevice {
ImmutableList.of(),
0,
mockTypeProvider,
- metadata))
+ metadata,
+ null))
: null;
return new DeviceBlackListConstructor(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java
index 6f2aaef76ab..5719a4555ff 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java
@@ -35,6 +35,7 @@ import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE;
import static org.apache.tsfile.read.common.type.FloatType.FLOAT;
import static org.apache.tsfile.read.common.type.IntType.INT32;
import static org.apache.tsfile.read.common.type.LongType.INT64;
+import static org.apache.tsfile.read.common.type.ObjectType.OBJECT;
import static org.apache.tsfile.read.common.type.StringType.STRING;
import static org.apache.tsfile.read.common.type.TimestampType.TIMESTAMP;
import static org.apache.tsfile.read.common.type.UnknownType.UNKNOWN;
@@ -52,6 +53,7 @@ public class InternalTypeManager implements TypeManager {
types.put(new TypeSignature(TypeEnum.TEXT.name().toLowerCase(Locale.ENGLISH)), TEXT);
types.put(new TypeSignature(TypeEnum.STRING.name().toLowerCase(Locale.ENGLISH)), STRING);
types.put(new TypeSignature(TypeEnum.BLOB.name().toLowerCase(Locale.ENGLISH)), BLOB);
+ types.put(new TypeSignature(TypeEnum.OBJECT.name().toLowerCase(Locale.ENGLISH)), OBJECT);
types.put(new TypeSignature(TypeEnum.DATE.name().toLowerCase(Locale.ENGLISH)), DATE);
types.put(new TypeSignature(TypeEnum.TIMESTAMP.name().toLowerCase(Locale.ENGLISH)), TIMESTAMP);
types.put(new TypeSignature(TypeEnum.UNKNOWN.name().toLowerCase(Locale.ENGLISH)), UNKNOWN);
@@ -105,6 +107,8 @@ public class InternalTypeManager implements TypeManager {
return TSDataType.BLOB;
case STRING:
return TSDataType.STRING;
+ case OBJECT:
+ return TSDataType.OBJECT;
default:
throw new IllegalArgumentException();
}
@@ -132,6 +136,8 @@ public class InternalTypeManager implements TypeManager {
return TIMESTAMP;
case BLOB:
return BLOB;
+ case OBJECT:
+ return OBJECT;
case STRING:
return STRING;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java
index 0cc31381072..fe951d1e4ce 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java
@@ -28,6 +28,7 @@ import org.apache.tsfile.read.common.type.BinaryType;
import org.apache.tsfile.read.common.type.BlobType;
import org.apache.tsfile.read.common.type.DoubleType;
import org.apache.tsfile.read.common.type.FloatType;
+import org.apache.tsfile.read.common.type.ObjectType;
import org.apache.tsfile.read.common.type.RowType;
import org.apache.tsfile.read.common.type.StringType;
import org.apache.tsfile.read.common.type.TimestampType;
@@ -116,6 +117,8 @@ public class TypeUtil {
return TimestampType.TIMESTAMP;
case DATE:
return DATE;
+ case OBJECT:
+ return ObjectType.OBJECT;
case ROW:
return RowType.anonymous(subTypes);
default:
@@ -123,7 +126,6 @@ public class TypeUtil {
}
}
- // TODO move these methods into each Type to avoid branch miss
public static boolean isFlatVariableWidth(Type type) {
switch (type.getTypeEnum()) {
case BOOLEAN:
@@ -137,6 +139,7 @@ public class TypeUtil {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
return true;
default:
throw new UnsupportedOperationException();
@@ -160,6 +163,7 @@ public class TypeUtil {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
return 16;
default:
throw new UnsupportedOperationException();
@@ -179,6 +183,7 @@ public class TypeUtil {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
return column.isNull(position) ? 0 : column.getBinary(position).getLength();
default:
throw new UnsupportedOperationException();
@@ -198,6 +203,7 @@ public class TypeUtil {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
int result = 0;
for (int i = 0; i < position.length; i++) {
if (!column.isNull(i)) {
@@ -237,6 +243,7 @@ public class TypeUtil {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
int length = bytesToInt(fixedChunk, fixedOffset);
byte[] result = new byte[length];
if (length <= 12) {
@@ -282,6 +289,7 @@ public class TypeUtil {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
byte[] value = column.getBinary(position).getValues();
intToBytes(value.length, fixedChunk, fixedOffset);
if (value.length <= 12) {
@@ -320,6 +328,7 @@ public class TypeUtil {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
int leftLength = bytesToInt(fixedChunk, fixedOffset);
byte[] leftValue = new byte[leftLength];
byte[] rightValue = column.getBinary(position).getValues();
@@ -414,6 +423,7 @@ public class TypeUtil {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
int length = bytesToInt(fixedChunk, fixedOffset);
byte[] values = new byte[length];
@@ -490,6 +500,7 @@ public class TypeUtil {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
return XxHash64.hash(column.getBinary(position).getValues());
default:
throw new UnsupportedOperationException();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java
new file mode 100644
index 00000000000..84a094e39fe
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar;
+
+import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
+import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
+import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer;
+import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer;
+import org.apache.iotdb.db.utils.ObjectTypeUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.block.column.Column;
+import org.apache.tsfile.block.column.ColumnBuilder;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.read.common.type.Type;
+import org.apache.tsfile.utils.Binary;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
+import java.util.Optional;
+
+public class ReadObjectColumnTransformer extends UnaryColumnTransformer {
+
+ private final Optional<FragmentInstanceContext> fragmentInstanceContext;
+ private long offset = 0;
+ private long length = -1;
+
+ public ReadObjectColumnTransformer(
+ Type type,
+ ColumnTransformer childColumnTransformer,
+ Optional<FragmentInstanceContext> fragmentInstanceContext) {
+ super(type, childColumnTransformer);
+ this.fragmentInstanceContext = fragmentInstanceContext;
+ }
+
+ public ReadObjectColumnTransformer(
+ Type type,
+ long offset,
+ ColumnTransformer childColumnTransformer,
+ Optional<FragmentInstanceContext> fragmentInstanceContext) {
+ super(type, childColumnTransformer);
+ this.offset = offset;
+ this.fragmentInstanceContext = fragmentInstanceContext;
+ }
+
+ public ReadObjectColumnTransformer(
+ Type type,
+ long offset,
+ long length,
+ ColumnTransformer childColumnTransformer,
+ Optional<FragmentInstanceContext> fragmentInstanceContext) {
+ super(type, childColumnTransformer);
+ this.offset = offset;
+ this.length = length;
+ this.fragmentInstanceContext = fragmentInstanceContext;
+ }
+
+ @Override
+ protected void doTransform(Column column, ColumnBuilder columnBuilder) {
+ for (int i = 0, n = column.getPositionCount(); i < n; i++) {
+ if (!column.isNull(i)) {
+ transform(column, columnBuilder, i);
+ } else {
+ columnBuilder.appendNull();
+ }
+ }
+ }
+
+ @Override
+ protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) {
+ for (int i = 0, n = column.getPositionCount(); i < n; i++) {
+ if (selection[i] && !column.isNull(i)) {
+ transform(column, columnBuilder, i);
+ } else {
+ columnBuilder.appendNull();
+ }
+ }
+ }
+
+ private void transform(Column column, ColumnBuilder columnBuilder, int i) {
+ // BinaryColumn.getDataType() returns TSDataType.TEXT
+ if (TSDataType.TEXT == column.getDataType()) {
+ Binary binary = column.getBinary(i);
+ columnBuilder.writeBinary(readObject(binary));
+ }
+ }
+
+ private Binary readObject(Binary binary) {
+ File file = ObjectTypeUtils.getObjectPathFromBinary(binary);
+ long fileSize = file.length();
+ if (offset >= fileSize) {
+ throw new UnsupportedOperationException("offset is greater than object size");
+ }
+ long actualReadSize = Math.min(length < 0 ? fileSize : length, fileSize - offset);
+ if (actualReadSize > Integer.MAX_VALUE) {
+ throw new UnsupportedOperationException("Read object size is too large (size > 2G)");
+ }
+ fragmentInstanceContext.ifPresent(
+ context -> context.getMemoryReservationContext().reserveMemoryCumulatively(actualReadSize));
+ byte[] bytes = new byte[(int) actualReadSize];
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ try (FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) {
+ fileChannel.read(buffer);
+ } catch (IOException e) {
+ throw new IoTDBRuntimeException(e, TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ return new Binary(bytes);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java
index 32125c110ef..14643a7b6ac 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java
@@ -92,6 +92,7 @@ public class ConstantInputReader implements LayerReader {
cachedColumns[0] = new RunLengthEncodedColumn(booleanColumn, count);
break;
case BLOB:
+ case OBJECT:
case STRING:
case TIMESTAMP:
case DATE:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java
index fcbb30b7ca1..59b3dbbc8b6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java
@@ -59,6 +59,7 @@ public class ArithmeticNegationTransformer extends UnaryTransformer {
case TEXT:
case TIMESTAMP:
case BLOB:
+ case OBJECT:
case BOOLEAN:
case STRING:
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java
index 5bdfa8f96a7..09cc6e76ad3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java
@@ -88,6 +88,7 @@ public class InTransformer extends UnaryTransformer {
stringSet = values;
break;
case BLOB:
+ case OBJECT:
default:
throw new UnsupportedOperationException("unsupported data type: " + layerReaderDataType);
}
@@ -124,6 +125,7 @@ public class InTransformer extends UnaryTransformer {
transformBinary(columns, builder);
return;
case BLOB:
+ case OBJECT:
default:
throw new QueryProcessException("unsupported data type: " + layerReaderDataType);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java
index 11cf952e0b4..195acb6de85 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java
@@ -68,6 +68,7 @@ public class CastFunctionTransformer extends UnaryTransformer {
case TEXT:
return castBinaries(columns);
case BLOB:
+ case OBJECT:
case STRING:
case TIMESTAMP:
case DATE:
@@ -139,6 +140,7 @@ public class CastFunctionTransformer extends UnaryTransformer {
break;
case STRING:
case BLOB:
+ case OBJECT:
case TIMESTAMP:
case DATE:
case INT32:
@@ -213,6 +215,7 @@ public class CastFunctionTransformer extends UnaryTransformer {
}
break;
case BLOB:
+ case OBJECT:
case STRING:
case DATE:
case TIMESTAMP:
@@ -288,6 +291,7 @@ public class CastFunctionTransformer extends UnaryTransformer {
}
break;
case BLOB:
+ case OBJECT:
case STRING:
case TIMESTAMP:
case DATE:
@@ -363,6 +367,7 @@ public class CastFunctionTransformer extends UnaryTransformer {
}
break;
case BLOB:
+ case OBJECT:
case STRING:
case TIMESTAMP:
case DATE:
@@ -439,6 +444,7 @@ public class CastFunctionTransformer extends UnaryTransformer {
break;
case STRING:
case BLOB:
+ case OBJECT:
case DATE:
case TIMESTAMP:
case BOOLEAN:
@@ -521,6 +527,7 @@ public class CastFunctionTransformer extends UnaryTransformer {
case DATE:
case STRING:
case BLOB:
+ case OBJECT:
case TEXT:
default:
throw new UnsupportedOperationException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java
index ee7ba10e60e..03081f9206f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java
@@ -62,6 +62,7 @@ public class DiffFunctionTransformer extends UnaryTransformer {
transformDouble(columns, builder);
return;
case BLOB:
+ case OBJECT:
case TEXT:
case DATE:
case STRING:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java
index a4ddff58bab..47e42618077 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java
@@ -67,6 +67,7 @@ public class RoundFunctionTransformer extends UnaryTransformer {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
default:
throw new UnsupportedOperationException(
String.format("Unsupported source dataType: %s", layerReaderDataType));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java
index c05d7adfecc..015904399c2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java
@@ -78,6 +78,7 @@ public class TransformUtils {
return new BooleanColumn(1, Optional.empty(), new boolean[] {(boolean) value});
case STRING:
case BLOB:
+ case OBJECT:
case DATE:
case TIMESTAMP:
default:
@@ -158,6 +159,7 @@ public class TransformUtils {
case TIMESTAMP:
case DATE:
case BLOB:
+ case OBJECT:
case STRING:
default:
throw new UnsupportedOperationException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java
index 0eedb86a31a..e8c679e2bb5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java
@@ -50,6 +50,7 @@ public class TypeUtils {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return new BinaryColumnBuilder(null, count);
default:
throw new UnSupportedDataTypeException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java
index 6ec6537de36..3bacd1ac5e7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java
@@ -107,6 +107,7 @@ public class SerializableRowList implements SerializableList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
rowLength +=
MIN_OBJECT_HEADER_SIZE + MIN_ARRAY_HEADER_SIZE + byteArrayLengthForMemoryControl;
break;
@@ -214,6 +215,7 @@ public class SerializableRowList implements SerializableList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
row[i] = block[i].getBinary(offset);
break;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java
index 5f76cd46c3d..57ac89f073a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java
@@ -92,6 +92,7 @@ public class SerializableTVList implements SerializableList {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
rowLength +=
MIN_OBJECT_HEADER_SIZE
+ MIN_ARRAY_HEADER_SIZE
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
index 9b125f4f111..5f6486f8be4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
@@ -1549,7 +1549,8 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
ImmutableList.of(),
0,
mockTypeProvider,
- metadata))
+ metadata,
+ null))
: null;
final List<TSDataType> filterOutputDataTypes =
@@ -1576,7 +1577,8 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
filterOutputDataTypes,
inputLocations.size(),
mockTypeProvider,
- metadata);
+ metadata,
+ null);
final List<String> attributeNames =
updateNode.getAssignments().stream()
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
index 8c3ac35c5d2..987788a69fc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java
@@ -76,6 +76,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALException;
import org.apache.iotdb.db.storageengine.dataregion.wal.recover.WALRecoverManager;
import org.apache.iotdb.db.storageengine.load.LoadTsFileManager;
import org.apache.iotdb.db.storageengine.load.limiter.LoadTsFileRateLimiter;
+import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.db.utils.ThreadUtils;
import org.apache.iotdb.rpc.RpcUtils;
@@ -83,6 +84,8 @@ import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.exception.write.PageException;
import org.apache.tsfile.external.commons.io.FileUtils;
+import org.apache.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.tsfile.fileSystem.fsFactory.FSFactory;
import org.apache.tsfile.utils.FilePathUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,6 +93,8 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -106,6 +111,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -119,6 +125,8 @@ public class StorageEngine implements IService {
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
private static final WritingMetrics WRITING_METRICS = WritingMetrics.getInstance();
+ private final FSFactory fsFactory = FSFactoryProducer.getFSFactory();
+
/**
* a folder (system/databases/ by default) that persist system info. Each database will have a
* subfolder under the systemDir.
@@ -156,6 +164,8 @@ public class StorageEngine implements IService {
private final LoadTsFileManager loadTsFileManager = new LoadTsFileManager();
+ public final AtomicLong objectFileId = new AtomicLong(0);
+
private StorageEngine() {}
public static StorageEngine getInstance() {
@@ -226,6 +236,8 @@ public class StorageEngine implements IService {
}
private void asyncRecover(List<Future<Void>> futures) {
+ checkObjectFiles();
+
Map<String, List<DataRegionId>> localDataRegionInfo = getLocalDataRegionInfo();
localDataRegionInfo.values().forEach(list -> recoverDataRegionNum += list.size());
readyDataRegionNum = new AtomicInteger(0);
@@ -1066,6 +1078,32 @@ public class StorageEngine implements IService {
systemDir + File.separator + dataBaseName, dataRegionId);
}
+ private void checkObjectFiles() {
+ List<String> folders = TierManager.getInstance().getAllObjectFileFolders();
+ for (String baseDir : folders) {
+ File fileFolder = fsFactory.getFile(baseDir);
+ try (Stream<Path> paths = Files.walk(fileFolder.toPath())) {
+ paths
+ .filter(Files::isRegularFile)
+ .filter(
+ path -> {
+ String name = path.getFileName().toString();
+ return name.endsWith(".bin.back");
+ })
+ .forEach(
+ path -> {
+ try {
+ Files.delete(path);
+ } catch (IOException e) {
+ LOGGER.error("Failed to delete: {} -> {}", path, e.getMessage());
+ }
+ });
+ } catch (IOException e) {
+ LOGGER.error("Failed to check Object Files: {}", e.getMessage());
+ }
+ }
+ }
+
public Runnable executeCompactFileTimeIndexCache() {
return () -> {
if (!isReadyForNonReadWriteFunctions()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 9293f9867b6..8e398919631 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -84,6 +84,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNod
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.LastCacheLoadStrategy;
@@ -157,6 +158,7 @@ import org.apache.iotdb.db.utils.CommonUtils;
import org.apache.iotdb.db.utils.DateTimeUtils;
import org.apache.iotdb.db.utils.EncryptDBUtils;
import org.apache.iotdb.db.utils.ModificationUtils;
+import org.apache.iotdb.db.utils.ObjectWriter;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -186,6 +188,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -1338,7 +1341,10 @@ public class DataRegion implements IDataRegionForQuery {
InsertTabletNode insertTabletNode, TSStatus[] results, long[] infoForMetrics)
throws OutOfTTLException {
boolean noFailure;
- int loc = insertTabletNode.checkTTL(results, getTTL(insertTabletNode));
+ int loc =
+ insertTabletNode.shouldCheckTTL()
+ ? insertTabletNode.checkTTL(results, getTTL(insertTabletNode))
+ : 0;
noFailure = loc == 0;
List<Pair<IDeviceID, Integer>> deviceEndOffsetPairs =
insertTabletNode.splitByDevice(loc, insertTabletNode.getRowCount());
@@ -1946,6 +1952,7 @@ public class DataRegion implements IDataRegionForQuery {
}
});
deleteAllSGFolders(TierManager.getInstance().getAllFilesFolders());
+ deleteAllObjectFiles(TierManager.getInstance().getAllObjectFileFolders());
this.workSequenceTsFileProcessors.clear();
this.workUnsequenceTsFileProcessors.clear();
this.tsFileManager.clear();
@@ -1983,6 +1990,23 @@ public class DataRegion implements IDataRegionForQuery {
}
}
+ private void deleteAllObjectFiles(List<String> folders) {
+ for (String objectFolder : folders) {
+ File dataRegionObjectFolder = fsFactory.getFile(objectFolder, dataRegionId);
+ if (FSUtils.getFSType(dataRegionObjectFolder) != FSType.LOCAL) {
+ try {
+ fsFactory.deleteDirectory(dataRegionObjectFolder.getPath());
+ } catch (IOException e) {
+ logger.error("Fail to delete data region object folder {}", dataRegionObjectFolder);
+ }
+ } else {
+ if (dataRegionObjectFolder.exists()) {
+ org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(dataRegionObjectFolder);
+ }
+ }
+ }
+ }
+
public void timedFlushSeqMemTable() {
int count = 0;
writeLock("timedFlushSeqMemTable");
@@ -3540,6 +3564,43 @@ public class DataRegion implements IDataRegionForQuery {
}
}
+ public void writeObject(ObjectNode objectNode) throws Exception {
+ writeLock("writeObject");
+ try {
+ String relativeTmpPathString = objectNode.getFilePath() + ".tmp";
+ String objectFileDir = TierManager.getInstance().getNextFolderForObjectFile();
+ File objectTmpFile =
+ FSFactoryProducer.getFSFactory().getFile(objectFileDir, relativeTmpPathString);
+ try (ObjectWriter writer = new ObjectWriter(objectTmpFile)) {
+ writer.write(
+ objectNode.isGeneratedByRemoteConsensusLeader(),
+ objectNode.getOffset(),
+ objectNode.getContent());
+ }
+ if (objectNode.isEOF()) {
+ File objectFile =
+ FSFactoryProducer.getFSFactory().getFile(objectFileDir, objectNode.getFilePath());
+ if (objectFile.exists()) {
+ String relativeBackPathString = objectNode.getFilePath() + ".back";
+ File objectBackFile =
+ FSFactoryProducer.getFSFactory().getFile(objectFileDir, relativeBackPathString);
+ Files.move(
+ objectFile.toPath(), objectBackFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ Files.move(
+ objectTmpFile.toPath(), objectFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ Files.delete(objectBackFile.toPath());
+ } else {
+ Files.move(
+ objectTmpFile.toPath(), objectFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+ getWALNode()
+ .ifPresent(walNode -> walNode.log(TsFileProcessor.MEMTABLE_NOT_EXIST, objectNode));
+ } finally {
+ writeUnlock();
+ }
+ }
+
/**
* Load a new tsfile to unsequence dir.
*
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
index 91184aaec82..3a5695a7376 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
@@ -181,8 +181,7 @@ public class FastCompactionPerformer
sortedSourceFiles.addAll(unseqFiles);
boolean isTreeModel = !isAligned || device.getTableName().startsWith("root.");
long ttl = deviceIterator.getTTLForCurrentDevice();
- sortedSourceFiles.removeIf(
- x -> x.definitelyNotContains(device) || !x.isDeviceAlive(device, ttl));
+ sortedSourceFiles.removeIf(x -> x.definitelyNotContains(device));
// checked above
//noinspection OptionalGetWithoutIsPresent
sortedSourceFiles.sort(Comparator.comparingLong(x -> x.getStartTime(device).get()));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
index 41d859924f0..290466ffeb6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.Compacti
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICrossCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer;
@@ -192,6 +193,11 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
performer.setTargetFiles(targetTsfileResourceList);
performer.setSummary(summary);
performer.perform();
+ if (performer instanceof ReadPointCompactionPerformer) {
+ for (TsFileResource resource : getAllSourceTsFiles()) {
+ CompactionUtils.removeDeletedObjectFiles(resource);
+ }
+ }
CompactionUtils.updateProgressIndexAndMark(
targetTsfileResourceList, selectedSequenceFiles, selectedUnsequenceFiles);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index c61d2275ac1..034e4eb48b8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.IInnerCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer;
@@ -389,6 +390,11 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
performer.setTargetFiles(filesView.targetFilesInPerformer);
performer.setSummary(summary);
performer.perform();
+ if (performer instanceof ReadPointCompactionPerformer) {
+ for (TsFileResource resource : filesView.sourceFilesInCompactionPerformer) {
+ CompactionUtils.removeDeletedObjectFiles(resource);
+ }
+ }
prepareTargetFiles();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java
index 0967089ecb3..7543d56ce80 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;
+import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException;
@@ -209,7 +210,7 @@ public class SettleCompactionTask extends InnerSpaceCompactionTask {
return isSuccess;
}
- public boolean settleWithFullyDirtyFiles() {
+ public boolean settleWithFullyDirtyFiles() throws IllegalPathException, IOException {
if (fullyDirtyFiles.isEmpty()) {
return true;
}
@@ -218,6 +219,8 @@ public class SettleCompactionTask extends InnerSpaceCompactionTask {
if (recoverMemoryStatus) {
tsFileManager.remove(resource, resource.isSeq());
}
+ CompactionUtils.removeDeletedObjectFiles(resource);
+
boolean res = deleteTsFileOnDisk(resource);
if (res) {
fullyDeletedSuccessNum++;
@@ -293,7 +296,7 @@ public class SettleCompactionTask extends InnerSpaceCompactionTask {
}
}
- public void recoverFullyDirtyFiles() {
+ public void recoverFullyDirtyFiles() throws IllegalPathException, IOException {
if (!settleWithFullyDirtyFiles()) {
throw new CompactionRecoverException("Failed to delete fully_dirty source file.");
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
index 4f58e8fc132..1668355db31 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
@@ -31,6 +31,8 @@ import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.reader.CompactionAlignedChunkReader;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.reader.CompactionChunkReader;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate;
import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch;
@@ -42,21 +44,33 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEnt
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex;
import org.apache.iotdb.db.utils.ModificationUtils;
+import org.apache.iotdb.db.utils.ObjectTypeUtils;
import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.metrics.utils.SystemMetric;
import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.header.PageHeader;
+import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
import org.apache.tsfile.file.metadata.ChunkMetadata;
+import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.tsfile.read.TimeValuePair;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.common.Chunk;
import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.reader.IPointReader;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.write.writer.TsFileIOWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -497,4 +511,158 @@ public class CompactionUtils {
new TimeRange(Long.MIN_VALUE, timeLowerBound));
}
}
+
+ public static void removeDeletedObjectFiles(TsFileResource resource)
+ throws IOException, IllegalPathException {
+ try (MultiTsFileDeviceIterator deviceIterator =
+ new MultiTsFileDeviceIterator(Collections.singletonList(resource))) {
+ while (deviceIterator.hasNextDevice()) {
+ deviceIterator.nextDevice();
+ deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries();
+ }
+ }
+ }
+
+ public static void removeDeletedObjectFiles(
+ TsFileSequenceReader reader,
+ List<AbstractAlignedChunkMetadata> alignedChunkMetadataList,
+ List<ModEntry> timeMods,
+ List<List<ModEntry>> valueMods)
+ throws IOException {
+ if (alignedChunkMetadataList.isEmpty()) {
+ return;
+ }
+ List<Integer> objectColumnIndexList = new ArrayList<>();
+ List<List<ModEntry>> objectDeletionIntervalList = new ArrayList<>();
+ boolean objectColumnHasDeletion = false;
+
+ TSDataType[] dataTypes = new TSDataType[valueMods.size()];
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) {
+ boolean hasNull = false;
+ for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) {
+ if (dataTypes[i] != null) {
+ continue;
+ }
+ IChunkMetadata chunkMetadata = alignedChunkMetadata.getValueChunkMetadataList().get(i);
+ if (chunkMetadata == null) {
+ hasNull = true;
+ continue;
+ }
+ dataTypes[i] = chunkMetadata.getDataType();
+ if (dataTypes[i] == TSDataType.OBJECT) {
+ objectColumnIndexList.add(i);
+ List<ModEntry> deletionInterval = ModificationUtils.sortAndMerge(valueMods.get(i));
+ objectColumnHasDeletion |= (!deletionInterval.isEmpty() || !timeMods.isEmpty());
+ objectDeletionIntervalList.add(deletionInterval);
+ }
+ }
+ if (!hasNull) {
+ break;
+ }
+ }
+ if (!objectColumnHasDeletion) {
+ return;
+ }
+ int[] deletionCursors = new int[objectColumnIndexList.size() + 1];
+ List<ModEntry> timeDeletionIntervalList = ModificationUtils.sortAndMerge(timeMods);
+ for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) {
+ CompactionUtils.removeDeletedObjectFiles(
+ reader,
+ alignedChunkMetadata,
+ objectColumnIndexList,
+ timeDeletionIntervalList,
+ objectDeletionIntervalList,
+ deletionCursors);
+ }
+ }
+
+ private static void removeDeletedObjectFiles(
+ TsFileSequenceReader reader,
+ AbstractAlignedChunkMetadata alignedChunkMetadata,
+ List<Integer> objectColumnIndexList,
+ List<ModEntry> timeDeletions,
+ List<List<ModEntry>> objectDeletions,
+ int[] deletionCursors)
+ throws IOException {
+ Chunk timeChunk =
+ reader.readMemChunk((ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata());
+ CompactionChunkReader compactionChunkReader = new CompactionChunkReader(timeChunk);
+ List<Pair<PageHeader, ByteBuffer>> timePages =
+ compactionChunkReader.readPageDataWithoutUncompressing();
+
+ List<Chunk> valueChunks = new ArrayList<>();
+ List<List<Pair<PageHeader, ByteBuffer>>> valuePages = new ArrayList<>();
+
+ for (int i = 0; i < objectColumnIndexList.size(); i++) {
+ int idxInAlignedChunkMetadata = objectColumnIndexList.get(i);
+ if (timeDeletions.isEmpty() && objectDeletions.get(i).isEmpty()) {
+ continue;
+ }
+ ChunkMetadata valueChunkMetadata =
+ (ChunkMetadata)
+ alignedChunkMetadata.getValueChunkMetadataList().get(idxInAlignedChunkMetadata);
+ if (valueChunkMetadata == null) {
+ continue;
+ }
+ Chunk chunk = reader.readMemChunk(valueChunkMetadata);
+ valueChunks.add(chunk);
+ valuePages.add(
+ chunk == null
+ ? null
+ : new CompactionChunkReader(chunk).readPageDataWithoutUncompressing());
+ }
+
+ CompactionAlignedChunkReader alignedChunkReader =
+ new CompactionAlignedChunkReader(timeChunk, valueChunks, true);
+ for (int i = 0; i < timePages.size(); i++) {
+ Pair<PageHeader, ByteBuffer> timePage = timePages.get(i);
+ List<PageHeader> valuePageHeaders = new ArrayList<>(valuePages.size());
+ List<ByteBuffer> compressedValuePages = new ArrayList<>(valuePages.size());
+ for (int j = 0; j < valuePages.size(); j++) {
+ Pair<PageHeader, ByteBuffer> valuePage = valuePages.get(j).get(i);
+ valuePageHeaders.add(valuePage.getLeft());
+ compressedValuePages.add(valuePage.getRight());
+ }
+ IPointReader pagePointReader =
+ alignedChunkReader.getPagePointReader(
+ timePage.getLeft(), valuePageHeaders, timePage.getRight(), compressedValuePages);
+
+ while (pagePointReader.hasNextTimeValuePair()) {
+ TimeValuePair timeValuePair = pagePointReader.nextTimeValuePair();
+ removeDeletedObjectFiles(timeValuePair, deletionCursors, timeDeletions, objectDeletions);
+ }
+ }
+ }
+
+ private static void removeDeletedObjectFiles(
+ TimeValuePair timeValuePair,
+ int[] cursors,
+ List<ModEntry> timeDeletions,
+ List<List<ModEntry>> objectDeletions) {
+ long timestamp = timeValuePair.getTimestamp();
+ boolean timeDeleted = isDeleted(timestamp, timeDeletions, cursors, 0);
+ for (int i = 0; i < timeValuePair.getValues().length; i++) {
+ Binary value = (Binary) timeValuePair.getValues()[i];
+ if (value == null) {
+ continue;
+ }
+ if (timeDeleted || isDeleted(timestamp, objectDeletions.get(i), cursors, i + 1)) {
+ ObjectTypeUtils.deleteObjectPathFromBinary(value);
+ }
+ }
+ }
+
+ private static boolean isDeleted(
+ long timestamp, List<ModEntry> deleteIntervalList, int[] deleteCursors, int idx) {
+ while (deleteIntervalList != null && deleteCursors[idx] < deleteIntervalList.size()) {
+ if (deleteIntervalList.get(deleteCursors[idx]).getTimeRange().contains(timestamp)) {
+ return true;
+ } else if (deleteIntervalList.get(deleteCursors[idx]).getTimeRange().getMax() < timestamp) {
+ deleteCursors[idx]++;
+ } else {
+ return false;
+ }
+ }
+ return false;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
index 1889182a2db..75900689ca2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -443,7 +443,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
*/
private void applyModificationForAlignedChunkMetadataList(
TsFileResource tsFileResource, List<AbstractAlignedChunkMetadata> alignedChunkMetadataList)
- throws IllegalPathException {
+ throws IllegalPathException, IOException {
if (alignedChunkMetadataList.isEmpty()) {
// all the value chunks is empty chunk
return;
@@ -482,6 +482,15 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
modificationList.isEmpty() ? Collections.emptyList() : modificationList);
}
+ if (ttlDeletion != null) {
+ List<ModEntry> emptyList = Collections.emptyList();
+ CompactionUtils.removeDeletedObjectFiles(
+ readerMap.get(tsFileResource),
+ alignedChunkMetadataList,
+ Collections.singletonList(ttlDeletion),
+ modificationForValueColumns.stream().map(v -> emptyList).collect(Collectors.toList()));
+ }
+
ModificationUtils.modifyAlignedChunkMetaData(
alignedChunkMetadataList,
modificationForTimeColumn,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
index 4e71c03d25f..166417a97ea 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
@@ -22,8 +22,11 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.ex
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.AlignedPath;
import org.apache.iotdb.commons.path.PatternTreeMap;
+import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.db.exception.WriteProcessException;
+import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.batch.utils.AlignedSeriesBatchCompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.AlignedPageElement;
@@ -59,9 +62,11 @@ import org.apache.tsfile.write.schema.IMeasurementSchema;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
public class FastAlignedSeriesCompactionExecutor extends SeriesCompactionExecutor {
@@ -175,7 +180,8 @@ public class FastAlignedSeriesCompactionExecutor extends SeriesCompactionExecuto
new ChunkMetadataElement(
alignedChunkMetadataList.get(i),
i == alignedChunkMetadataList.size() - 1,
- fileElement));
+ fileElement,
+ measurementSchemas));
}
}
}
@@ -264,6 +270,23 @@ public class FastAlignedSeriesCompactionExecutor extends SeriesCompactionExecuto
}
});
+ long ttlForTable =
+ deviceId.isTableModel()
+ ? DataNodeTTLCache.getInstance()
+ .getTTLForTable(resource.getDatabaseName(), deviceId.getTableName())
+ : Long.MAX_VALUE;
+ if (ttlForTable != Long.MAX_VALUE) {
+ ModEntry ttlDeletion =
+ CompactionUtils.convertTtlToDeletion(
+ deviceId, CommonDateTimeUtils.currentTime() - ttlForTable);
+ List<ModEntry> emptyList = Collections.emptyList();
+ CompactionUtils.removeDeletedObjectFiles(
+ readerCacheMap.get(resource),
+ alignedChunkMetadataList,
+ Collections.singletonList(ttlDeletion),
+ valueModifications.stream().map(v -> emptyList).collect(Collectors.toList()));
+ }
+
// modify aligned chunk metadatas
ModificationUtils.modifyAlignedChunkMetaData(
alignedChunkMetadataList, timeModifications, valueModifications, ignoreAllNullRows);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java
index b07407c7db6..0a77bf16ea7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java
@@ -52,6 +52,7 @@ import org.apache.tsfile.write.schema.MeasurementSchema;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -160,7 +161,10 @@ public class FastNonAlignedSeriesCompactionExecutor extends SeriesCompactionExec
// add into queue
chunkMetadataQueue.add(
new ChunkMetadataElement(
- chunkMetadata, i == iChunkMetadataList.size() - 1, fileElement));
+ chunkMetadata,
+ i == iChunkMetadataList.size() - 1,
+ fileElement,
+ Collections.emptyList()));
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java
index 6828841f1bd..2d29e3944d8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.storageengine.dataregion.read.reader.common.MergeRead
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.read.common.Chunk;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
import java.util.List;
@@ -42,12 +43,19 @@ public class ChunkMetadataElement {
public boolean needForceDecodingPage;
+ // for aligned series
+ public List<IMeasurementSchema> measurementSchemasOfValueChunks;
+
public ChunkMetadataElement(
- IChunkMetadata chunkMetadata, boolean isLastChunk, FileElement fileElement) {
+ IChunkMetadata chunkMetadata,
+ boolean isLastChunk,
+ FileElement fileElement,
+ List<IMeasurementSchema> measurementSchemasOfValueChunks) {
this.chunkMetadata = chunkMetadata;
this.startTime = chunkMetadata.getStartTime();
this.isLastChunk = isLastChunk;
this.fileElement = fileElement;
+ this.measurementSchemasOfValueChunks = measurementSchemasOfValueChunks;
}
public void clearChunks() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
index 928c0cdbc6b..c2e0959e652 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
@@ -565,6 +565,7 @@ public class ReadChunkAlignedSeriesCompactionExecutor {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
size = pageLoader.getHeader().getUncompressedSize();
break;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java
index b09adb17bba..cff44c7618a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java
@@ -322,6 +322,7 @@ public class SingleSeriesCompactionExecutor {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary());
break;
case FLOAT:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java
index 3458f9870d4..f3cd5185b58 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java
@@ -149,6 +149,7 @@ public abstract class AbstractCompactionWriter implements AutoCloseable {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
chunkWriterImpl.write(timestamp, value.getBinary());
break;
case DOUBLE:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
index e69a5b8b8c3..25c4a876ab7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
@@ -220,6 +220,7 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
for (int i = 0; i < tsBlock.getPositionCount(); i++) {
if (tsBlock.getColumn(column).isNull(i)) {
continue;
@@ -412,6 +413,7 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueBuilder.writeBinary(values[columnIndex].getBinary());
break;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
index 49f8ab0dada..f42867d4772 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java
@@ -696,6 +696,7 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
alignedChunkWriter.writeByColumn(
time,
isNull ? null : list.getBinaryByValueIndex(originRowIndex, columnIndex),
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
index dbc3183df80..6499d33fcd9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -166,6 +166,7 @@ public class WritableMemChunk extends AbstractWritableMemChunk {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary[] binaryValues = (Binary[]) valueList;
putBinaries(times, binaryValues, bitMap, start, end);
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
index 7ee43a93aea..dbe948c01b8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
@@ -293,6 +293,7 @@ public class MemAlignedPageReader implements IPageReader {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
for (int i = 0; i < tsBlock.getPositionCount(); i++) {
valueStatistics[column].update(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java
index a8680240072..e7427ef87d8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java
@@ -105,6 +105,7 @@ public class MemPageReader implements IPageReader {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
batchData.putBinary(
tsBlock.getTimeColumn().getLong(i), tsBlock.getColumn(0).getBinary(i));
break;
@@ -269,6 +270,7 @@ public class MemPageReader implements IPageReader {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
for (int i = 0; i < tsBlock.getPositionCount(); i++) {
statistics.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(0).getBinary(i));
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
index 16be82188e9..b3f1327483a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.storageengine.rescon.disk.FolderManager;
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.fileSystem.FSFactoryProducer;
import org.apache.tsfile.fileSystem.fsFactory.FSFactory;
import org.slf4j.Logger;
@@ -175,6 +176,20 @@ public class TsFileNameGenerator {
}
}
+ public static String generateObjectFilePath(int regionId, long time, IDeviceID iDeviceID) {
+ String objectFileName = time + ".bin";
+ Object[] segments = iDeviceID.getSegments();
+ StringBuilder relativePathString =
+ new StringBuilder(String.valueOf(regionId)).append(File.separator);
+ for (Object segment : segments) {
+ relativePathString
+ .append(segment == null ? "null" : segment.toString().toLowerCase())
+ .append(File.separator);
+ }
+ relativePathString.append(objectFileName);
+ return relativePathString.toString();
+ }
+
@TestOnly
public static TsFileResource increaseCrossCompactionCnt(TsFileResource tsFileResource)
throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
index 8cc080016ca..a7d79f92b57 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint;
@@ -332,6 +333,8 @@ public class WALBuffer extends AbstractWALBuffer {
searchIndex = ((DeleteDataNode) walEntry.getValue()).getSearchIndex();
} else if (walEntry.getType() == WALEntryType.RELATIONAL_DELETE_DATA_NODE) {
searchIndex = ((RelationalDeleteDataNode) walEntry.getValue()).getSearchIndex();
+ } else if (walEntry.getType() == WALEntryType.OBJECT_FILE_NODE) {
+ searchIndex = ((ObjectNode) walEntry.getValue()).getSearchIndex();
} else {
searchIndex = ((InsertNode) walEntry.getValue()).getSearchIndex();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
index f7b85089feb..4c0ff8809d2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNo
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable;
import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
@@ -77,6 +78,8 @@ public abstract class WALEntry implements SerializedSize {
this.type = WALEntryType.CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE;
} else if (value instanceof RelationalDeleteDataNode) {
this.type = WALEntryType.RELATIONAL_DELETE_DATA_NODE;
+ } else if (value instanceof ObjectNode) {
+ this.type = WALEntryType.OBJECT_FILE_NODE;
} else {
throw new RuntimeException("Unknown WALEntry type");
}
@@ -134,6 +137,9 @@ public abstract class WALEntry implements SerializedSize {
case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE:
value = (ContinuousSameSearchIndexSeparatorNode) PlanNodeType.deserializeFromWAL(stream);
break;
+ case OBJECT_FILE_NODE:
+ value = (ObjectNode) PlanNodeType.deserializeFromWAL(stream);
+ break;
default:
throw new RuntimeException("Unknown WALEntry type " + type);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java
index 1c3ddcbf702..829affd1b4e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java
@@ -45,6 +45,7 @@ public enum WALEntryType {
/** {@link org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable} */
MEMORY_TABLE_SNAPSHOT((byte) 10),
RELATIONAL_DELETE_DATA_NODE((byte) 11),
+ OBJECT_FILE_NODE((byte) 12),
// endregion
// region signal entry type
// signal wal buffer has been closed
@@ -71,7 +72,8 @@ public enum WALEntryType {
|| this == INSERT_ROW_NODE
|| this == INSERT_ROWS_NODE
|| this == DELETE_DATA_NODE
- || this == RELATIONAL_DELETE_DATA_NODE;
+ || this == RELATIONAL_DELETE_DATA_NODE
+ || this == OBJECT_FILE_NODE;
}
public static WALEntryType valueOf(byte code) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java
index 6da50edba4f..791ef63a752 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode;
@@ -89,6 +90,7 @@ public class WALInfoEntry extends WALEntry {
case RELATIONAL_DELETE_DATA_NODE:
case MEMORY_TABLE_SNAPSHOT:
case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE:
+ case OBJECT_FILE_NODE:
value.serializeToWAL(buffer);
break;
case MEMORY_TABLE_CHECKPOINT:
@@ -166,6 +168,8 @@ public class WALInfoEntry extends WALEntry {
case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE:
case MEMORY_TABLE_CHECKPOINT:
return RamUsageEstimator.sizeOfObject(value);
+ case OBJECT_FILE_NODE:
+ return ((ObjectNode) value).serializedSize();
default:
throw new RuntimeException("Unsupported wal entry type " + type);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java
index 0d0e0a527f0..a8fbbee0dc4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNo
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.storageengine.dataregion.flush.FlushListener;
import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
@@ -54,6 +55,8 @@ public interface IWALNode extends FlushListener, AutoCloseable, ConsensusReqRead
/** Log BatchDoneNode */
WALFlushListener log(long memTableId, ContinuousSameSearchIndexSeparatorNode separatorNode);
+ WALFlushListener log(long memTableId, ObjectNode objectNode);
+
/** Callback when memTable created. */
void onMemTableCreated(IMemTable memTable, String targetTsFile);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
index 48bf4c1cfb9..e35d5e79fc0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNo
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALException;
@@ -82,6 +83,11 @@ public class WALFakeNode implements IWALNode {
return getResult();
}
+ @Override
+ public WALFlushListener log(long memTableId, ObjectNode objectNode) {
+ return getResult();
+ }
+
private WALFlushListener getResult() {
switch (status) {
case SUCCESS:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
index 8803aca56c4..07dd4d78f66 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNo
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.StorageEngine;
@@ -62,6 +63,8 @@ import org.apache.tsfile.utils.TsFileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -190,6 +193,12 @@ public class WALNode implements IWALNode {
return log(walEntry);
}
+ @Override
+ public WALFlushListener log(long memTableId, ObjectNode objectNode) {
+ WALEntry walEntry = new WALInfoEntry(memTableId, objectNode);
+ return log(walEntry);
+ }
+
private WALFlushListener log(WALEntry walEntry) {
buffer.write(walEntry);
@@ -646,6 +655,8 @@ public class WALNode implements IWALNode {
AtomicBoolean notFirstFile = new AtomicBoolean(false);
AtomicBoolean hasCollectedSufficientData = new AtomicBoolean(false);
+ long memorySize = 0;
+
// try to collect current tmpNodes to insertNodes, return true if successfully collect an
// insert node
Runnable tryToCollectInsertNodeAndBumpIndex =
@@ -684,7 +695,23 @@ public class WALNode implements IWALNode {
} else if (currentWalEntryIndex < nextSearchIndex) {
// WAL entry is outdated, do nothing, continue to see next WAL entry
} else if (currentWalEntryIndex == nextSearchIndex) {
- tmpNodes.get().add(new IoTConsensusRequest(buffer));
+ if (type == WALEntryType.OBJECT_FILE_NODE) {
+ WALEntry walEntry =
+ WALEntry.deserialize(
+ new DataInputStream(new ByteArrayInputStream(buffer.array())));
+ // only be called by leader read from wal
+ // wal only has relativePath, offset, eof, length
+ // need to add WALEntryType + memtableId + relativePath, offset, eof, length +
+ // content
+ // need to add IoTConsensusRequest instead of ObjectNode
+ tmpNodes
+ .get()
+ .add(new IoTConsensusRequest(((ObjectNode) walEntry.getValue()).serialize()));
+ memorySize += ((ObjectNode) walEntry.getValue()).getMemorySize();
+ } else {
+ tmpNodes.get().add(new IoTConsensusRequest(buffer));
+ memorySize += buffer.remaining();
+ }
} else {
// currentWalEntryIndex > targetIndex
// WAL entry of targetIndex has been fully collected, put them into insertNodes
@@ -696,12 +723,31 @@ public class WALNode implements IWALNode {
currentWalEntryIndex);
nextSearchIndex = currentWalEntryIndex;
}
- tmpNodes.get().add(new IoTConsensusRequest(buffer));
+ if (type == WALEntryType.OBJECT_FILE_NODE) {
+ WALEntry walEntry =
+ WALEntry.deserialize(
+ new DataInputStream(new ByteArrayInputStream(buffer.array())));
+ // only be called by leader read from wal
+ // wal only has relativePath, offset, eof, length
+ // need to add WALEntryType + memtableId + relativePath, offset, eof, length +
+ // content
+ // need to add IoTConsensusRequest instead of ObjectNode
+ tmpNodes
+ .get()
+ .add(new IoTConsensusRequest(((ObjectNode) walEntry.getValue()).serialize()));
+ memorySize += ((ObjectNode) walEntry.getValue()).getMemorySize();
+ } else {
+ tmpNodes.get().add(new IoTConsensusRequest(buffer));
+ memorySize += buffer.remaining();
+ }
}
} else {
tryToCollectInsertNodeAndBumpIndex.run();
}
- if (hasCollectedSufficientData.get()) {
+ if (memorySize > config.getWalBufferSize()) {
+ tryToCollectInsertNodeAndBumpIndex.run();
+ }
+ if (memorySize > config.getWalBufferSize() || hasCollectedSufficientData.get()) {
break COLLECT_FILE_LOOP;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
index 798f25d4e26..f4da4107334 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
@@ -223,6 +223,8 @@ public class UnsealedTsFileRecoverPerformer extends AbstractTsFileRecoverPerform
case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE:
// The CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE doesn't need redo
break;
+ case OBJECT_FILE_NODE:
+ break;
default:
throw new RuntimeException("Unsupported type " + walEntry.getType());
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java
index be31a03d76a..6e8d3850b8e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java
@@ -109,6 +109,7 @@ public class BatchedAlignedValueChunkData extends AlignedChunkData {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
dataSize += ReadWriteIOUtils.write(values[i].getBinary(), stream);
break;
@@ -199,6 +200,7 @@ public class BatchedAlignedValueChunkData extends AlignedChunkData {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
final Binary binaryValue =
isNull ? DEFAULT_BINARY : ReadWriteIOUtils.readBinary(stream);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
index d929f765607..61bb6ffafba 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.storageengine.rescon.disk;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.exception.ObjectFileNotExist;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
@@ -45,6 +46,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -65,12 +67,16 @@ public class TierManager {
*/
private final List<FolderManager> unSeqTiers = new ArrayList<>();
+ private final List<FolderManager> objectTiers = new ArrayList<>();
+
/** seq file folder's rawFsPath path -> tier level */
private final Map<String, Integer> seqDir2TierLevel = new HashMap<>();
/** unSeq file folder's rawFsPath path -> tier level */
private final Map<String, Integer> unSeqDir2TierLevel = new HashMap<>();
+ private List<String> objectDirs;
+
/** total space of each tier, Long.MAX_VALUE when one tier contains remote storage */
private long[] tierDiskTotalSpace;
@@ -151,6 +157,22 @@ public class TierManager {
for (String dir : unSeqDirs) {
unSeqDir2TierLevel.put(dir, tierLevel);
}
+
+ objectDirs =
+ Arrays.stream(tierDirs[tierLevel])
+ .filter(Objects::nonNull)
+ .map(
+ v ->
+ FSFactoryProducer.getFSFactory()
+ .getFile(v, IoTDBConstant.OBJECT_FOLDER_NAME)
+ .getPath())
+ .collect(Collectors.toList());
+
+ try {
+ objectTiers.add(new FolderManager(objectDirs, directoryStrategyType));
+ } catch (DiskSpaceInsufficientException e) {
+ logger.error("All disks of tier {} are full.", tierLevel, e);
+ }
}
tierDiskTotalSpace = getTierDiskSpace(DiskSpaceType.TOTAL);
@@ -160,6 +182,7 @@ public class TierManager {
long startTime = System.currentTimeMillis();
seqTiers.clear();
unSeqTiers.clear();
+ objectTiers.clear();
seqDir2TierLevel.clear();
unSeqDir2TierLevel.clear();
@@ -190,6 +213,10 @@ public class TierManager {
: unSeqTiers.get(tierLevel).getNextFolder();
}
+ public String getNextFolderForObjectFile() throws DiskSpaceInsufficientException {
+ return objectTiers.get(0).getNextFolder();
+ }
+
public FolderManager getFolderManager(int tierLevel, boolean sequence) {
return sequence ? seqTiers.get(tierLevel) : unSeqTiers.get(tierLevel);
}
@@ -222,6 +249,20 @@ public class TierManager {
.collect(Collectors.toList());
}
+ public List<String> getAllObjectFileFolders() {
+ return objectDirs;
+ }
+
+ public Optional<File> getAbsoluteObjectFilePath(String filePath) {
+ for (String objectDir : objectDirs) {
+ File objectFile = FSFactoryProducer.getFSFactory().getFile(objectDir, filePath);
+ if (objectFile.exists()) {
+ return Optional.of(objectFile);
+ }
+ }
+ return Optional.empty();
+ }
+
public int getTiersNum() {
return seqTiers.size();
}
@@ -294,6 +335,16 @@ public class TierManager {
return tierDiskSpace;
}
+ public File getObjectFile(String relativePath) {
+ for (String folder : objectDirs) {
+ File file = new File(folder, relativePath);
+ if (file.exists()) {
+ return file;
+ }
+ }
+ throw new ObjectFileNotExist(relativePath);
+ }
+
private enum DiskSpaceType {
TOTAL,
USABLE,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java
index 3281b67c034..dbf3b989800 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java
@@ -343,6 +343,7 @@ public class PrimitiveArrayManager {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
Binary[][] binaries = new Binary[arrayNumber][];
for (int i = 0; i < arrayNumber; i++) {
binaries[i] = new Binary[ARRAY_SIZE];
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java
index f50e472c414..38686bee047 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java
@@ -430,6 +430,7 @@ public class TsFileSplitByPartitionTool implements AutoCloseable {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
chunkWriter.write(time, (Binary) value);
break;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
index b9dbcfe8685..1fa1ec44fc1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java
@@ -380,6 +380,7 @@ public class CommonUtils {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
valueColumn = new Binary[rowNum];
break;
case DATE:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java
index 402921338a4..d9d4cecc4e4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java
@@ -51,6 +51,7 @@ public class EncodingInferenceUtils {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return conf.getDefaultTextEncoding();
default:
throw new UnSupportedDataTypeException(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
index 843bcdb8937..1de1422b07b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
@@ -108,7 +108,9 @@ public class MemUtils {
if (results == null
|| results[i] == null
|| results[i].code == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- memSize += RamUsageEstimator.sizeOf(column[i].getValues());
+ if (column[i] != null) {
+ memSize += RamUsageEstimator.sizeOf(column[i].getValues());
+ }
}
}
return memSize;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
new file mode 100644
index 00000000000..fc1bb40b93f
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils;
+
+import org.apache.iotdb.commons.exception.ObjectFileNotExist;
+import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.utils.Binary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Optional;
+
+public class ObjectTypeUtils {
+
+ private static final Logger logger = LoggerFactory.getLogger(ObjectTypeUtils.class);
+ private static final TierManager TIER_MANAGER = TierManager.getInstance();
+
+ private ObjectTypeUtils() {}
+
+ public static File getObjectPathFromBinary(Binary binary) {
+ byte[] bytes = binary.getValues();
+ String relativeObjectFilePath =
+ new String(bytes, 8, bytes.length - 8, TSFileConfig.STRING_CHARSET);
+ Optional<File> file = TIER_MANAGER.getAbsoluteObjectFilePath(relativeObjectFilePath);
+ if (!file.isPresent()) {
+ throw new ObjectFileNotExist(relativeObjectFilePath);
+ }
+ return file.get();
+ }
+
+ public static Optional<File> getNullableObjectPathFromBinary(Binary binary) {
+ byte[] bytes = binary.getValues();
+ String relativeObjectFilePath =
+ new String(bytes, 8, bytes.length - 8, TSFileConfig.STRING_CHARSET);
+ return TIER_MANAGER.getAbsoluteObjectFilePath(relativeObjectFilePath);
+ }
+
+ public static void deleteObjectPathFromBinary(Binary binary) {
+ Optional<File> file = ObjectTypeUtils.getNullableObjectPathFromBinary(binary);
+ if (!file.isPresent()) {
+ return;
+ }
+ File parentFile = file.get().getParentFile();
+ File tmpFile = new File(parentFile, file.get().getName().replace(".bin", ".tmp"));
+ File bakFile = new File(parentFile, file.get().getName().replace(".bin", ".bak"));
+ logger.info("Remove object file {}", file.get().getAbsolutePath());
+ for (int i = 0; i < 2; i++) {
+ try {
+ if (Files.deleteIfExists(file.get().toPath())) {
+ return;
+ }
+ if (Files.deleteIfExists(tmpFile.toPath())) {
+ return;
+ }
+ if (Files.deleteIfExists(bakFile.toPath())) {
+ return;
+ }
+ } catch (IOException e) {
+ logger.error("Failed to remove object file {}", file.get().getAbsolutePath(), e);
+ }
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java
new file mode 100644
index 00000000000..e007027bd6d
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import org.apache.tsfile.external.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+
+public class ObjectWriter implements AutoCloseable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ObjectWriter.class);
+
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+
+ private final FileOutputStream fos;
+
+ private final File file;
+
+ public ObjectWriter(File filePath) throws FileNotFoundException {
+ try {
+ FileUtils.forceMkdir(filePath.getParentFile());
+ } catch (final IOException e) {
+ throw new FileNotFoundException("Error occurred during creating directory " + filePath);
+ }
+ if (!Files.exists(filePath.toPath())) {
+ try {
+ Files.createFile(filePath.toPath());
+ } catch (IOException e) {
+ throw new FileNotFoundException(e.getMessage());
+ }
+ }
+ file = filePath;
+ fos = new FileOutputStream(filePath, true);
+ }
+
+ public void write(boolean isGeneratedByConsensus, long offset, byte[] content)
+ throws IOException {
+ if (file.length() != offset) {
+ if (isGeneratedByConsensus || offset == 0) {
+ fos.getChannel().truncate(offset);
+ } else {
+ throw new IOException(
+ "The file length " + file.length() + " is not equal to the offset " + offset);
+ }
+ }
+ if (file.length() + content.length > config.getMaxObjectSizeInByte()) {
+ throw new IOException("The file length is larger than max_object_file_size_in_bytes");
+ }
+ fos.write(content);
+ }
+
+ @Override
+ public void close() throws Exception {
+ fos.close();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java
index cd4ca79ac43..fed80815f69 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java
@@ -236,6 +236,7 @@ public class QueryDataSetUtils {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
for (int i = 0; i < currentCount; i++) {
rowCount++;
if (column.isNull(i)) {
@@ -379,6 +380,7 @@ public class QueryDataSetUtils {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
doWithTextColumn(
rowCount,
column,
@@ -753,6 +755,7 @@ public class QueryDataSetUtils {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary[] binaryValues = new Binary[size];
for (int index = 0; index < size; index++) {
int binarySize = buffer.getInt();
@@ -795,6 +798,7 @@ public class QueryDataSetUtils {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
parseTextColumn(size, stream, values, i);
break;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java
index e11b586e7e4..a66e7d7b856 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java
@@ -62,6 +62,7 @@ public class TimeValuePairUtils {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
to.getValue().setBinary(from.getValue().getBinary());
break;
case BOOLEAN:
@@ -89,6 +90,7 @@ public class TimeValuePairUtils {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return new TimeValuePair(0, new TsBinary(new Binary("", TSFileConfig.STRING_CHARSET)));
default:
throw new UnsupportedOperationException("Unrecognized datatype: " + dataType);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
index cca6cd86a6e..aa2855f7b05 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
@@ -325,6 +325,7 @@ public class TypeInferenceUtils {
case DATE:
case TIMESTAMP:
case BLOB:
+ case OBJECT:
case STRING:
return false;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 4be8e9f437e..844adfb0e51 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -204,6 +204,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
((Binary[]) columnValues.get(arrayIndex))[elementIndex] =
columnValue != null ? (Binary) columnValue : Binary.EMPTY_VALUE;
memoryBinaryChunkSize[i] +=
@@ -304,6 +305,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary valueT = ((Binary[]) columnValues.get(arrayIndex))[elementIndex];
vector[columnIndex] = TsPrimitiveType.getByType(TSDataType.TEXT, valueT);
break;
@@ -365,6 +367,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
columnValue.add(getPrimitiveArraysByType(TSDataType.TEXT));
break;
case FLOAT:
@@ -642,6 +645,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary[] valueT = (Binary[]) value;
Binary[] cloneT = new Binary[valueT.length];
System.arraycopy(valueT, 0, cloneT, 0, valueT.length);
@@ -872,6 +876,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary[] arrayT = ((Binary[]) columnValues.get(arrayIndex));
System.arraycopy(value[i], idx, arrayT, elementIndex, remaining);
@@ -1174,6 +1179,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueBuilder.writeBinary(getBinaryByValueIndex(originRowIndex, columnIndex));
break;
default:
@@ -1251,6 +1257,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
for (int rowIdx = 0; rowIdx < rowCount; ++rowIdx) {
size += ReadWriteIOUtils.sizeToWrite(getBinaryByValueIndex(rowIdx, columnIndex));
}
@@ -1311,6 +1318,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary valueT = ((Binary[]) columnValues.get(arrayIndex))[elementIndex];
// In some scenario, the Binary in AlignedTVList will be null if this field is empty in
// current row. We need to handle this scenario to get rid of NPE. See the similar issue
@@ -1391,6 +1399,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary[] binaryValues = new Binary[rowCount];
for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) {
binaryValues[rowIndex] = ReadWriteIOUtils.readBinary(stream);
@@ -1853,6 +1862,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return TsPrimitiveType.getByType(
TSDataType.TEXT, getBinaryByValueIndex(valueIndex, validColumnIndex));
default:
@@ -2089,6 +2099,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueBuilder.writeBinary(getBinaryByValueIndex(originRowIndex, validColumnIndex));
break;
default:
@@ -2269,6 +2280,7 @@ public abstract class AlignedTVList extends TVList {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueChunkWriter.write(
time,
isNull ? null : getBinaryByValueIndex(originRowIndex, validColumnIndex),
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java
index b115713b6eb..a1001815981 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java
@@ -272,6 +272,7 @@ public class MergeSortMultiAlignedTVListIterator extends MultiAlignedTVListItera
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
valueChunkWriter.write(
currentTime,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java
index 9f6b6c1b4f6..8e6dd012265 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java
@@ -165,6 +165,7 @@ public class MergeSortMultiTVListIterator extends MultiTVListIterator {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
Binary value = currIterator.getTVList().getBinary(row);
chunkWriterImpl.write(time, value);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java
index 42de51ac4b7..3a522f5e714 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java
@@ -254,6 +254,7 @@ public abstract class MultiAlignedTVListIterator extends MemPointIterator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
valueBuilder.writeBinary(
alignedTVList.getBinaryByValueIndex(valueIndex, validColumnIndex));
break;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java
index b735cc7927b..4b394351839 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java
@@ -219,6 +219,7 @@ public abstract class MultiTVListIterator extends MemPointIterator {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
while (hasNextTimeValuePair() && builder.getPositionCount() < maxNumberOfPointsInPage) {
TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex);
Binary binary = iterator.getTVList().getBinary(iterator.getScanOrderIndex(rowIndex));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index 45aee62f3b0..a1f2842d34a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -114,6 +114,7 @@ public abstract class TVList implements WALEntryValue {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return BinaryTVList.newList();
case FLOAT:
return FloatTVList.newList();
@@ -676,6 +677,7 @@ public abstract class TVList implements WALEntryValue {
switch (dataType) {
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
return BinaryTVList.deserialize(stream);
case FLOAT:
@@ -702,6 +704,7 @@ public abstract class TVList implements WALEntryValue {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
return BinaryTVList.deserializeWithoutBitMap(stream);
case FLOAT:
return FloatTVList.deserializeWithoutBitMap(stream);
@@ -1126,6 +1129,7 @@ public abstract class TVList implements WALEntryValue {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
while (index < rows
&& builder.getPositionCount() < maxNumberOfPointsInPage
&& paginationController.hasCurLimit()) {
@@ -1226,6 +1230,7 @@ public abstract class TVList implements WALEntryValue {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
Binary value = getBinary(index);
chunkWriterImpl.write(time, value);
encodeInfo.dataSizeInChunk += 8L + getBinarySize(value);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java
index 0bc72b3205a..a87fb480d9d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java
@@ -87,6 +87,7 @@ public class WindowImpl implements Window {
case TEXT:
case BLOB:
case STRING:
+ case OBJECT:
binaryValues = new Binary[size];
for (int i = 0; i < size; ++i) {
binaryValues[i] = list.getBinaryByIndex(begin + i);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDeleteObjectFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDeleteObjectFileTest.java
new file mode 100644
index 00000000000..a4b35c01da6
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionDeleteObjectFileTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.compaction;
+
+import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.commons.schema.table.TsTable;
+import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema;
+import org.apache.iotdb.commons.schema.table.column.TagColumnSchema;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.SettleCompactionTask;
+import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate;
+import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate;
+import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
+import org.apache.tsfile.file.metadata.StringArrayDeviceID;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.common.TimeRange;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public class CompactionDeleteObjectFileTest extends AbstractCompactionTest {
+ @Before
+ public void setUp()
+ throws IOException, WriteProcessException, MetadataException, InterruptedException {
+ super.setUp();
+ }
+
+ @After
+ public void tearDown() throws IOException, StorageEngineException {
+ super.tearDown();
+ }
+
+ @Test
+ public void test1() throws IOException {
+ createTable("tsfile_table", 100);
+ File dir = new File("/Users/shuww/Downloads/0708/1_副本");
+ List<TsFileResource> resources = new ArrayList<>();
+ for (File file : dir.listFiles()) {
+ if (!file.getName().endsWith(".tsfile")) {
+ continue;
+ }
+ TsFileResource resource = new TsFileResource(file);
+
+ try (ModificationFile modificationFile = resource.getExclusiveModFile()) {
+ modificationFile.write(
+ new TableDeletionEntry(
+ new DeletionPredicate(
+ "tsfile_table",
+ new IDPredicate.FullExactMatch(
+ new StringArrayDeviceID(new String[] {"tsfile_table", "1", "5", "3"})),
+ Arrays.asList("file")),
+ new TimeRange(-1, 0)));
+ modificationFile.write(
+ new TableDeletionEntry(
+ new DeletionPredicate(
+ "tsfile_table",
+ new IDPredicate.FullExactMatch(
+ new StringArrayDeviceID(new String[] {"tsfile_table", "1", "5", "3"})),
+ Arrays.asList("file")),
+ new TimeRange(2, 2)));
+ }
+ resource.deserialize();
+ resources.add(resource);
+ }
+
+ // InnerSpaceCompactionTask task =
+ // new InnerSpaceCompactionTask(
+ // 0, tsFileManager, resources, true, new ReadChunkCompactionPerformer(), 0);
+ SettleCompactionTask task =
+ new SettleCompactionTask(
+ 0,
+ tsFileManager,
+ resources,
+ Collections.emptyList(),
+ true,
+ new FastCompactionPerformer(false),
+ 0);
+ task.start();
+ }
+
+ public void createTable(String tableName, long ttl) {
+ TsTable tsTable = new TsTable(tableName);
+ tsTable.addColumnSchema(new TagColumnSchema("id_column", TSDataType.STRING));
+ tsTable.addColumnSchema(
+ new FieldColumnSchema("s1", TSDataType.STRING, TSEncoding.PLAIN, CompressionType.LZ4));
+ tsTable.addProp(TsTable.TTL_PROPERTY, ttl + "");
+ DataNodeTableCache.getInstance().preUpdateTable("Downloads", tsTable, null);
+ DataNodeTableCache.getInstance().commitUpdateTable("Downloads", tableName, null);
+ }
+}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java
index 3ab79997baf..d5c7b3e865f 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java
@@ -92,6 +92,7 @@ public class PrimitiveArrayManagerTest {
break;
case TEXT:
case BLOB:
+ case OBJECT:
case STRING:
Assert.assertTrue(o instanceof Binary[]);
Assert.assertEquals(ARRAY_SIZE, ((Binary[]) o).length);
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 6bbc4124a9f..0a9a47ee890 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1302,6 +1302,11 @@ enable_tsfile_validation=false
# Unit: ms
tier_ttl_in_ms=-1
+# The maximum size limit for a single object file.
+# effectiveMode: hot_reload
+# Datatype: long
+max_object_file_size_in_byte=4294967296
+
####################
### Compaction Configurations
####################
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index 13fbf23972f..adf72842797 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -248,6 +248,7 @@ public class IoTDBConstant {
public static final String DATA_FOLDER_NAME = "data";
public static final String SEQUENCE_FOLDER_NAME = "sequence";
public static final String UNSEQUENCE_FOLDER_NAME = "unsequence";
+ public static final String OBJECT_FOLDER_NAME = "object";
public static final String FILE_NAME_SEPARATOR = "-";
public static final String CONSENSUS_FOLDER_NAME = "consensus";
public static final String DATA_REGION_FOLDER_NAME = "data_region";
diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/ObjectFileNotExist.java
similarity index 53%
copy from iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java
copy to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/ObjectFileNotExist.java
index 9093133631a..05add08b218 100644
--- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/ObjectFileNotExist.java
@@ -17,47 +17,15 @@
* under the License.
*/
-package org.apache.iotdb.pipe.api.type;
+package org.apache.iotdb.commons.exception;
-public enum Type {
- /* BOOLEAN */
- BOOLEAN((byte) 0),
+import static org.apache.iotdb.rpc.TSStatusCode.OBJECT_NOT_EXISTS;
- /* INT32 */
- INT32((byte) 1),
+public class ObjectFileNotExist extends IoTDBRuntimeException {
- /* INT64 */
- INT64((byte) 2),
+ private static final String ERROR_MSG = "Object file %s does not exist";
- /* FLOAT */
- FLOAT((byte) 3),
-
- /* DOUBLE */
- DOUBLE((byte) 4),
-
- /* TEXT */
- TEXT((byte) 5),
-
- /* TsDataType.Vector and TsDataType.UNKNOWN are inner types of TsFile-module, which should not be supported in UDF APIs. To be consistent with TsDataType, the next value starts with 8 */
- /* TIMESTAMP */
- TIMESTAMP((byte) 8),
-
- /* DATE */
- DATE((byte) 9),
-
- /* BLOB */
- BLOB((byte) 10),
-
- /* STRING */
- STRING((byte) 11);
-
- private final byte dataType;
-
- Type(byte type) {
- this.dataType = type;
- }
-
- public byte getType() {
- return dataType;
+ public ObjectFileNotExist(String relativeObjectPath) {
+ super(String.format(ERROR_MSG, relativeObjectPath), OBJECT_NOT_EXISTS.getStatusCode());
}
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java
index be0d16807e2..c3fe9342066 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java
@@ -64,6 +64,7 @@ public class UDTFAbs extends UDTFMath {
collector.putDouble(time, Math.abs(row.getDouble(0)));
break;
case BLOB:
+ case OBJECT:
case STRING:
case TIMESTAMP:
case TEXT:
@@ -103,6 +104,7 @@ public class UDTFAbs extends UDTFMath {
case TIMESTAMP:
case STRING:
case BLOB:
+ case OBJECT:
default:
// This will not happen.
throw new UDFInputSeriesDataTypeNotValidException(
@@ -131,6 +133,7 @@ public class UDTFAbs extends UDTFMath {
transformDouble(columns, builder);
return;
case BLOB:
+ case OBJECT:
case STRING:
case TEXT:
case TIMESTAMP:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java
index bb12f7ff4a5..045e49b064b 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java
@@ -73,6 +73,7 @@ public class UDTFBottomK extends UDTFSelectK {
});
break;
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
// This will not happen.
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java
index 00b3fd47550..cb5cbcc0486 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java
@@ -61,6 +61,7 @@ public class UDTFCommonDerivative extends UDTFDerivative {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
default:
// This will not happen.
throw new UDFInputSeriesDataTypeNotValidException(
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java
index 75305f73165..7f8d81f6b82 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java
@@ -56,6 +56,7 @@ public class UDTFCommonValueDifference extends UDTFValueDifference {
break;
case STRING:
case BLOB:
+ case OBJECT:
case TIMESTAMP:
case TEXT:
case BOOLEAN:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java
index 209802c26cb..f3436859f2a 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java
@@ -57,6 +57,7 @@ public class UDTFConst implements UDTF {
VALID_TYPES.add(TSDataType.TEXT.name());
VALID_TYPES.add(TSDataType.STRING.name());
VALID_TYPES.add(TSDataType.BLOB.name());
+ VALID_TYPES.add(TSDataType.OBJECT.name());
}
private TSDataType dataType;
@@ -107,6 +108,7 @@ public class UDTFConst implements UDTF {
binaryValue = BytesUtils.valueOf(parameters.getString("value"));
break;
case BLOB:
+ case OBJECT:
binaryValue = new Binary(BlobUtils.parseBlobString(parameters.getString("value")));
break;
default:
@@ -141,6 +143,7 @@ public class UDTFConst implements UDTF {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
collector.putBinary(row.getTime(), UDFBinaryTransformer.transformToUDFBinary(binaryValue));
break;
default:
@@ -166,6 +169,7 @@ public class UDTFConst implements UDTF {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
return UDFBinaryTransformer.transformToUDFBinary(binaryValue);
default:
throw new UnsupportedOperationException();
@@ -257,6 +261,7 @@ public class UDTFConst implements UDTF {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
for (int i = 0; i < count; i++) {
boolean hasWritten = false;
for (int j = 0; j < columns.length - 1; j++) {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java
index b76b0276e7c..de6232ac6db 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java
@@ -103,6 +103,7 @@ public abstract class UDTFContinuouslySatisfy implements UDTF {
case TEXT:
case STRING:
case BLOB:
+ case OBJECT:
case TIMESTAMP:
case DATE:
default:
@@ -230,6 +231,7 @@ public abstract class UDTFContinuouslySatisfy implements UDTF {
case DATE:
case STRING:
case BLOB:
+ case OBJECT:
case TEXT:
default:
// This will not happen.
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java
index d0167327613..21506698605 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java
@@ -482,6 +482,7 @@ public class UDTFEqualSizeBucketAggSample extends UDTFEqualSizeBucketSample {
aggregator.aggregateDouble(rowWindow, collector);
break;
case BLOB:
+ case OBJECT:
case TEXT:
case DATE:
case STRING:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java
index cf80380eea2..fea5cb694fa 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java
@@ -64,6 +64,7 @@ public class UDTFEqualSizeBucketM4Sample extends UDTFEqualSizeBucketSample {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
default:
// This will not happen
throw new UDFInputSeriesDataTypeNotValidException(
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java
index f3b3aacba37..85490ac534e 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java
@@ -658,6 +658,7 @@ public class UDTFEqualSizeBucketOutlierSample extends UDTFEqualSizeBucketSample
break;
case TEXT:
case BLOB:
+ case OBJECT:
case DATE:
case STRING:
case BOOLEAN:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java
index f77b7ab8a72..566e02e7b58 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java
@@ -66,6 +66,7 @@ public class UDTFEqualSizeBucketRandomSample extends UDTFEqualSizeBucketSample {
case DATE:
case STRING:
case BLOB:
+ case OBJECT:
case TEXT:
default:
// This will not happen
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java
index f24b9d6ae7b..4c17a270697 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java
@@ -86,6 +86,7 @@ public class UDTFInRange implements UDTF {
collector.putBoolean(time, row.getDouble(0) >= lower && upper >= row.getDouble(0));
break;
case BLOB:
+ case OBJECT:
case TEXT:
case DATE:
case STRING:
@@ -123,6 +124,7 @@ public class UDTFInRange implements UDTF {
case STRING:
case TEXT:
case BLOB:
+ case OBJECT:
default:
// This will not happen.
throw new UDFInputSeriesDataTypeNotValidException(
@@ -151,6 +153,7 @@ public class UDTFInRange implements UDTF {
transformDouble(columns, builder);
return;
case BLOB:
+ case OBJECT:
case TEXT:
case DATE:
case STRING:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java
index d554cfe2572..0c5ccd73b49 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java
@@ -98,6 +98,7 @@ public class UDTFJexl implements UDTF {
case TIMESTAMP:
case DATE:
case BLOB:
+ case OBJECT:
default:
throw new UDFInputSeriesDataTypeNotValidException(
0,
@@ -170,6 +171,7 @@ public class UDTFJexl implements UDTF {
case DATE:
case STRING:
case BLOB:
+ case OBJECT:
case INT64:
case INT32:
case FLOAT:
@@ -346,6 +348,7 @@ public class UDTFJexl implements UDTF {
break;
case STRING:
case BLOB:
+ case OBJECT:
case DATE:
case TIMESTAMP:
default:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java
index 824740b8c9b..3de46df72a0 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java
@@ -138,6 +138,7 @@ public class UDTFM4 implements UDTF {
transformDouble(rowWindow, collector);
break;
case BLOB:
+ case OBJECT:
case DATE:
case STRING:
case TIMESTAMP:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java
index de4f8edcec5..947cbdb2c01 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java
@@ -91,6 +91,7 @@ public abstract class UDTFMath implements UDTF {
case TIMESTAMP:
case DATE:
case BLOB:
+ case OBJECT:
default:
// This will not happen.
throw new UDFInputSeriesDataTypeNotValidException(
@@ -119,6 +120,7 @@ public abstract class UDTFMath implements UDTF {
return transformer.transform(row.getDouble(0));
case DATE:
case BLOB:
+ case OBJECT:
case STRING:
case TIMESTAMP:
case TEXT:
@@ -155,6 +157,7 @@ public abstract class UDTFMath implements UDTF {
case STRING:
case TIMESTAMP:
case BLOB:
+ case OBJECT:
case DATE:
default:
// This will not happen.
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java
index 6812d2d2d8e..077db957f6e 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java
@@ -57,6 +57,7 @@ public class UDTFNonNegativeDerivative extends UDTFDerivative {
break;
case DATE:
case BLOB:
+ case OBJECT:
case STRING:
case TIMESTAMP:
case BOOLEAN:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java
index 1183e5397fe..fcb68817f15 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java
@@ -59,6 +59,7 @@ public class UDTFNonNegativeValueDifference extends UDTFValueDifference {
case STRING:
case TIMESTAMP:
case BLOB:
+ case OBJECT:
case DATE:
default:
// This will not happen.
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java
index 66b5cd1add8..b0c95bd0d23 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java
@@ -145,6 +145,7 @@ public class UDTFOnOff implements UDTF {
transformDouble(columns, builder);
return;
case BLOB:
+ case OBJECT:
case DATE:
case STRING:
case TIMESTAMP:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java
index 49faa398f1a..a888c998d8d 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java
@@ -107,6 +107,7 @@ public abstract class UDTFSelectK implements UDTF {
transformString(row.getTime(), row.getString(0));
break;
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
// This will not happen.
@@ -180,6 +181,7 @@ public abstract class UDTFSelectK implements UDTF {
}
break;
case BLOB:
+ case OBJECT:
case BOOLEAN:
default:
// This will not happen.
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java
index 1838b85022b..b8d94308ed9 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java
@@ -53,6 +53,7 @@ public class UDTFTopK extends UDTFSelectK {
break;
case BOOLEAN:
case BLOB:
+ case OBJECT:
default:
// This will not happen.
throw new UDFInputSeriesDataTypeNotValidException(
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java
index c9900219329..d4487fdfd95 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java
@@ -71,6 +71,7 @@ public abstract class UDTFValueTrend implements UDTF {
case STRING:
case DATE:
case BLOB:
+ case OBJECT:
default:
// This will not happen.
throw new UDFInputSeriesDataTypeNotValidException(
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java
index e20ca4dd073..5d22ff58d81 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java
@@ -111,6 +111,7 @@ public enum TableBuiltinScalarFunction {
SPOOKY_HASH_V2_64("spooky_hash_v2_64"),
LPAD("lpad"),
RPAD("rpad"),
+ READ_OBJECT("read_object"),
;
private final String functionName;
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java
index 83792ebf66a..cf9ecf89f96 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java
@@ -30,6 +30,7 @@ import org.apache.tsfile.read.common.type.DoubleType;
import org.apache.tsfile.read.common.type.FloatType;
import org.apache.tsfile.read.common.type.IntType;
import org.apache.tsfile.read.common.type.LongType;
+import org.apache.tsfile.read.common.type.ObjectType;
import org.apache.tsfile.read.common.type.StringType;
import org.apache.tsfile.read.common.type.TimestampType;
@@ -82,6 +83,8 @@ public class UDFDataTypeTransformer {
return Type.BLOB;
case STRING:
return Type.STRING;
+ case OBJECT:
+ return Type.OBJECT;
default:
throw new IllegalArgumentException("Invalid input: " + type);
}
@@ -112,6 +115,8 @@ public class UDFDataTypeTransformer {
return BlobType.BLOB;
case STRING:
return StringType.STRING;
+ case OBJECT:
+ return ObjectType.OBJECT;
default:
throw new IllegalArgumentException("Invalid input: " + type);
}
@@ -139,6 +144,8 @@ public class UDFDataTypeTransformer {
return Type.BLOB;
case 11:
return Type.STRING;
+ case 12:
+ return Type.OBJECT;
default:
throw new IllegalArgumentException("Invalid input: " + type);
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java
index d0b57328dcb..bbc3a101edb 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java
@@ -174,6 +174,7 @@ public class SerializeUtils {
}
break;
case BLOB:
+ case OBJECT:
case STRING:
case TEXT:
for (int i = 0; i < length; i++) {
@@ -218,6 +219,7 @@ public class SerializeUtils {
values[j] = new TsPrimitiveType.TsFloat(buffer.getFloat());
break;
case BLOB:
+ case OBJECT:
case STRING:
case TEXT:
int len = buffer.getInt();
@@ -315,6 +317,7 @@ public class SerializeUtils {
dataOutputStream.writeInt(timeValuePairs.size());
switch (timeValuePairs.get(0).getValue().getDataType()) {
case BLOB:
+ case OBJECT:
case STRING:
case TEXT:
serializeTextTVPairs(timeValuePairs, dataOutputStream);
@@ -355,6 +358,7 @@ public class SerializeUtils {
switch (dataType) {
case STRING:
case BLOB:
+ case OBJECT:
case TEXT:
dataOutputStream.writeLong(timeValuePair.getTimestamp());
if (timeValuePair.getTimestamp() != Long.MIN_VALUE) {
@@ -506,6 +510,7 @@ public class SerializeUtils {
deserializeBooleanTVPairs(buffer, ret, size, dataType);
break;
case BLOB:
+ case OBJECT:
case STRING:
case TEXT:
deserializeTextTVPairs(buffer, ret, size, dataType);
@@ -539,6 +544,7 @@ public class SerializeUtils {
case BOOLEAN:
return new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.get() == 1));
case BLOB:
+ case OBJECT:
case STRING:
case TEXT:
int bytesLen = buffer.getInt();
diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift
index eb0a3430899..48afb89d336 100644
--- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift
+++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift
@@ -669,5 +669,5 @@ service IClientRPCService {
TSConnectionInfoResp fetchAllConnectionsInfo();
/** For other node's call */
- common.TSStatus testConnectionEmptyRPC()
+ common.TSStatus testConnectionEmptyRPC();
}
\ No newline at end of file