You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2026/06/29 07:40:29 UTC
(iotdb) branch rc/2.0.10 updated (5c003770c41 -> ecdc54728e5)
This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a change to branch rc/2.0.10
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 5c003770c41 [Pipe] Fix omitted legacy tablet compatibility follow-ups (#17929)
new 1c692299ad0 Propagate snapshot load failure during IoTConsensus AddPeer (#17935)
new ef5f7bea7de feat(client-cpp): add SessionC DATE/BLOB support and RowRecord getters (#17956)
new c7ba5da546a Fix aggregate write-back output database metadata (#17938)
new 178fb90a75b Fix tree model load type mismatch conversion (#17949)
new f8d3fbb9e26 Add envelope function to UDF registration script (#17971)
new ecdc54728e5 Fix IoTConsensus multi-folder snapshot load for DataRegion (#17974)
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../db/it/schema/IoTDBAlterTimeSeriesTypeIT.java | 4 +-
iotdb-client/client-cpp/src/main/SessionC.cpp | 33 ++
iotdb-client/client-cpp/src/main/SessionC.h | 12 +
.../client-cpp/src/test/cpp/sessionCIT.cpp | 42 ++-
.../statemachine/ConfigRegionStateMachine.java | 10 +-
.../persistence/executor/ConfigPlanExecutor.java | 7 +-
.../org/apache/iotdb/consensus/IStateMachine.java | 28 +-
.../consensus/iot/IoTConsensusServerImpl.java | 34 +-
.../service/IoTConsensusRPCServiceProcessor.java | 12 +-
.../ratis/ApplicationStateMachineProxy.java | 13 +-
.../simple/SimpleConsensusServerImpl.java | 4 +-
.../apache/iotdb/consensus/EmptyStateMachine.java | 4 +-
.../iot/AddPeerSnapshotLoadFailureTest.java | 354 +++++++++++++++++++++
.../iotdb/consensus/iot/util/TestStateMachine.java | 4 +-
.../apache/iotdb/consensus/ratis/TestUtils.java | 4 +-
.../consensus/simple/SimpleConsensusTest.java | 4 +-
.../dataregion/DataRegionStateMachine.java | 50 ++-
.../schemaregion/SchemaRegionStateMachine.java | 25 +-
.../db/pipe/event/common/row/PipeRowCollector.java | 24 +-
.../common/tablet/PipeRawTabletEventConverter.java | 42 ++-
.../event/common/tablet/PipeTabletCollector.java | 23 +-
.../processor/aggregate/AggregateProcessor.java | 13 +-
.../load/TreeSchemaAutoCreatorAndVerifier.java | 17 +-
.../schemaengine/schemaregion/ISchemaRegion.java | 10 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 6 +-
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 6 +-
.../dataregion/snapshot/SnapshotLoader.java | 47 +++
.../schemaRegion/SchemaRegionManagementTest.java | 28 +-
.../pipe/event/PipeTabletInsertionEventTest.java | 32 ++
.../plan/analyze/load/LoadTsFileAnalyzerTest.java | 64 ++++
.../dataregion/snapshot/IoTDBSnapshotTest.java | 98 ++++++
.../apache/iotdb/commons/i18n/CommonMessages.java | 8 +
.../apache/iotdb/commons/i18n/CommonMessages.java | 8 +
.../iotdb/commons/partition/DataPartition.java | 18 +-
library-udf/src/assembly/tools/register-UDF.bat | 1 +
library-udf/src/assembly/tools/register-UDF.sh | 1 +
36 files changed, 996 insertions(+), 94 deletions(-)
create mode 100644 iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/AddPeerSnapshotLoadFailureTest.java
(iotdb) 02/06: feat(client-cpp): add SessionC DATE/BLOB support and RowRecord getters (#17956)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch rc/2.0.10
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ef5f7bea7ded2a27e49bb43002986017f6ac59f7
Author: Hongzhi Gao <76...@qq.com>
AuthorDate: Tue Jun 16 16:59:09 2026 +0800
feat(client-cpp): add SessionC DATE/BLOB support and RowRecord getters (#17956)
* feat(client-cpp): add SessionC DATE/BLOB support and RowRecord getters
Introduce TSDate_C for typed DATE inserts, ts_row_record_get_date_int32
and ts_row_record_get_string_byte_length, and expand c_rowDelete IT to
cover DATE/BLOB readback and deletion paths.
* test(client-cpp): cover TEXT and STRING in c_rowDelete IT
Add TS_TYPE_STRING readback assertions alongside explicit TEXT checks
in the RowRecord integration test.
* chore(client-cpp): remove redundant TSDate_C comment
---
iotdb-client/client-cpp/src/main/SessionC.cpp | 33 +++++++++++++++++
iotdb-client/client-cpp/src/main/SessionC.h | 12 +++++++
.../client-cpp/src/test/cpp/sessionCIT.cpp | 42 +++++++++++++++++-----
3 files changed, 79 insertions(+), 8 deletions(-)
diff --git a/iotdb-client/client-cpp/src/main/SessionC.cpp b/iotdb-client/client-cpp/src/main/SessionC.cpp
index e7c43cdb9f1..7d86bd348f8 100644
--- a/iotdb-client/client-cpp/src/main/SessionC.cpp
+++ b/iotdb-client/client-cpp/src/main/SessionC.cpp
@@ -18,6 +18,7 @@
*/
#include "SessionC.h"
+#include "Date.h"
#include "Session.h"
#include "TableSession.h"
#include "TableSessionBuilder.h"
@@ -184,6 +185,12 @@ static std::vector<char*> toCharPtrVec(const TSDataType_C* types, const void* co
result[i] = reinterpret_cast<char*>(p);
break;
}
+ case TS_TYPE_DATE: {
+ const TSDate_C* src = static_cast<const TSDate_C*>(values[i]);
+ IoTDBDate* p = new IoTDBDate(src->year, src->month, src->day);
+ result[i] = reinterpret_cast<char*>(p);
+ break;
+ }
case TS_TYPE_TEXT:
case TS_TYPE_STRING:
case TS_TYPE_BLOB:
@@ -219,6 +226,9 @@ static void freeCharPtrVec(std::vector<char*>& vec, const TSDataType_C* types, i
case TS_TYPE_DOUBLE:
delete reinterpret_cast<double*>(vec[i]);
break;
+ case TS_TYPE_DATE:
+ delete reinterpret_cast<IoTDBDate*>(vec[i]);
+ break;
default:
delete[] vec[i];
break;
@@ -1471,6 +1481,29 @@ const char* ts_row_record_get_string(CRowRecord* record, int index) {
return "";
}
+int32_t ts_row_record_get_date_int32(CRowRecord* record, int index) {
+ if (!record || !record->cpp)
+ return 0;
+ if (index < 0 || index >= (int)record->cpp->fields.size())
+ return 0;
+ const Field& f = record->cpp->fields[index];
+ if (f.dataType != TSDataType::DATE || !f.dateV.is_initialized())
+ return 0;
+ return parseDateExpressionToInt(f.dateV.value());
+}
+
+size_t ts_row_record_get_string_byte_length(CRowRecord* record, int index) {
+ if (!record || !record->cpp)
+ return 0;
+ if (index < 0 || index >= (int)record->cpp->fields.size())
+ return 0;
+ const Field& f = record->cpp->fields[index];
+ if (f.stringV.is_initialized()) {
+ return f.stringV.value().size();
+ }
+ return 0;
+}
+
TSDataType_C ts_row_record_get_data_type(CRowRecord* record, int index) {
if (!record || !record->cpp)
return TS_TYPE_INVALID;
diff --git a/iotdb-client/client-cpp/src/main/SessionC.h b/iotdb-client/client-cpp/src/main/SessionC.h
index 5c850c09c12..fdce5801a9d 100644
--- a/iotdb-client/client-cpp/src/main/SessionC.h
+++ b/iotdb-client/client-cpp/src/main/SessionC.h
@@ -105,6 +105,12 @@ typedef enum {
typedef enum { TS_COL_TAG = 0, TS_COL_FIELD = 1, TS_COL_ATTRIBUTE = 2 } TSColumnCategory_C;
+typedef struct {
+ int year;
+ int month;
+ int day;
+} TSDate_C;
+
/* ============================================================
* Session Lifecycle — Tree Model
* ============================================================ */
@@ -382,6 +388,12 @@ double ts_row_record_get_double(CRowRecord* record, int index);
const char* ts_row_record_get_string(CRowRecord* record, int index);
+/** Returns YYYYMMDD for DATE fields; 0 if record is null, index is out of range, field is null, or not DATE. */
+int32_t ts_row_record_get_date_int32(CRowRecord* record, int index);
+
+/** Byte length of the string/binary field (TEXT, BLOB, STRING, OBJECT, ...); 0 if null or out of range. */
+size_t ts_row_record_get_string_byte_length(CRowRecord* record, int index);
+
/** Returns TS_TYPE_INVALID if record is null or index is out of range. */
TSDataType_C ts_row_record_get_data_type(CRowRecord* record, int index);
diff --git a/iotdb-client/client-cpp/src/test/cpp/sessionCIT.cpp b/iotdb-client/client-cpp/src/test/cpp/sessionCIT.cpp
index f56e10bb5ae..9db5361635d 100644
--- a/iotdb-client/client-cpp/src/test/cpp/sessionCIT.cpp
+++ b/iotdb-client/client-cpp/src/test/cpp/sessionCIT.cpp
@@ -687,7 +687,10 @@ TEST_CASE("C API - RowRecord and delete data APIs", "[c_rowDelete]") {
const char* pf = "root.cov_types.d1.sf";
const char* pd = "root.cov_types.d1.sd";
const char* pt = "root.cov_types.d1.st";
- const char* tpaths[] = {pb, pi, pf, pd, pt};
+ const char* pstr = "root.cov_types.d1.sstr";
+ const char* pdate = "root.cov_types.d1.sdate";
+ const char* pblob = "root.cov_types.d1.sblob";
+ const char* tpaths[] = {pb, pi, pf, pd, pt, pstr, pdate, pblob};
for (const char* tp : tpaths) {
dropTimeseriesIfExists(g_session, tp);
}
@@ -701,36 +704,56 @@ TEST_CASE("C API - RowRecord and delete data APIs", "[c_rowDelete]") {
TS_COMPRESSION_SNAPPY) == TS_OK);
REQUIRE(ts_session_create_timeseries(g_session, pt, TS_TYPE_TEXT, TS_ENCODING_PLAIN,
TS_COMPRESSION_SNAPPY) == TS_OK);
+ REQUIRE(ts_session_create_timeseries(g_session, pstr, TS_TYPE_STRING, TS_ENCODING_PLAIN,
+ TS_COMPRESSION_SNAPPY) == TS_OK);
+ REQUIRE(ts_session_create_timeseries(g_session, pdate, TS_TYPE_DATE, TS_ENCODING_PLAIN,
+ TS_COMPRESSION_SNAPPY) == TS_OK);
+ REQUIRE(ts_session_create_timeseries(g_session, pblob, TS_TYPE_BLOB, TS_ENCODING_PLAIN,
+ TS_COMPRESSION_SNAPPY) == TS_OK);
const char* dev = "root.cov_types.d1";
- const char* names[] = {"sb", "si", "sf", "sd", "st"};
+ const char* names[] = {"sb", "si", "sf", "sd", "st", "sstr", "sdate", "sblob"};
TSDataType_C types[] = {TS_TYPE_BOOLEAN, TS_TYPE_INT32, TS_TYPE_FLOAT, TS_TYPE_DOUBLE,
- TS_TYPE_TEXT};
+ TS_TYPE_TEXT, TS_TYPE_STRING, TS_TYPE_DATE, TS_TYPE_BLOB};
bool bv = true;
int32_t iv = 42;
float fv = 2.5f;
double dv = 3.25;
const char* tv = "hi";
- const void* vals[] = {&bv, &iv, &fv, &dv, tv};
- REQUIRE(ts_session_insert_record(g_session, dev, 500LL, 5, names, types, vals) == TS_OK);
+ const char* strVal = "hello";
+ TSDate_C dateVal = {2025, 5, 7};
+ const char* blobVal = "blobXY";
+ const void* vals[] = {&bv, &iv, &fv, &dv, tv, strVal, &dateVal, blobVal};
+ REQUIRE(ts_session_insert_record(g_session, dev, 500LL, 8, names, types, vals) == TS_OK);
CSessionDataSet* dataSet = nullptr;
REQUIRE(ts_session_execute_query(g_session,
- "select sb,si,sf,sd,st from root.cov_types.d1 where time=500",
+ "select sb,si,sf,sd,st,sstr,sdate,sblob from root.cov_types.d1 "
+ "where time=500",
&dataSet) == TS_OK);
REQUIRE(dataSet != nullptr);
REQUIRE(ts_dataset_has_next(dataSet));
CRowRecord* record = ts_dataset_next(dataSet);
REQUIRE(record != nullptr);
REQUIRE(ts_row_record_get_timestamp(record) == 500LL);
- REQUIRE(ts_row_record_get_field_count(record) == 5);
+ REQUIRE(ts_row_record_get_field_count(record) == 8);
REQUIRE_FALSE(ts_row_record_is_null(record, 0));
REQUIRE(ts_row_record_get_bool(record, 0) == true);
REQUIRE(ts_row_record_get_int32(record, 1) == 42);
REQUIRE(std::fabs(ts_row_record_get_float(record, 2) - 2.5f) < 1e-4f);
REQUIRE(std::fabs(ts_row_record_get_double(record, 3) - 3.25) < 1e-9);
+ REQUIRE(ts_row_record_get_data_type(record, 4) == TS_TYPE_TEXT);
REQUIRE(std::string(ts_row_record_get_string(record, 4)) == "hi");
- REQUIRE(ts_row_record_get_data_type(record, 0) == TS_TYPE_BOOLEAN);
+ REQUIRE(ts_row_record_get_string_byte_length(record, 4) == 2);
+ REQUIRE(ts_row_record_get_data_type(record, 5) == TS_TYPE_STRING);
+ REQUIRE(std::string(ts_row_record_get_string(record, 5)) == "hello");
+ REQUIRE(ts_row_record_get_string_byte_length(record, 5) == 5);
+ REQUIRE(ts_row_record_get_date_int32(record, 6) == 20250507);
+ REQUIRE(std::string(ts_row_record_get_string(record, 7)) == "blobXY");
+ REQUIRE(ts_row_record_get_string_byte_length(record, 7) == 6);
+ for (int i = 0; i < 8; i++) {
+ REQUIRE(ts_row_record_get_data_type(record, i) == types[i]);
+ }
ts_row_record_destroy(record);
ts_dataset_destroy(dataSet);
@@ -743,5 +766,8 @@ TEST_CASE("C API - RowRecord and delete data APIs", "[c_rowDelete]") {
REQUIRE(ts_session_delete_timeseries(g_session, pf) == TS_OK);
REQUIRE(ts_session_delete_timeseries(g_session, pd) == TS_OK);
REQUIRE(ts_session_delete_timeseries(g_session, pt) == TS_OK);
+ REQUIRE(ts_session_delete_timeseries(g_session, pstr) == TS_OK);
+ REQUIRE(ts_session_delete_timeseries(g_session, pdate) == TS_OK);
+ REQUIRE(ts_session_delete_timeseries(g_session, pblob) == TS_OK);
REQUIRE(ts_session_delete_database(g_session, sg) == TS_OK);
}
(iotdb) 04/06: Fix tree model load type mismatch conversion (#17949)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch rc/2.0.10
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 178fb90a75b0554fccc9dd82b64aa741cfbb2986
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Tue Jun 16 17:15:35 2026 +0800
Fix tree model load type mismatch conversion (#17949)
* Fix tree load type mismatch conversion
* Fix load-and-alter expected count
---
.../db/it/schema/IoTDBAlterTimeSeriesTypeIT.java | 4 +-
.../load/TreeSchemaAutoCreatorAndVerifier.java | 17 +++---
.../plan/analyze/load/LoadTsFileAnalyzerTest.java | 64 ++++++++++++++++++++++
3 files changed, 75 insertions(+), 10 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java
index fcde3f61e37..713b926d1af 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterTimeSeriesTypeIT.java
@@ -879,8 +879,8 @@ public class IoTDBAlterTimeSeriesTypeIT {
session.executeQueryStatement("select count(s1) from " + database + ".load_and_alter");
RowRecord rec;
rec = dataSet.next();
- // Before alter, DOUBLE TsFiles loaded directly are invisible under the existing INT32 schema.
- assertEquals(9, rec.getFields().get(0).getLongV());
+ // Before alter, DOUBLE TsFiles are converted to INT32 and visible under the existing schema.
+ assertEquals(15, rec.getFields().get(0).getLongV());
assertFalse(dataSet.hasNext());
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java
index bfebb7db0d7..d4f4210727e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java
@@ -483,14 +483,15 @@ public class TreeSchemaAutoCreatorAndVerifier {
}
// check datatype
- if (LOGGER.isDebugEnabled() && !tsFileSchema.getType().equals(iotdbSchema.getType())) {
- LOGGER.debug(
- "Measurement {}{}{} datatype not match, TsFile: {}, IoTDB: {}",
- device,
- TsFileConstant.PATH_SEPARATOR,
- iotdbSchema.getMeasurementName(),
- tsFileSchema.getType(),
- iotdbSchema.getType());
+ if (!tsFileSchema.getType().equals(iotdbSchema.getType())) {
+ throw new LoadAnalyzeTypeMismatchException(
+ String.format(
+ "Data type mismatch for measurement %s%s%s, type in TsFile: %s, type in IoTDB: %s",
+ device,
+ TsFileConstant.PATH_SEPARATOR,
+ iotdbSchema.getMeasurementName(),
+ tsFileSchema.getType(),
+ iotdbSchema.getType()));
}
// check encoding
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java
index 01406b0fd06..c533b0043e9 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzerTest.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.queryengine.plan.analyze.load;
+import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.queryengine.plan.relational.metadata.ColumnSchema;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -26,7 +27,10 @@ import org.apache.iotdb.db.exception.load.LoadAnalyzeTypeMismatchException;
import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.QueryId;
+import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree;
+import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.tsfile.enums.ColumnCategory;
@@ -149,6 +153,51 @@ public class LoadTsFileAnalyzerTest {
}
}
+ @Test
+ public void testTreeSchemaVerifierShouldThrowMismatchWhenVerifyingDataType() throws Exception {
+ final File tsFile = new File("load-tree-type-mismatch.tsfile");
+ if (tsFile.exists()) {
+ Assert.assertTrue(tsFile.delete());
+ }
+ Assert.assertTrue(tsFile.createNewFile());
+
+ try (final LoadTsFileAnalyzer analyzer =
+ new LoadTsFileAnalyzer(
+ LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath()),
+ false,
+ new MPPQueryContext(new QueryId("load_tree_test")))) {
+ final TreeSchemaAutoCreatorAndVerifier verifier =
+ new TreeSchemaAutoCreatorAndVerifier(analyzer);
+ try {
+ final IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d1");
+ final LoadTsFileTreeSchemaCache schemaCache = getTreeSchemaCache(verifier);
+ schemaCache.addTimeSeries(device, new MeasurementSchema("s1", TSDataType.BOOLEAN));
+ schemaCache.addIsAlignedCache(device, true, true);
+
+ final ClusterSchemaTree schemaTree = new ClusterSchemaTree();
+ schemaTree.appendSingleMeasurement(
+ new PartialPath("root.sg.d1.s1"),
+ new MeasurementSchema("s1", TSDataType.INT32),
+ null,
+ null,
+ null,
+ true);
+
+ final InvocationTargetException exception =
+ Assert.assertThrows(
+ InvocationTargetException.class,
+ () -> getVerifyTreeSchemaMethod().invoke(verifier, schemaTree));
+ Assert.assertTrue(exception.getCause() instanceof LoadAnalyzeTypeMismatchException);
+ } finally {
+ verifier.close();
+ }
+ } finally {
+ if (tsFile.exists()) {
+ Assert.assertTrue(tsFile.delete());
+ }
+ }
+ }
+
private void writeTableTsFileWithMixedDevices(final File tsFile) throws Exception {
if (tsFile.exists()) {
Assert.assertTrue(tsFile.delete());
@@ -203,6 +252,14 @@ public class LoadTsFileAnalyzerTest {
tableSchemaCacheField.set(analyzer, schemaCache);
}
+ private LoadTsFileTreeSchemaCache getTreeSchemaCache(
+ final TreeSchemaAutoCreatorAndVerifier verifier) throws Exception {
+ final Field schemaCacheField =
+ TreeSchemaAutoCreatorAndVerifier.class.getDeclaredField("schemaCache");
+ schemaCacheField.setAccessible(true);
+ return (LoadTsFileTreeSchemaCache) schemaCacheField.get(verifier);
+ }
+
private LoadTsFileTableSchemaCache createTableSchemaCache(final boolean shouldVerifyDataType)
throws LoadRuntimeOutOfMemoryException {
return new LoadTsFileTableSchemaCache(
@@ -219,6 +276,13 @@ public class LoadTsFileAnalyzerTest {
return method;
}
+ private Method getVerifyTreeSchemaMethod() throws NoSuchMethodException {
+ final Method method =
+ TreeSchemaAutoCreatorAndVerifier.class.getDeclaredMethod("verifySchema", ISchemaTree.class);
+ method.setAccessible(true);
+ return method;
+ }
+
private org.apache.iotdb.commons.queryengine.plan.relational.metadata.TableSchema
createTableSchema(final TSDataType fieldType) {
return new org.apache.iotdb.commons.queryengine.plan.relational.metadata.TableSchema(
(iotdb) 01/06: Propagate snapshot load failure during IoTConsensus AddPeer (#17935)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch rc/2.0.10
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1c692299ad0caff0bdad14138bed3a2c616d6081
Author: Yongzao <yo...@apache.org>
AuthorDate: Tue Jun 16 11:30:46 2026 +0800
Propagate snapshot load failure during IoTConsensus AddPeer (#17935)
---
.../statemachine/ConfigRegionStateMachine.java | 10 +-
.../persistence/executor/ConfigPlanExecutor.java | 7 +-
.../org/apache/iotdb/consensus/IStateMachine.java | 5 +-
.../consensus/iot/IoTConsensusServerImpl.java | 31 +-
.../service/IoTConsensusRPCServiceProcessor.java | 12 +-
.../ratis/ApplicationStateMachineProxy.java | 13 +-
.../simple/SimpleConsensusServerImpl.java | 4 +-
.../apache/iotdb/consensus/EmptyStateMachine.java | 4 +-
.../iot/AddPeerSnapshotLoadFailureTest.java | 354 +++++++++++++++++++++
.../iotdb/consensus/iot/util/TestStateMachine.java | 4 +-
.../apache/iotdb/consensus/ratis/TestUtils.java | 4 +-
.../consensus/simple/SimpleConsensusTest.java | 4 +-
.../dataregion/DataRegionStateMachine.java | 6 +-
.../schemaregion/SchemaRegionStateMachine.java | 25 +-
.../schemaengine/schemaregion/ISchemaRegion.java | 10 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 6 +-
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 6 +-
.../schemaRegion/SchemaRegionManagementTest.java | 28 +-
18 files changed, 493 insertions(+), 40 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
index b3029e66028..ad0a82bcf56 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/ConfigRegionStateMachine.java
@@ -246,9 +246,14 @@ public class ConfigRegionStateMachine implements IStateMachine, IStateMachine.Ev
}
@Override
- public void loadSnapshot(final File latestSnapshotRootDir) {
+ public boolean loadSnapshot(final File latestSnapshotRootDir) {
+ // The boolean result must reflect whether the ConfigRegion state-machine data was loaded, so
+ // callers (e.g. the AddPeer flow) can detect a real failure. The pipe-listener recomputation
+ // below is best-effort post-processing: a failure there is logged but must NOT be reported as a
+ // snapshot-load failure, otherwise it would (e.g.) abort ConfigNode (re)initialization on what
+ // is actually a healthy data load.
+ final boolean loadSucceeded = executor.loadSnapshot(latestSnapshotRootDir);
try {
- executor.loadSnapshot(latestSnapshotRootDir);
// We recompute the snapshot for pipe listener when loading snapshot
// to recover the newest snapshot in cache
PipeConfigNodeAgent.runtime()
@@ -261,6 +266,7 @@ public class ConfigRegionStateMachine implements IStateMachine, IStateMachine.Ev
e);
}
}
+ return loadSucceeded;
}
@Override
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
index 1f2729bff39..e37c3ae9960 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
@@ -767,12 +767,12 @@ public class ConfigPlanExecutor {
return result.get();
}
- public void loadSnapshot(final File latestSnapshotRootDir) {
+ public boolean loadSnapshot(final File latestSnapshotRootDir) {
if (!latestSnapshotRootDir.exists()) {
LOGGER.error(
ConfigNodeMessages.SNAPSHOT_DIRECTORY_IS_NOT_EXIST_CAN_NOT_LOAD_SNAPSHOT_WITH,
latestSnapshotRootDir.getAbsolutePath());
- return;
+ return false;
}
final AtomicBoolean result = new AtomicBoolean(true);
@@ -800,6 +800,9 @@ public class ConfigPlanExecutor {
ConfigNodeMessages.CONFIGNODESNAPSHOT_LOAD_SNAPSHOT_SUCCESS_LATESTSNAPSHOTROOTDIR,
latestSnapshotRootDir);
}
+ // Propagate any snapshot-load failure so callers (e.g. the AddPeer flow) do not treat a
+ // partially or wholly failed load as success.
+ return result.get();
}
private DataSet getSchemaNodeManagementPartition(ConfigPhysicalPlan req) {
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java
index 3354c83699b..c7705d93896 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java
@@ -112,8 +112,11 @@ public interface IStateMachine {
* Load the latest snapshot from given dir.
*
* @param latestSnapshotRootDir dir where the latest snapshot sits
+ * @return {@code true} if the snapshot was loaded successfully, {@code false} otherwise. Callers
+ * (e.g. the IoTConsensus AddPeer flow) rely on this to avoid activating a new peer whose
+ * snapshot failed to load, which would otherwise silently lose data.
*/
- void loadSnapshot(File latestSnapshotRootDir);
+ boolean loadSnapshot(File latestSnapshotRootDir);
/**
* given a snapshot dir, ask statemachine to provide all snapshot files. By default, it will list
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java
index 33ff3a4c273..7f28d24cd53 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java
@@ -447,14 +447,29 @@ public class IoTConsensusServerImpl {
}
}
- public void loadSnapshot(String snapshotId) {
- // TODO: (xingtanzjr) throw exception if the snapshot load failed
- recvFolderManager
- .getFolders()
- .forEach(
- dir -> {
- stateMachine.loadSnapshot(getSnapshotPath(dir, snapshotId));
- });
+ public boolean loadSnapshot(String snapshotId) {
+ // Snapshot fragments are spread across the receive folders by the FolderManager (a DataRegion,
+ // for example, uses one receive folder per local data dir), so a given snapshot only exists
+ // under the folders that actually received fragments. Load from those folders and skip the
+ // others; otherwise the state machine would fail on a folder that never received this snapshot
+ // and turn a healthy multi-data-dir transfer into a spurious failure.
+ //
+ // Note: an empty region produces a snapshot with zero fragments, so none of the receive folders
+ // contains it. That is a legitimate (no-op) load, not a failure, so an absent snapshot must not
+ // be reported as failure here.
+ for (String dir : recvFolderManager.getFolders()) {
+ File snapshotDir = getSnapshotPath(dir, snapshotId);
+ if (!snapshotDir.exists()) {
+ continue;
+ }
+ if (!stateMachine.loadSnapshot(snapshotDir)) {
+ // Stop at the first failure. The snapshot is already broken on this replica, and loading
+ // the remaining folders is both pointless and harmful: a load wipes the data dirs before
+ // relinking. Report the failure so the AddPeer coordinator does not activate this peer.
+ return false;
+ }
+ }
+ return true;
}
private File getSnapshotPath(String curStorageDir, String snapshotRelativePath) {
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java
index c5babea464d..fe804e9d7d0 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/service/IoTConsensusRPCServiceProcessor.java
@@ -312,7 +312,17 @@ public class IoTConsensusRPCServiceProcessor implements IoTConsensusIService.Ifa
status.setMessage(message);
return new TTriggerSnapshotLoadRes(status);
}
- impl.loadSnapshot(req.snapshotId);
+ if (!impl.loadSnapshot(req.snapshotId)) {
+ String message =
+ String.format(
+ "Failed to load snapshot %s for consensus group %s", req.snapshotId, groupId);
+ LOGGER.error(message);
+ // Surface a region-migration-specific code (the snapshot load runs as part of the AddPeer
+ // flow) rather than a generic internal error, so the coordinator's failure is meaningful.
+ TSStatus status = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
+ status.setMessage(message);
+ return new TTriggerSnapshotLoadRes(status);
+ }
KillPoint.setKillPoint(DataNodeKillPoints.DESTINATION_ADD_PEER_TRANSITION);
return new TTriggerSnapshotLoadRes(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
index 1134d8fd6f2..3e2451e65dd 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
@@ -98,7 +98,7 @@ public class ApplicationStateMachineProxy extends BaseStateMachine {
}
@Override
- public void reinitialize() {
+ public void reinitialize() throws IOException {
setLastAppliedTermIndex(null);
loadSnapshot(snapshotStorage.findLatestSnapshotDir());
if (getLifeCycleState() == LifeCycle.State.PAUSED) {
@@ -261,14 +261,21 @@ public class ApplicationStateMachineProxy extends BaseStateMachine {
}
}
- private void loadSnapshot(File latestSnapshotDir) {
+ private void loadSnapshot(File latestSnapshotDir) throws IOException {
snapshotStorage.updateSnapshotCache();
if (latestSnapshotDir == null) {
return;
}
// require the application statemachine to load the latest snapshot
- applicationStateMachine.loadSnapshot(latestSnapshotDir);
+ if (!applicationStateMachine.loadSnapshot(latestSnapshotDir)) {
+ // The application state machine rejected this snapshot. Do not advance lastAppliedTermIndex:
+ // claiming the snapshot as applied would let Ratis proceed as if it were installed and run on
+ // incomplete data (silent data loss). Fail (re)initialization instead so the snapshot install
+ // is treated as failed and can be retried.
+ throw new IOException(
+ String.format("%s: failed to load snapshot from %s", this, latestSnapshotDir));
+ }
TermIndex snapshotTermIndex = Utils.getTermIndexFromDir(latestSnapshotDir);
updateLastAppliedTermIndex(snapshotTermIndex.getTerm(), snapshotTermIndex.getIndex());
}
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/simple/SimpleConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/simple/SimpleConsensusServerImpl.java
index 4ed4a7b41ce..b76bcdeaaeb 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/simple/SimpleConsensusServerImpl.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/simple/SimpleConsensusServerImpl.java
@@ -89,7 +89,7 @@ public class SimpleConsensusServerImpl implements IStateMachine {
}
@Override
- public synchronized void loadSnapshot(File latestSnapshotRootDir) {
- stateMachine.loadSnapshot(latestSnapshotRootDir);
+ public synchronized boolean loadSnapshot(File latestSnapshotRootDir) {
+ return stateMachine.loadSnapshot(latestSnapshotRootDir);
}
}
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/EmptyStateMachine.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/EmptyStateMachine.java
index aafa0be5bb5..997120b00f9 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/EmptyStateMachine.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/EmptyStateMachine.java
@@ -54,5 +54,7 @@ public class EmptyStateMachine implements IStateMachine, IStateMachine.EventApi
}
@Override
- public void loadSnapshot(File latestSnapshotRootDir) {}
+ public boolean loadSnapshot(File latestSnapshotRootDir) {
+ return true;
+ }
}
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/AddPeerSnapshotLoadFailureTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/AddPeerSnapshotLoadFailureTest.java
new file mode 100644
index 00000000000..64c81a84b58
--- /dev/null
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/AddPeerSnapshotLoadFailureTest.java
@@ -0,0 +1,354 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.consensus.iot;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.consensus.ConsensusGroupId;
+import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.commons.exception.StartupException;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.consensus.common.Peer;
+import org.apache.iotdb.consensus.config.ConsensusConfig;
+import org.apache.iotdb.consensus.exception.ConsensusException;
+import org.apache.iotdb.consensus.iot.util.TestEntry;
+import org.apache.iotdb.consensus.iot.util.TestStateMachine;
+
+import org.apache.ratis.util.FileUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Regression test for the snapshot-load-failure bug (a failed snapshot load on the AddPeer target
+ * was silently swallowed, so the new peer was activated and the migration falsely reported
+ * successful, losing data on the new replica).
+ *
+ * <p>It builds a real two-node IoTConsensus group, forces the target peer's {@link
+ * org.apache.iotdb.consensus.IStateMachine#loadSnapshot} to fail, and verifies that {@code
+ * addRemotePeer}:
+ *
+ * <ul>
+ * <li>actually reaches the snapshot-load step (so the failure is the one under test, not an
+ * earlier step),
+ * <li>fails with a {@link ConsensusException} instead of silently succeeding,
+ * <li>does not leave the target peer active with an incompletely-loaded snapshot.
+ * </ul>
+ */
+public class AddPeerSnapshotLoadFailureTest {
+
+ private final Logger logger = LoggerFactory.getLogger(AddPeerSnapshotLoadFailureTest.class);
+
+ private final ConsensusGroupId gid = new DataRegionId(1);
+
+ private final int basePort = 9200;
+
+ private final List<Peer> peers =
+ Arrays.asList(
+ new Peer(gid, 1, new TEndPoint("127.0.0.1", basePort - 1)),
+ new Peer(gid, 2, new TEndPoint("127.0.0.1", basePort)));
+
+ private final List<File> peersStorage =
+ Arrays.asList(
+ new File("target" + File.separator + "snapshot-load-fail-1"),
+ new File("target" + File.separator + "snapshot-load-fail-2"));
+
+ private final List<List<String>> peersRecvSnapshotDirs =
+ Arrays.asList(
+ Arrays.asList(
+ "target" + File.separator + "snapshot-load-fail-1-recv-1",
+ "target" + File.separator + "snapshot-load-fail-1-recv-2"),
+ Arrays.asList(
+ "target" + File.separator + "snapshot-load-fail-2-recv-1",
+ "target" + File.separator + "snapshot-load-fail-2-recv-2"));
+
+ private final List<IoTConsensus> servers = new ArrayList<>();
+ private final List<ControllableStateMachine> stateMachines = new ArrayList<>();
+
+ /**
+ * A {@link TestStateMachine} that takes a real (single-file) snapshot and whose snapshot load can
+ * be made to fail on demand. When not forced to fail, {@link #loadSnapshot} mirrors the real
+ * {@code SnapshotLoader} contract by reporting failure when the snapshot root does not exist —
+ * which is exactly what happens for the receive folders that never got a fragment of a snapshot
+ * in a multi-data-dir deployment.
+ */
+ private static class ControllableStateMachine extends TestStateMachine {
+ private volatile boolean failLoadSnapshot = false;
+ private volatile boolean loadSnapshotInvoked = false;
+ private volatile boolean emptySnapshot = false;
+
+ void setFailLoadSnapshot(boolean failLoadSnapshot) {
+ this.failLoadSnapshot = failLoadSnapshot;
+ }
+
+ void setEmptySnapshot(boolean emptySnapshot) {
+ this.emptySnapshot = emptySnapshot;
+ }
+
+ boolean isLoadSnapshotInvoked() {
+ return loadSnapshotInvoked;
+ }
+
+ @Override
+ public boolean loadSnapshot(File latestSnapshotRootDir) {
+ loadSnapshotInvoked = true;
+ if (failLoadSnapshot) {
+ return false;
+ }
+ // Mirror SnapshotLoader: a receive folder that never received a fragment of this snapshot has
+ // no snapshot root, so loading from it must report failure.
+ return latestSnapshotRootDir.exists();
+ }
+
+ @Override
+ public boolean takeSnapshot(File snapshotDir) {
+ if (emptySnapshot) {
+ // Mirror an empty region: the snapshot has no fragments to transmit. The receiver then
+ // materializes no snapshot under any receive folder, which must still load successfully.
+ return true;
+ }
+ // Write a real (single) snapshot file so the transfer actually moves data and the receiver
+ // materializes the snapshot under a subset of its receive folders.
+ try {
+ Files.write(
+ new File(snapshotDir, "snapshot.data").toPath(),
+ "snapshot".getBytes(StandardCharsets.UTF_8));
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ // TestStateMachine does not implement clearSnapshot (the IStateMachine default throws). The
+ // AddPeer flow calls it in a finally block to clean up the local snapshot, so we provide a
+ // no-op here; otherwise that cleanup would mask the ConsensusException we are asserting on.
+ @Override
+ public boolean clearSnapshot() {
+ return true;
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ for (File file : peersStorage) {
+ file.mkdirs();
+ stateMachines.add(new ControllableStateMachine());
+ }
+ peersRecvSnapshotDirs.forEach(innerList -> innerList.forEach(dir -> new File(dir).mkdirs()));
+ initServer();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ servers.parallelStream().forEach(IoTConsensus::stop);
+ servers.clear();
+ for (File file : peersStorage) {
+ FileUtils.deleteFully(file);
+ }
+ peersRecvSnapshotDirs.forEach(
+ innerList ->
+ innerList.forEach(
+ dir -> {
+ try {
+ FileUtils.deleteFully(new File(dir));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }));
+ }
+
+ private void initServer() throws IOException {
+ Assume.assumeTrue(checkPortAvailable());
+ try {
+ for (int i = 0; i < peers.size(); i++) {
+ int finalI = i;
+ servers.add(
+ (IoTConsensus)
+ ConsensusFactory.getConsensusImpl(
+ ConsensusFactory.IOT_CONSENSUS,
+ ConsensusConfig.newBuilder()
+ .setThisNodeId(peers.get(i).getNodeId())
+ .setThisNode(peers.get(i).getEndpoint())
+ .setStorageDir(peersStorage.get(i).getAbsolutePath())
+ .setRecvSnapshotDirs(peersRecvSnapshotDirs.get(i))
+ .setConsensusGroupType(TConsensusGroupType.DataRegion)
+ .build(),
+ groupId -> stateMachines.get(finalI))
+ .orElseThrow(
+ () ->
+ new IllegalArgumentException(
+ String.format(
+ ConsensusFactory.CONSTRUCT_FAILED_MSG,
+ ConsensusFactory.IOT_CONSENSUS))));
+ }
+ for (int i = 0; i < peers.size(); i++) {
+ servers.get(i).start();
+ }
+ } catch (IOException e) {
+ if (e.getCause() instanceof StartupException) {
+ // just succeed when can not bind socket
+ logger.info("Can not start IoTConsensus because", e);
+ Assume.assumeTrue(false);
+ } else {
+ logger.error("Failed because", e);
+ Assert.fail("Failed because " + e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void addRemotePeerMustFailWhenTargetSnapshotLoadFails() throws Exception {
+ // node 0 is the sole initial member; node 1 will be added as a new peer. Mirroring the real
+ // region-migration flow, the destination peer (node 1) is pre-created locally with the full
+ // target peer list (IoTConsensus, unlike Ratis, requires a non-empty peer list here).
+ servers.get(0).createLocalPeer(gid, peers.subList(0, 1));
+ servers.get(1).createLocalPeer(gid, peers);
+
+ // Put some data into the group so the snapshot transfer is meaningful.
+ for (int i = 0; i < 10; i++) {
+ servers.get(0).write(gid, new TestEntry(i, peers.get(0)));
+ }
+
+ // Force the target peer (node 1) to fail loading the transferred snapshot.
+ stateMachines.get(1).setFailLoadSnapshot(true);
+
+ // Before the fix, addRemotePeer swallowed the load failure and returned normally, leaving the
+ // target peer active with incomplete data. It must now surface the failure.
+ Assert.assertThrows(
+ ConsensusException.class, () -> servers.get(0).addRemotePeer(gid, peers.get(1)));
+
+ // The failure must be the snapshot load itself, i.e. the AddPeer flow actually reached the
+ // load step on the target rather than aborting earlier.
+ Assert.assertTrue(
+ "Target peer's loadSnapshot was never invoked; the failure came from an earlier step",
+ stateMachines.get(1).isLoadSnapshotInvoked());
+
+ // The target peer must not be left active with an incompletely-loaded snapshot.
+ Assert.assertFalse(
+ "Target peer was activated despite a failed snapshot load",
+ servers.get(1).getImpl(gid).isActive());
+ }
+
+ /**
+ * A target peer configures one receive folder per local data dir, and snapshot fragments are
+ * spread across those folders by the FolderManager, so a small snapshot only materializes under a
+ * subset of them. A healthy transfer must therefore still succeed even though some receive
+ * folders never received any fragment of the snapshot (regression for the multi-data-dir
+ * false-failure reported on the load-failure-propagation PR).
+ */
+ @Test
+ public void addRemotePeerSucceedsWhenSnapshotSpansSubsetOfRecvDirs() throws Exception {
+ servers.get(0).createLocalPeer(gid, peers.subList(0, 1));
+ servers.get(1).createLocalPeer(gid, peers);
+
+ for (int i = 0; i < 10; i++) {
+ servers.get(0).write(gid, new TestEntry(i, peers.get(0)));
+ }
+
+ // Do NOT force a load failure: this is a healthy transfer. The (small) snapshot lands in only a
+ // subset of the target's receive folders, so the others legitimately have no snapshot root.
+ // Before the fix, loadSnapshot() loaded from every receive folder and treated the missing ones
+ // as failures, turning this healthy multi-data-dir transfer into a spurious failure.
+ servers.get(0).addRemotePeer(gid, peers.get(1));
+
+ Assert.assertTrue(
+ "Target peer's loadSnapshot was never invoked",
+ stateMachines.get(1).isLoadSnapshotInvoked());
+ Assert.assertTrue(
+ "Target peer was not activated after a successful snapshot load",
+ servers.get(1).getImpl(gid).isActive());
+
+ // Sanity-check that the test actually exercised a partial spread: at least one of the target's
+ // receive folders must hold no snapshot at all, otherwise the skipped-folder path is untested.
+ long emptyRecvFolders =
+ peersRecvSnapshotDirs.get(1).stream()
+ .map(dir -> new File(dir, IoTConsensusServerImpl.SNAPSHOT_DIR_NAME))
+ .filter(recvFolder -> isEmptyOrMissing(recvFolder))
+ .count();
+ Assert.assertTrue(
+ "Expected at least one receive folder without the snapshot, but every folder had it",
+ emptyRecvFolders > 0);
+ }
+
+ private static boolean isEmptyOrMissing(File dir) {
+ String[] children = dir.list();
+ return children == null || children.length == 0;
+ }
+
+ /**
+ * An empty region produces a snapshot with zero fragments, so nothing is transmitted and the
+ * target materializes no snapshot under any receive folder. This must still be treated as a
+ * successful (no-op) load — otherwise migrating an empty region (e.g. while removing a DataNode)
+ * fails. Regression for the multi-data-dir AddPeer false-failure on empty regions.
+ */
+ @Test
+ public void addRemotePeerSucceedsWhenSnapshotIsEmpty() throws Exception {
+ servers.get(0).createLocalPeer(gid, peers.subList(0, 1));
+ servers.get(1).createLocalPeer(gid, peers);
+
+ // No data is written, and the source takes an empty snapshot: zero fragments are transmitted.
+ stateMachines.get(0).setEmptySnapshot(true);
+
+ // Before the fix, loadSnapshot() reported failure when no receive folder contained the
+ // snapshot, so adding a peer for an empty region failed and DataNode removal timed out.
+ servers.get(0).addRemotePeer(gid, peers.get(1));
+
+ Assert.assertTrue(
+ "Target peer was not activated after an empty snapshot load",
+ servers.get(1).getImpl(gid).isActive());
+
+ // Confirm the test really exercised the empty-snapshot path: no receive folder holds a
+ // snapshot.
+ long nonEmptyRecvFolders =
+ peersRecvSnapshotDirs.get(1).stream()
+ .map(dir -> new File(dir, IoTConsensusServerImpl.SNAPSHOT_DIR_NAME))
+ .filter(recvFolder -> !isEmptyOrMissing(recvFolder))
+ .count();
+ Assert.assertEquals(
+ "Expected no receive folder to contain a snapshot for an empty region",
+ 0,
+ nonEmptyRecvFolders);
+ }
+
+ private boolean checkPortAvailable() {
+ for (Peer peer : this.peers) {
+ try (ServerSocket ignored = new ServerSocket(peer.getEndpoint().port)) {
+ logger.info("check port {} success for node {}", peer.getEndpoint().port, peer.getNodeId());
+ } catch (IOException e) {
+ logger.error("check port {} failed for node {}", peer.getEndpoint().port, peer.getNodeId());
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/util/TestStateMachine.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/util/TestStateMachine.java
index 54acdec30bb..2a2adb25461 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/util/TestStateMachine.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/util/TestStateMachine.java
@@ -136,5 +136,7 @@ public class TestStateMachine implements IStateMachine, IStateMachine.EventApi {
}
@Override
- public void loadSnapshot(File latestSnapshotRootDir) {}
+ public boolean loadSnapshot(File latestSnapshotRootDir) {
+ return true;
+ }
}
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/TestUtils.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/TestUtils.java
index 69725d68be6..b0d0ef14ee7 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/TestUtils.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/TestUtils.java
@@ -171,13 +171,15 @@ public class TestUtils {
}
@Override
- public void loadSnapshot(File latestSnapshotRootDir) {
+ public boolean loadSnapshot(File latestSnapshotRootDir) {
File snapshot =
new File(latestSnapshotRootDir.getAbsolutePath() + File.separator + "snapshot");
try (Scanner scanner = new Scanner(snapshot)) {
integer.set(Integer.parseInt(scanner.next()));
+ return true;
} catch (FileNotFoundException e) {
logger.error("cannot find snapshot file {}", snapshot);
+ return false;
}
}
diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/simple/SimpleConsensusTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/simple/SimpleConsensusTest.java
index d50ab992f6f..ff062fb63a8 100644
--- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/simple/SimpleConsensusTest.java
+++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/simple/SimpleConsensusTest.java
@@ -123,7 +123,9 @@ public class SimpleConsensusTest {
}
@Override
- public void loadSnapshot(File latestSnapshotRootDir) {}
+ public boolean loadSnapshot(File latestSnapshotRootDir) {
+ return true;
+ }
}
@Before
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
index 5c2c5207db6..0946756aaba 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
@@ -124,7 +124,7 @@ public class DataRegionStateMachine extends BaseStateMachine {
}
@Override
- public void loadSnapshot(File latestSnapshotRootDir) {
+ public boolean loadSnapshot(File latestSnapshotRootDir) {
String databaseName = region.getDatabaseName();
String dataRegionIdString = region.getDataRegionIdString();
DataRegionId regionId = new DataRegionId(Integer.parseInt(dataRegionIdString));
@@ -141,14 +141,16 @@ public class DataRegionStateMachine extends BaseStateMachine {
.loadSnapshotForStateMachine());
if (newRegion == null) {
logger.error(DataNodeMiscMessages.FAIL_LOAD_SNAPSHOT, latestSnapshotRootDir);
- return;
+ return false;
}
this.region = newRegion;
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();
BloomFilterCache.getInstance().clear();
+ return true;
} catch (Exception e) {
logger.error(DataNodeMiscMessages.EXCEPTION_REPLACING_DATA_REGION, e);
+ return false;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java
index edab5862d11..2f5ae61e85e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaRegionStateMachine.java
@@ -136,14 +136,23 @@ public class SchemaRegionStateMachine extends BaseStateMachine {
}
@Override
- public void loadSnapshot(final File latestSnapshotRootDir) {
- schemaRegion.loadSnapshot(latestSnapshotRootDir);
- PipeDataNodeAgent.runtime()
- .schemaListener(schemaRegion.getSchemaRegionId())
- .loadSnapshot(latestSnapshotRootDir);
- // We recompute the snapshot for pipe listener when loading snapshot
- // to recover the newest snapshot in cache
- listen2Snapshot4PipeListener(false);
+ public boolean loadSnapshot(final File latestSnapshotRootDir) {
+ try {
+ // The boolean result must reflect whether the schema-region data was actually loaded, so
+ // callers (e.g. the AddPeer flow and the Ratis snapshot-install path) can detect a real
+ // failure instead of treating a fallback-to-empty load as success.
+ final boolean loadSucceeded = schemaRegion.loadSnapshot(latestSnapshotRootDir);
+ PipeDataNodeAgent.runtime()
+ .schemaListener(schemaRegion.getSchemaRegionId())
+ .loadSnapshot(latestSnapshotRootDir);
+ // We recompute the snapshot for pipe listener when loading snapshot
+ // to recover the newest snapshot in cache
+ listen2Snapshot4PipeListener(false);
+ return loadSucceeded;
+ } catch (Exception e) {
+ logger.error("Failed to load snapshot from {}", latestSnapshotRootDir, e);
+ return false;
+ }
}
public void listen2Snapshot4PipeListener(final boolean isTmp) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java
index 8d87435e108..540d5d4274c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java
@@ -115,7 +115,15 @@ public interface ISchemaRegion {
boolean createSnapshot(final File snapshotDir);
- void loadSnapshot(final File latestSnapshotRootDir);
+ /**
+ * Load the latest schema snapshot from the given dir.
+ *
+ * @return {@code true} if the snapshot was loaded successfully, {@code false} if loading failed
+ * (in which case the region falls back to an empty/re-initialized state). Callers rely on
+ * this to honor the {@link org.apache.iotdb.consensus.IStateMachine#loadSnapshot}
+ * success/failure contract (e.g. the AddPeer flow and Ratis snapshot install).
+ */
+ boolean loadSnapshot(final File latestSnapshotRootDir);
// endregion
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
index 0394a81680f..1f2408cca12 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
@@ -556,7 +556,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
// currently, this method is only used for cluster-ratis mode
@Override
- public void loadSnapshot(final File latestSnapshotRootDir) {
+ public boolean loadSnapshot(final File latestSnapshotRootDir) {
clear();
logger.info(DataNodeSchemaMessages.START_LOADING_SNAPSHOT, schemaRegionId);
@@ -651,6 +651,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
schemaRegionId,
System.currentTimeMillis() - startTime);
logger.info(DataNodeSchemaMessages.SUCCESSFULLY_LOAD_SNAPSHOT, schemaRegionId);
+ return true;
} catch (final Exception e) {
logger.error(
DataNodeSchemaMessages.FAILED_TO_LOAD_SNAPSHOT, schemaRegionId, e.getMessage(), e);
@@ -662,6 +663,9 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
logger.error(
DataNodeSchemaMessages.ERROR_DURING_INIT_SCHEMA_REGION, schemaRegionId, exception);
}
+ // The snapshot was not loaded (the region fell back to an empty re-initialized state). Report
+ // the failure so callers honoring the loadSnapshot success/failure contract can react.
+ return false;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
index df5ad41cff6..8d87ef3f061 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
@@ -534,7 +534,7 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion {
// currently, this method is only used for cluster-ratis mode
@Override
- public void loadSnapshot(File latestSnapshotRootDir) {
+ public boolean loadSnapshot(File latestSnapshotRootDir) {
clear();
logger.info(DataNodeSchemaMessages.START_LOADING_SNAPSHOT, schemaRegionId);
@@ -579,6 +579,7 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion {
schemaRegionId,
System.currentTimeMillis() - startTime);
logger.info(DataNodeSchemaMessages.SUCCESSFULLY_LOAD_SNAPSHOT, schemaRegionId);
+ return true;
} catch (IOException | MetadataException e) {
logger.error(
DataNodeSchemaMessages.FAILED_TO_LOAD_SNAPSHOT, schemaRegionId, e.getMessage(), e);
@@ -592,6 +593,9 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion {
schemaRegionId,
metadataException);
}
+ // The snapshot was not loaded (the region fell back to an empty re-initialized state). Report
+ // the failure so callers honoring the loadSnapshot success/failure contract can react.
+ return false;
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java
index 38f56b66dd9..0c0b3a5b953 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java
@@ -96,7 +96,7 @@ public class SchemaRegionManagementTest extends AbstractSchemaRegionTest {
snapshotDir.mkdir();
schemaRegion.createSnapshot(snapshotDir);
- schemaRegion.loadSnapshot(snapshotDir);
+ Assert.assertTrue(schemaRegion.loadSnapshot(snapshotDir));
List<ITimeSeriesSchemaInfo> result =
SchemaRegionTestUtil.showTimeseries(
@@ -112,7 +112,7 @@ public class SchemaRegionManagementTest extends AbstractSchemaRegionTest {
simulateRestart();
ISchemaRegion newSchemaRegion = getSchemaRegion("root.sg", 0);
- newSchemaRegion.loadSnapshot(snapshotDir);
+ Assert.assertTrue(newSchemaRegion.loadSnapshot(snapshotDir));
result =
SchemaRegionTestUtil.showTimeseries(
newSchemaRegion, new PartialPath("root.sg.**"), false, "tag-key", "tag-value");
@@ -171,7 +171,7 @@ public class SchemaRegionManagementTest extends AbstractSchemaRegionTest {
snapshotDir.mkdir();
schemaRegion.createSnapshot(snapshotDir);
- schemaRegion.loadSnapshot(snapshotDir);
+ Assert.assertTrue(schemaRegion.loadSnapshot(snapshotDir));
List<ITimeSeriesSchemaInfo> result =
SchemaRegionTestUtil.showTimeseries(
@@ -182,7 +182,7 @@ public class SchemaRegionManagementTest extends AbstractSchemaRegionTest {
simulateRestart();
ISchemaRegion newSchemaRegion = getSchemaRegion("root.sg", 0);
- newSchemaRegion.loadSnapshot(snapshotDir);
+ Assert.assertTrue(newSchemaRegion.loadSnapshot(snapshotDir));
result =
SchemaRegionTestUtil.showTimeseries(
newSchemaRegion, new PartialPath("root.sg.**"), false, "tag-key", "tag-value");
@@ -193,6 +193,26 @@ public class SchemaRegionManagementTest extends AbstractSchemaRegionTest {
}
}
+ @Test
+ public void testLoadSnapshotReportsFailureWhenSnapshotIsMissing() throws Exception {
+ String schemaRegionConsensusProtocolClass = config.getSchemaRegionConsensusProtocolClass();
+ config.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS);
+ try {
+ ISchemaRegion schemaRegion = getSchemaRegion("root.sg", 0);
+
+ // Loading from a directory that does not contain a snapshot must report failure rather than
+ // silently falling back to an empty region and reporting success. Callers (the AddPeer flow
+ // and the Ratis snapshot-install path) rely on this success/failure contract.
+ File missingSnapshotDir =
+ new File(config.getSchemaDir() + File.separator + "non-existent-snapshot");
+ Assert.assertFalse(missingSnapshotDir.exists());
+
+ Assert.assertFalse(schemaRegion.loadSnapshot(missingSnapshotDir));
+ } finally {
+ config.setSchemaRegionConsensusProtocolClass(schemaRegionConsensusProtocolClass);
+ }
+ }
+
@Test
@Ignore
public void testSnapshotPerformance() throws Exception {
(iotdb) 03/06: Fix aggregate write-back output database metadata (#17938)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch rc/2.0.10
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c7ba5da546a3e18816d974c765769200094eaa30
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Tue Jun 16 16:14:15 2026 +0800
Fix aggregate write-back output database metadata (#17938)
* Fix aggregate write-back output database metadata
* Fixed i18n
---
.../db/pipe/event/common/row/PipeRowCollector.java | 24 +++++++++----
.../common/tablet/PipeRawTabletEventConverter.java | 42 ++++++++++++++++++----
.../event/common/tablet/PipeTabletCollector.java | 23 +++++++++---
.../processor/aggregate/AggregateProcessor.java | 13 ++++++-
.../pipe/event/PipeTabletInsertionEventTest.java | 32 +++++++++++++++++
.../apache/iotdb/commons/i18n/CommonMessages.java | 8 +++++
.../apache/iotdb/commons/i18n/CommonMessages.java | 8 +++++
.../iotdb/commons/partition/DataPartition.java | 18 ++++++----
8 files changed, 143 insertions(+), 25 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java
index 326d9c7d31e..efd442fe428 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.pipe.event.common.row;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.db.i18n.DataNodePipeMessages;
-import org.apache.iotdb.db.pipe.event.common.PipeInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletEventConverter;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils;
@@ -56,6 +55,22 @@ public class PipeRowCollector extends PipeRawTabletEventConverter implements Row
super(pipeTaskMeta, sourceEvent, sourceEventDataBase, isTableModel);
}
+ public PipeRowCollector(
+ PipeTaskMeta pipeTaskMeta,
+ EnrichedEvent sourceEvent,
+ String sourceEventDataBase,
+ Boolean isTableModel,
+ String rawTableModelDataBaseName,
+ String rawTreeModelDataBaseName) {
+ super(
+ pipeTaskMeta,
+ sourceEvent,
+ sourceEventDataBase,
+ isTableModel,
+ rawTableModelDataBaseName,
+ rawTreeModelDataBaseName);
+ }
+
@Override
public void collectRow(Row row) {
if (!(row instanceof PipeRow)) {
@@ -106,15 +121,12 @@ public class PipeRowCollector extends PipeRawTabletEventConverter implements Row
private void collectTabletInsertionEvent() {
if (tablet != null) {
PipeTabletUtils.compactBitMaps(tablet);
- // TODO: non-PipeInsertionEvent sourceEvent is not supported?
- final PipeInsertionEvent pipeInsertionEvent =
- sourceEvent instanceof PipeInsertionEvent ? ((PipeInsertionEvent) sourceEvent) : null;
tabletInsertionEventList.add(
new PipeRawTabletInsertionEvent(
isTableModel,
sourceEventDataBaseName,
- pipeInsertionEvent == null ? null : pipeInsertionEvent.getRawTableModelDataBase(),
- pipeInsertionEvent == null ? null : pipeInsertionEvent.getRawTreeModelDataBase(),
+ rawTableModelDataBaseName,
+ rawTreeModelDataBaseName,
tablet,
isAligned,
sourceEvent == null ? null : sourceEvent.getPipeName(),
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletEventConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletEventConverter.java
index 4f387e71883..6a356aeca6d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletEventConverter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletEventConverter.java
@@ -33,19 +33,25 @@ public abstract class PipeRawTabletEventConverter implements DataCollector {
protected boolean isAligned = false;
protected final PipeTaskMeta pipeTaskMeta; // Used to report progress
protected final EnrichedEvent sourceEvent; // Used to report progress
- protected final String sourceEventDataBaseName;
- protected final Boolean isTableModel;
+ protected String sourceEventDataBaseName;
+ protected Boolean isTableModel;
+ protected String rawTableModelDataBaseName;
+ protected String rawTreeModelDataBaseName;
public PipeRawTabletEventConverter(PipeTaskMeta pipeTaskMeta, EnrichedEvent sourceEvent) {
this.pipeTaskMeta = pipeTaskMeta;
this.sourceEvent = sourceEvent;
if (sourceEvent instanceof PipeInsertionEvent) {
- sourceEventDataBaseName =
- ((PipeInsertionEvent) sourceEvent).getSourceDatabaseNameFromDataRegion();
- isTableModel = ((PipeInsertionEvent) sourceEvent).getRawIsTableModelEvent();
+ final PipeInsertionEvent pipeInsertionEvent = (PipeInsertionEvent) sourceEvent;
+ sourceEventDataBaseName = pipeInsertionEvent.getSourceDatabaseNameFromDataRegion();
+ isTableModel = pipeInsertionEvent.getRawIsTableModelEvent();
+ rawTableModelDataBaseName = pipeInsertionEvent.getRawTableModelDataBase();
+ rawTreeModelDataBaseName = pipeInsertionEvent.getRawTreeModelDataBase();
} else {
sourceEventDataBaseName = null;
isTableModel = null;
+ rawTableModelDataBaseName = null;
+ rawTreeModelDataBaseName = null;
}
}
@@ -54,12 +60,34 @@ public abstract class PipeRawTabletEventConverter implements DataCollector {
EnrichedEvent sourceEvent,
String sourceEventDataBase,
Boolean isTableModel) {
- this.pipeTaskMeta = pipeTaskMeta;
- this.sourceEvent = sourceEvent;
+ this(pipeTaskMeta, sourceEvent);
this.sourceEventDataBaseName = sourceEventDataBase;
this.isTableModel = isTableModel;
}
+ public PipeRawTabletEventConverter(
+ PipeTaskMeta pipeTaskMeta,
+ EnrichedEvent sourceEvent,
+ String sourceEventDataBase,
+ Boolean isTableModel,
+ String rawTableModelDataBaseName,
+ String rawTreeModelDataBaseName) {
+ this(pipeTaskMeta, sourceEvent, sourceEventDataBase, isTableModel);
+ this.rawTableModelDataBaseName = rawTableModelDataBaseName;
+ this.rawTreeModelDataBaseName = rawTreeModelDataBaseName;
+ }
+
+ public void resetDatabaseInfo(
+ final String sourceEventDataBaseName,
+ final Boolean isTableModel,
+ final String rawTableModelDataBaseName,
+ final String rawTreeModelDataBaseName) {
+ this.sourceEventDataBaseName = sourceEventDataBaseName;
+ this.isTableModel = isTableModel;
+ this.rawTableModelDataBaseName = rawTableModelDataBaseName;
+ this.rawTreeModelDataBaseName = rawTreeModelDataBaseName;
+ }
+
@Override
public List<TabletInsertionEvent> convertToTabletInsertionEvents(final boolean shouldReport) {
final int eventListSize = tabletInsertionEventList.size();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletCollector.java
index beacc54e705..b9da59f3111 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletCollector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletCollector.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.db.pipe.event.common.tablet;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
-import org.apache.iotdb.db.pipe.event.common.PipeInsertionEvent;
import org.apache.iotdb.pipe.api.collector.TabletCollector;
import org.apache.tsfile.write.record.Tablet;
@@ -40,16 +39,30 @@ public class PipeTabletCollector extends PipeRawTabletEventConverter implements
super(pipeTaskMeta, sourceEvent, sourceEventDataBase, isTableModel);
}
+ public PipeTabletCollector(
+ PipeTaskMeta pipeTaskMeta,
+ EnrichedEvent sourceEvent,
+ String sourceEventDataBase,
+ Boolean isTableModel,
+ String rawTableModelDataBaseName,
+ String rawTreeModelDataBaseName) {
+ super(
+ pipeTaskMeta,
+ sourceEvent,
+ sourceEventDataBase,
+ isTableModel,
+ rawTableModelDataBaseName,
+ rawTreeModelDataBaseName);
+ }
+
@Override
public void collectTablet(final Tablet tablet) {
- final PipeInsertionEvent pipeInsertionEvent =
- sourceEvent instanceof PipeInsertionEvent ? ((PipeInsertionEvent) sourceEvent) : null;
tabletInsertionEventList.add(
new PipeRawTabletInsertionEvent(
isTableModel,
sourceEventDataBaseName,
- pipeInsertionEvent == null ? null : pipeInsertionEvent.getRawTableModelDataBase(),
- pipeInsertionEvent == null ? null : pipeInsertionEvent.getRawTreeModelDataBase(),
+ rawTableModelDataBaseName,
+ rawTreeModelDataBaseName,
tablet,
isAligned,
sourceEvent == null ? null : sourceEvent.getPipeName(),
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
index b584db153d0..6d27b9f7dd5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java
@@ -116,6 +116,7 @@ public class AggregateProcessor implements PipeProcessor {
private PipeTaskMeta pipeTaskMeta;
private long outputMaxDelayMilliseconds;
private long outputMinReportIntervalMilliseconds;
+ private String outputDatabase;
private String outputDatabaseWithPathSeparator;
private final Map<String, AggregatedResultOperator> outputName2OperatorMap = new HashMap<>();
@@ -226,7 +227,7 @@ public class AggregateProcessor implements PipeProcessor {
PROCESSOR_OUTPUT_MIN_REPORT_INTERVAL_SECONDS_KEY,
PROCESSOR_OUTPUT_MIN_REPORT_INTERVAL_SECONDS_DEFAULT_VALUE)
* 1000;
- final String outputDatabase =
+ outputDatabase =
parameters.getStringOrDefault(
PROCESSOR_OUTPUT_DATABASE_KEY, PROCESSOR_OUTPUT_DATABASE_DEFAULT_VALUE);
outputDatabaseWithPathSeparator =
@@ -427,6 +428,8 @@ public class AggregateProcessor implements PipeProcessor {
final Row row, final RowCollector rowCollector, final AtomicReference<Exception> exception) {
final Map<String, Pair<Long, ByteBuffer>> resultMap = new HashMap<>();
+ resetOutputDatabaseForGeneratedEvent(rowCollector);
+
final long timestamp = row.getTime();
for (int index = 0, size = row.size(); index < size; ++index) {
// Do not calculate null values
@@ -580,6 +583,7 @@ public class AggregateProcessor implements PipeProcessor {
synchronized (stateReference) {
final PipeRowCollector rowCollector =
new PipeRowCollector(pipeTaskMeta, null, dataBaseName, isTableModel);
+ resetOutputDatabaseForGeneratedEvent(rowCollector);
try {
collectWindowOutputs(
stateReference.get().forceOutput(), timeSeries, rowCollector);
@@ -612,6 +616,13 @@ public class AggregateProcessor implements PipeProcessor {
eventCollector.collect(event);
}
+ private void resetOutputDatabaseForGeneratedEvent(final RowCollector rowCollector) {
+ if (!outputDatabase.isEmpty() && rowCollector instanceof PipeRowCollector) {
+ ((PipeRowCollector) rowCollector)
+ .resetDatabaseInfo(outputDatabase, Boolean.FALSE, null, outputDatabase);
+ }
+ }
+
/**
* Collect {@link WindowOutput}s of a single timeSeries in one turn. The {@link TSDataType}s shall
* be the same because the {@link AggregatedResultOperator}s shall return the same value for the
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
index da3dee91caa..052aab50732 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
@@ -31,6 +31,8 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern;
import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.db.auth.AuthorityChecker;
+import org.apache.iotdb.db.pipe.event.common.row.PipeResetTabletRow;
+import org.apache.iotdb.db.pipe.event.common.row.PipeRowCollector;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTreePatternParser;
@@ -52,6 +54,7 @@ import org.junit.Test;
import java.time.LocalDate;
import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
import static org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern.buildUnionPattern;
@@ -319,6 +322,35 @@ public class PipeTabletInsertionEventTest {
Assert.assertTrue(isAligned4);
}
+ @Test
+ public void collectRowWithOverriddenTreeDatabaseForTest() {
+ final PipeRowCollector rowCollector = new PipeRowCollector(null, null, "root.test.sg_0", false);
+ rowCollector.resetDatabaseInfo("root.userResultDB", false, null, "root.userResultDB");
+
+ final MeasurementSchema[] outputSchemas = {new MeasurementSchema("avg", TSDataType.INT32)};
+ rowCollector.collectRow(
+ new PipeResetTabletRow(
+ 0,
+ "root.userResultDB.d_0.s_1",
+ false,
+ outputSchemas,
+ new long[] {1L},
+ new TSDataType[] {TSDataType.INT32},
+ new Object[] {new int[] {1}},
+ null,
+ new String[] {"avg"}));
+
+ final List<org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent> events =
+ rowCollector.convertToTabletInsertionEvents(false);
+ Assert.assertEquals(1, events.size());
+
+ final PipeRawTabletInsertionEvent event = (PipeRawTabletInsertionEvent) events.get(0);
+ Assert.assertEquals("root.userResultDB", event.getSourceDatabaseNameFromDataRegion());
+ Assert.assertFalse(event.isTableModelEvent());
+ Assert.assertEquals("root.userResultDB", event.getTreeModelDatabaseName());
+ Assert.assertEquals("root.userResultDB.d_0.s_1", event.convertToTablet().getDeviceId());
+ }
+
@Test
public void convertToTabletSkipsUnnecessaryBitMapsForTest() throws Exception {
final BitMap[] bitMaps = new BitMap[schemas.length];
diff --git a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java
index 5a2e4196197..f470f876fcf 100644
--- a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java
+++ b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java
@@ -69,6 +69,14 @@ public final class CommonMessages {
// --- subscription ---
public static final String CONFIG_PRINT = "{}: {}";
+ // --- partition ---
+ public static final String DATABASE_NOT_EXISTS_AND_AUTO_CREATE_DISABLED =
+ "Database %s not exists and failed to create automatically because enable_auto_create_schema is FALSE.";
+ public static final String DATA_PARTITION_EMPTY =
+ "Data partition is empty. device: %s, seriesSlot: %s, database: %s";
+ public static final String TARGET_REGION_LIST_EMPTY =
+ "targetRegionList is empty. device: %s, timeSlot: %s";
+
// --- concurrent ---
public static final String EXCEPTION_IN_THREAD = "Exception in thread {}-{}";
public static final String INTERRUPTED_WHILE_AWAITING = "Interrupted while awaiting condition";
diff --git a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java
index a4b72b9e427..dc31b8296e9 100644
--- a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java
+++ b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java
@@ -68,6 +68,14 @@ public final class CommonMessages {
// --- subscription ---
public static final String CONFIG_PRINT = "{}:{}";
+ // --- partition ---
+ public static final String DATABASE_NOT_EXISTS_AND_AUTO_CREATE_DISABLED =
+ "Database %s 不存在,且由于 enable_auto_create_schema 为 FALSE,无法自动创建。";
+ public static final String DATA_PARTITION_EMPTY =
+ "数据分区为空。device: %s, seriesSlot: %s, database: %s";
+ public static final String TARGET_REGION_LIST_EMPTY =
+ "targetRegionList 为空。device: %s, timeSlot: %s";
+
// --- concurrent ---
public static final String EXCEPTION_IN_THREAD = "线程 {}-{} 中发生异常";
public static final String INTERRUPTED_WHILE_AWAITING = "等待条件时被中断";
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java
index 100c40eddcc..43085f04987 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.commons.partition;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
+import org.apache.iotdb.commons.i18n.CommonMessages;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.TimePartitionUtils;
@@ -221,15 +222,22 @@ public class DataPartition extends Partition {
final List<TRegionReplicaSet> dataRegionReplicaSets = new ArrayList<>();
final Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>>
dataBasePartitionMap = dataPartitionMap.get(databaseName);
+ if (dataBasePartitionMap == null) {
+ throw new RuntimeException(
+ String.format(CommonMessages.DATABASE_NOT_EXISTS_AND_AUTO_CREATE_DISABLED, databaseName));
+ }
final Map<TTimePartitionSlot, List<TRegionReplicaSet>> slotReplicaSetMap =
dataBasePartitionMap.get(seriesPartitionSlot);
+ if (slotReplicaSetMap == null) {
+ throw new RuntimeException(
+ String.format(
+ CommonMessages.DATA_PARTITION_EMPTY, deviceID, seriesPartitionSlot, databaseName));
+ }
for (final TTimePartitionSlot timePartitionSlot : timePartitionSlotList) {
final List<TRegionReplicaSet> targetRegionList = slotReplicaSetMap.get(timePartitionSlot);
if (targetRegionList == null || targetRegionList.isEmpty()) {
throw new RuntimeException(
- String.format(
- "targetRegionList is empty. device: %s, timeSlot: %s",
- deviceID, timePartitionSlot));
+ String.format(CommonMessages.TARGET_REGION_LIST_EMPTY, deviceID, timePartitionSlot));
} else {
dataRegionReplicaSets.add(targetRegionList.get(targetRegionList.size() - 1));
}
@@ -250,9 +258,7 @@ public class DataPartition extends Partition {
databasePartitionMap = dataPartitionMap.get(databaseName);
if (databasePartitionMap == null) {
throw new RuntimeException(
- "Database "
- + databaseName
- + " not exists and failed to create automatically because enable_auto_create_schema is FALSE.");
+ String.format(CommonMessages.DATABASE_NOT_EXISTS_AND_AUTO_CREATE_DISABLED, databaseName));
}
final List<TRegionReplicaSet> regions =
databasePartitionMap.get(seriesPartitionSlot).get(timePartitionSlot);
(iotdb) 06/06: Fix IoTConsensus multi-folder snapshot load for DataRegion (#17974)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch rc/2.0.10
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ecdc54728e5769b81e7b9e0708cf381e736cfad3
Author: Hongzhi Gao <76...@qq.com>
AuthorDate: Thu Jun 18 09:00:35 2026 +0800
Fix IoTConsensus multi-folder snapshot load for DataRegion (#17974)
Collect receive folders that actually hold snapshot fragments and load them in a single call. Loading per folder wipes data dirs before relinking, so repeated loads only kept the last folder's data. Add SnapshotLoader multi-dir support and regression tests.
---
.../org/apache/iotdb/consensus/IStateMachine.java | 23 +++++
.../consensus/iot/IoTConsensusServerImpl.java | 27 +++---
.../dataregion/DataRegionStateMachine.java | 44 +++++++---
.../dataregion/snapshot/SnapshotLoader.java | 47 +++++++++++
.../dataregion/snapshot/IoTDBSnapshotTest.java | 98 ++++++++++++++++++++++
5 files changed, 216 insertions(+), 23 deletions(-)
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java
index c7705d93896..c879c3bbb9b 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java
@@ -118,6 +118,29 @@ public interface IStateMachine {
*/
boolean loadSnapshot(File latestSnapshotRootDir);
+ /**
+ * Load the latest snapshot whose fragments are spread across several root dirs. A single snapshot
+ * may be received into more than one folder (e.g. IoTConsensus spreads a snapshot's fragments
+ * across one receive folder per local data dir), so the whole snapshot must be loaded in one call
+ * rather than once per folder: a state machine whose load wipes its data dirs before relinking
+ * (such as the DataRegion one) would otherwise have each per-folder load erase the fragments
+ * linked by the previous folders, leaving only the last folder's data.
+ *
+ * <p>The default implementation simply loads each dir in turn, which is correct only for state
+ * machines whose per-dir load is independent; such state machines should override this when their
+ * load is destructive across dirs.
+ *
+ * @param latestSnapshotRootDirs the dirs that actually hold fragments of the snapshot
+ * @return {@code true} if the snapshot was loaded successfully, {@code false} otherwise.
+ */
+ default boolean loadSnapshot(List<File> latestSnapshotRootDirs) {
+ boolean success = true;
+ for (File dir : latestSnapshotRootDirs) {
+ success = loadSnapshot(dir) && success;
+ }
+ return success;
+ }
+
/**
* given a snapshot dir, ask statemachine to provide all snapshot files. By default, it will list
* all files recursively under latestSnapshotDir
diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java
index 7f28d24cd53..651391eab01 100644
--- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java
+++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java
@@ -450,26 +450,29 @@ public class IoTConsensusServerImpl {
public boolean loadSnapshot(String snapshotId) {
// Snapshot fragments are spread across the receive folders by the FolderManager (a DataRegion,
// for example, uses one receive folder per local data dir), so a given snapshot only exists
- // under the folders that actually received fragments. Load from those folders and skip the
- // others; otherwise the state machine would fail on a folder that never received this snapshot
- // and turn a healthy multi-data-dir transfer into a spurious failure.
+ // under the folders that actually received fragments. Collect exactly those folders and hand
+ // them to the state machine in a single load call.
+ //
+ // It must be a single call rather than one call per folder: the state machine's load is
+ // destructive (a DataRegion load wipes the data dirs before relinking), so loading folders one
+ // at a time would make each load erase the fragments linked by the previous folders, leaving
+ // only the last folder's data. The state machine instead clears the data dirs once and relinks
+ // every folder's fragments together.
//
// Note: an empty region produces a snapshot with zero fragments, so none of the receive folders
// contains it. That is a legitimate (no-op) load, not a failure, so an absent snapshot must not
// be reported as failure here.
+ List<File> snapshotDirs = new ArrayList<>();
for (String dir : recvFolderManager.getFolders()) {
File snapshotDir = getSnapshotPath(dir, snapshotId);
- if (!snapshotDir.exists()) {
- continue;
- }
- if (!stateMachine.loadSnapshot(snapshotDir)) {
- // Stop at the first failure. The snapshot is already broken on this replica, and loading
- // the remaining folders is both pointless and harmful: a load wipes the data dirs before
- // relinking. Report the failure so the AddPeer coordinator does not activate this peer.
- return false;
+ if (snapshotDir.exists()) {
+ snapshotDirs.add(snapshotDir);
}
}
- return true;
+ if (snapshotDirs.isEmpty()) {
+ return true;
+ }
+ return stateMachine.loadSnapshot(snapshotDirs);
}
private File getSnapshotPath(String curStorageDir, String snapshotRelativePath) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
index 0946756aaba..aacde80c252 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataRegionStateMachine.java
@@ -52,6 +52,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.function.Supplier;
public class DataRegionStateMachine extends BaseStateMachine {
@@ -125,22 +126,43 @@ public class DataRegionStateMachine extends BaseStateMachine {
@Override
public boolean loadSnapshot(File latestSnapshotRootDir) {
- String databaseName = region.getDatabaseName();
+ final String databaseName = region.getDatabaseName();
+ final String dataRegionIdString = region.getDataRegionIdString();
+ return loadSnapshot(
+ () ->
+ new SnapshotLoader(
+ latestSnapshotRootDir.getAbsolutePath(), databaseName, dataRegionIdString)
+ .loadSnapshotForStateMachine(),
+ latestSnapshotRootDir);
+ }
+
+ @Override
+ public boolean loadSnapshot(List<File> latestSnapshotRootDirs) {
+ final String databaseName = region.getDatabaseName();
+ final String dataRegionIdString = region.getDataRegionIdString();
+ // A single snapshot is spread across several receive folders, and loading wipes the data dirs
+ // before relinking. It must therefore be loaded in one shot (clear once, relink every folder)
+ // rather than once per folder, otherwise each per-folder load would erase the previous folders'
+ // fragments and leave only the last one's data.
+ final List<String> snapshotRootPaths = new ArrayList<>();
+ for (File dir : latestSnapshotRootDirs) {
+ snapshotRootPaths.add(dir.getAbsolutePath());
+ }
+ return loadSnapshot(
+ () ->
+ new SnapshotLoader(snapshotRootPaths, databaseName, dataRegionIdString)
+ .loadSnapshotForStateMachine(),
+ latestSnapshotRootDirs);
+ }
+
+ private boolean loadSnapshot(Supplier<DataRegion> snapshotLoader, Object snapshotRootForLog) {
String dataRegionIdString = region.getDataRegionIdString();
DataRegionId regionId = new DataRegionId(Integer.parseInt(dataRegionIdString));
try {
DataRegion newRegion =
- StorageEngine.getInstance()
- .setDataRegionForSnapshotLoad(
- regionId,
- () ->
- new SnapshotLoader(
- latestSnapshotRootDir.getAbsolutePath(),
- databaseName,
- dataRegionIdString)
- .loadSnapshotForStateMachine());
+ StorageEngine.getInstance().setDataRegionForSnapshotLoad(regionId, snapshotLoader);
if (newRegion == null) {
- logger.error(DataNodeMiscMessages.FAIL_LOAD_SNAPSHOT, latestSnapshotRootDir);
+ logger.error(DataNodeMiscMessages.FAIL_LOAD_SNAPSHOT, snapshotRootForLog);
return false;
}
this.region = newRegion;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
index d39d964df7b..33f87187c91 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java
@@ -42,6 +42,7 @@ import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -52,11 +53,26 @@ public class SnapshotLoader {
private Logger LOGGER = LoggerFactory.getLogger(SnapshotLoader.class);
private String storageGroupName;
private String snapshotPath;
+ private List<String> snapshotPaths;
private String dataRegionId;
private SnapshotLogAnalyzer logAnalyzer;
public SnapshotLoader(String snapshotPath, String storageGroupName, String dataRegionId) {
this.snapshotPath = snapshotPath;
+ this.snapshotPaths = Collections.singletonList(snapshotPath);
+ this.storageGroupName = storageGroupName;
+ this.dataRegionId = dataRegionId;
+ }
+
+ /**
+ * A snapshot received by IoTConsensus is spread across several receive folders (one per local
+ * data dir), so loading it means relinking the fragments from all of them. The data dirs must be
+ * cleared exactly once, before relinking from any folder; see {@link
+ * #loadSnapshotFromMultipleDirs()}.
+ */
+ public SnapshotLoader(List<String> snapshotPaths, String storageGroupName, String dataRegionId) {
+ this.snapshotPaths = snapshotPaths;
+ this.snapshotPath = snapshotPaths.isEmpty() ? null : snapshotPaths.get(0);
this.storageGroupName = storageGroupName;
this.dataRegionId = dataRegionId;
}
@@ -98,6 +114,10 @@ public class SnapshotLoader {
* @return
*/
public DataRegion loadSnapshotForStateMachine() {
+ if (snapshotPaths.size() > 1) {
+ return loadSnapshotFromMultipleDirs();
+ }
+
LOGGER.info(
StorageEngineMessages.LOADING_SNAPSHOT_FOR, storageGroupName, dataRegionId, snapshotPath);
@@ -110,6 +130,33 @@ public class SnapshotLoader {
}
}
+ /**
+ * Load a snapshot whose fragments are spread across several dirs (the IoTConsensus receive
+ * folders). The snapshot log is not transferred during an IoTConsensus snapshot, so every
+ * received fragment dir takes the without-log path. Crucially, the data dirs are cleared exactly
+ * once before relinking from all dirs: clearing per-dir (as one load call per dir would) erases
+ * the fragments linked by the previous dirs and leaves only the last dir's data. Because each dir
+ * contributes a disjoint set of files, the relink order does not affect the result.
+ */
+ private DataRegion loadSnapshotFromMultipleDirs() {
+ LOGGER.info(
+ StorageEngineMessages.LOADING_SNAPSHOT_FOR, storageGroupName, dataRegionId, snapshotPaths);
+ try {
+ deleteAllFilesInDataDirs();
+ LOGGER.info(StorageEngineMessages.REMOVE_ALL_DATA_FILES_IN_ORIGINAL_DIR);
+ for (String path : snapshotPaths) {
+ File snapshotDir = new File(path);
+ createLinksFromSnapshotDirToDataDirWithoutLog(snapshotDir);
+ loadCompressionRatio(snapshotDir);
+ }
+ return loadSnapshot();
+ } catch (IOException | DiskSpaceInsufficientException e) {
+ LOGGER.error(
+ StorageEngineMessages.EXCEPTION_LOADING_SNAPSHOT_FOR, storageGroupName, dataRegionId, e);
+ return null;
+ }
+ }
+
private DataRegion loadSnapshotWithoutLog() {
try {
try {
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java
index 29fe4f13586..d042d2a2ae8 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java
@@ -245,6 +245,104 @@ public class IoTDBSnapshotTest {
}
}
+ /**
+ * Regression test for the multi-receive-folder snapshot load. IoTConsensus spreads a single
+ * snapshot's fragments across one receive folder per local data dir, so loading must clear the
+ * data dirs once and relink the fragments from every folder. Before the fix each folder was
+ * loaded with its own clear-then-relink, so every folder but the last had its just-linked
+ * fragments wiped by the next folder's clear, losing all but the last folder's data.
+ */
+ @Test
+ public void testLoadSnapshotSpreadAcrossReceiveFolders()
+ throws IOException, WriteProcessException {
+ loadSnapshotSpreadAcrossReceiveFolders(false);
+ }
+
+ /**
+ * The fragments of one snapshot are disjoint across the receive folders, so the order in which
+ * the folders are relinked must not change the loaded data. This loads the same spread snapshot
+ * with the receive folders presented in the opposite order and expects the identical result.
+ */
+ @Test
+ public void testLoadSnapshotFromReceiveFoldersIsOrderIndependent()
+ throws IOException, WriteProcessException {
+ loadSnapshotSpreadAcrossReceiveFolders(true);
+ }
+
+ private void loadSnapshotSpreadAcrossReceiveFolders(boolean reversedOrder)
+ throws IOException, WriteProcessException {
+ String[][] originDataDirs = IoTDBDescriptor.getInstance().getConfig().getTierDataDirs();
+ IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(testDataDirs);
+ TierManager.getInstance().resetFolders();
+ String recvBase0 = "target" + File.separator + "recv-snapshot-0";
+ String recvBase1 = "target" + File.separator + "recv-snapshot-1";
+ // Each receive folder holds the snapshot under a "snapshot" subdir, exactly as the IoTConsensus
+ // receiver materializes it, and carries no snapshot log (the log is never transferred).
+ File recvFolder0 = new File(recvBase0, SNAPSHOT_DIR_NAME);
+ File recvFolder1 = new File(recvBase1, SNAPSHOT_DIR_NAME);
+ try {
+ Assert.assertTrue(recvFolder0.mkdirs());
+ Assert.assertTrue(recvFolder1.mkdirs());
+
+ // Spread the fragments across the two receive folders: even-indexed files in the first,
+ // odd-indexed files in the second, so neither folder holds the whole snapshot.
+ int fileNum = 6;
+ for (int i = 0; i < fileNum; i++) {
+ writeSnapshotFragment((i % 2 == 0 ? recvFolder0 : recvFolder1).getAbsolutePath(), i);
+ }
+
+ List<String> snapshotDirs =
+ reversedOrder
+ ? Arrays.asList(recvFolder1.getAbsolutePath(), recvFolder0.getAbsolutePath())
+ : Arrays.asList(recvFolder0.getAbsolutePath(), recvFolder1.getAbsolutePath());
+
+ DataRegion dataRegion =
+ new SnapshotLoader(snapshotDirs, testSgName, "0").loadSnapshotForStateMachine();
+
+ Assert.assertNotNull(dataRegion);
+ // Every fragment from every receive folder must be present, regardless of relink order.
+ assertEquals(fileNum, dataRegion.getTsFileManager().getTsFileList(true).size());
+ } finally {
+ FileUtils.recursivelyDeleteFolder(recvBase0);
+ FileUtils.recursivelyDeleteFolder(recvBase1);
+ IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(originDataDirs);
+ TierManager.getInstance().resetFolders();
+ }
+ }
+
+ /**
+ * Materialize a single snapshot fragment (a TsFile and its resource) under {@code
+ * <recvSnapshotDir>/sequence/<sg>/0/0/}, i.e. the layout the without-log loader expects from a
+ * received snapshot.
+ */
+ private void writeSnapshotFragment(String recvSnapshotDir, int i)
+ throws IOException, WriteProcessException {
+ String filePath =
+ recvSnapshotDir
+ + File.separator
+ + "sequence"
+ + File.separator
+ + testSgName
+ + File.separator
+ + "0"
+ + File.separator
+ + "0"
+ + File.separator
+ + String.format("%d-%d-0-0.tsfile", i + 1, i + 1);
+ File newFile = new File(filePath);
+ Assert.assertTrue(newFile.getParentFile().exists() || newFile.getParentFile().mkdirs());
+ TsFileGeneratorUtils.generateMixTsFile(filePath, 5, 5, 10, i * 100, (i + 1) * 100, 10, 10);
+ TsFileResource resource = new TsFileResource(new File(filePath));
+ resource.updateStartTime(
+ IDeviceID.Factory.DEFAULT_FACTORY.create(testSgName + PATH_SEPARATOR + "d" + i), i * 100);
+ resource.updateEndTime(
+ IDeviceID.Factory.DEFAULT_FACTORY.create(testSgName + PATH_SEPARATOR + "d" + i),
+ (i + 1) * 100);
+ resource.updatePlanIndexes(i);
+ resource.setStatusForTest(TsFileResourceStatus.NORMAL);
+ resource.serialize();
+ }
+
@Ignore("Need manual execution to specify different disks")
@Test
public void testLoadSnapshotNoHardLink()
(iotdb) 05/06: Add envelope function to UDF registration script (#17971)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch rc/2.0.10
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f8d3fbb9e26daa5d83dbc2e772d8a029ee7633ff
Author: CritasWang <cr...@outlook.com>
AuthorDate: Wed Jun 17 11:28:07 2026 +0800
Add envelope function to UDF registration script (#17971)
---
library-udf/src/assembly/tools/register-UDF.bat | 1 +
library-udf/src/assembly/tools/register-UDF.sh | 1 +
2 files changed, 2 insertions(+)
diff --git a/library-udf/src/assembly/tools/register-UDF.bat b/library-udf/src/assembly/tools/register-UDF.bat
index f30db5d2a3a..08168b27527 100644
--- a/library-udf/src/assembly/tools/register-UDF.bat
+++ b/library-udf/src/assembly/tools/register-UDF.bat
@@ -88,6 +88,7 @@ call ../sbin/start-cli.bat -h %host% -p %rpcPort% -u %user% -pw %pass% -e "creat
call ../sbin/start-cli.bat -h %host% -p %rpcPort% -u %user% -pw %pass% -e "create function dwt as 'org.apache.iotdb.library.frequency.UDTFDWT'"
call ../sbin/start-cli.bat -h %host% -p %rpcPort% -u %user% -pw %pass% -e "create function idwt as 'org.apache.iotdb.library.frequency.UDTFIDWT'"
call ../sbin/start-cli.bat -h %host% -p %rpcPort% -u %user% -pw %pass% -e "create function ifft as 'org.apache.iotdb.library.frequency.UDTFIFFT'"
+call ../sbin/start-cli.bat -h %host% -p %rpcPort% -u %user% -pw %pass% -e "create function envelope as 'org.apache.iotdb.library.frequency.UDFEnvelopeAnalysis'"
@REM Series Discovery
diff --git a/library-udf/src/assembly/tools/register-UDF.sh b/library-udf/src/assembly/tools/register-UDF.sh
index faaa4df68b5..58aad2679b6 100755
--- a/library-udf/src/assembly/tools/register-UDF.sh
+++ b/library-udf/src/assembly/tools/register-UDF.sh
@@ -89,6 +89,7 @@ pass=root
../sbin/start-cli.sh -h $host -p $rpcPort -u $user -pw $pass -e "create function dwt as 'org.apache.iotdb.library.frequency.UDTFDWT'"
../sbin/start-cli.sh -h $host -p $rpcPort -u $user -pw $pass -e "create function idwt as 'org.apache.iotdb.library.frequency.UDTFIDWT'"
../sbin/start-cli.sh -h $host -p $rpcPort -u $user -pw $pass -e "create function ifft as 'org.apache.iotdb.library.frequency.UDTFIFFT'"
+../sbin/start-cli.sh -h $host -p $rpcPort -u $user -pw $pass -e "create function envelope as 'org.apache.iotdb.library.frequency.UDFEnvelopeAnalysis'"
# Series Discovery
../sbin/start-cli.sh -h $host -p $rpcPort -u $user -pw $pass -e "create function consecutivesequences as 'org.apache.iotdb.library.series.UDTFConsecutiveSequences'"