You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by cr...@apache.org on 2026/07/13 02:37:31 UTC
(iotdb-client-nodejs) branch fix/date-encoding-yyyymmdd created (now 40928b8)
This is an automated email from the ASF dual-hosted git repository.
CritasWang pushed a change to branch fix/date-encoding-yyyymmdd
in repository https://gitbox.apache.org/repos/asf/iotdb-client-nodejs.git
at 40928b8 fix: encode DATE as INT32 yyyyMMdd instead of days-since-epoch
This branch includes the following new commits:
new 40928b8 fix: encode DATE as INT32 yyyyMMdd instead of days-since-epoch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb-client-nodejs) 01/01: fix: encode DATE as INT32 yyyyMMdd instead of days-since-epoch
Posted by cr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
CritasWang pushed a commit to branch fix/date-encoding-yyyymmdd
in repository https://gitbox.apache.org/repos/asf/iotdb-client-nodejs.git
commit 40928b873f2a25824e825035473ef507fc740a2d
Author: CritasWang <cr...@outlook.com>
AuthorDate: Mon Jul 13 10:36:01 2026 +0800
fix: encode DATE as INT32 yyyyMMdd instead of days-since-epoch
IoTDB's DATE wire format is an INT32 encoded as year*10000 + month*100 + day
(e.g. 2026-07-13 -> 20260713), matching the Java client's
DateUtils.parseDateExpressionToInt and the C# client. The Node.js client was
writing and reading days-since-epoch, so DATE values written via tablets were
corrupted and unreadable by other clients (and vice versa).
Adjudicated against a live IoTDB 2.0.6 server: a row inserted via SQL literal
'2026-07-13' reads back as INT32 20260713 on the wire.
Changes:
- Add shared parseDateToInt / parseIntToDate utils in src/utils/DataTypes.ts
(UTC calendar components, matching Java LocalDate semantics) and export them
- Session.serializeColumn case 9: write yyyyMMdd (was days-since-epoch)
- FastSerializer.serializeDateColumn: same fix on the fast path
- Session.deserializeColumn case 9 (TSQueryDataSet): decode yyyyMMdd -> Date
- ColumnDecoder Int32ArrayColumnDecoder: decode DATE (type 9) columns to Date
- Session.parseTsBlock: DATE columns arrive typed as INT32 (1) in the TsBlock
header; convert them to Date using the metadata dataTypeList
- Unit tests: yyyyMMdd round-trip, exact wire bytes for 2026-07-13
(0x01352769), TsBlock decode; updated existing DATE serialization tests
Verified: tablet-written Date and SQL-literal date now read back identical
against a live IoTDB 2.0.6 (both 2026-07-13T00:00:00.000Z).
---
src/client/ColumnDecoder.ts | 13 ++++-
src/client/Session.ts | 40 ++++++++------
src/index.ts | 7 ++-
src/utils/DataTypes.ts | 49 ++++++++++++++++-
src/utils/FastSerializer.ts | 18 +++----
tests/unit/DataTypes.test.ts | 98 ++++++++++++++++++++++++++++++++++
tests/unit/FastSerializer.test.ts | 16 ++++--
tests/unit/TabletSerialization.test.ts | 18 +++++--
8 files changed, 218 insertions(+), 41 deletions(-)
diff --git a/src/client/ColumnDecoder.ts b/src/client/ColumnDecoder.ts
index 52701cc..25d5c13 100644
--- a/src/client/ColumnDecoder.ts
+++ b/src/client/ColumnDecoder.ts
@@ -18,6 +18,7 @@
*/
import { logger } from "../utils/Logger";
+import { parseIntToDate } from "../utils/DataTypes";
/**
* Column encoding types matching Apache IoTDB ColumnEncoding enum
@@ -152,7 +153,6 @@ class Int32ArrayColumnDecoder implements ColumnDecoder {
switch (dataType) {
case 1: // INT32
- case 9: // DATE
for (let i = 0; i < positionCount; i++) {
if (nullIndicators && nullIndicators[i]) {
values[i] = null;
@@ -163,6 +163,17 @@ class Int32ArrayColumnDecoder implements ColumnDecoder {
}
break;
+ case 9: // DATE (INT32 yyyyMMdd encoding -> Date object)
+ for (let i = 0; i < positionCount; i++) {
+ if (nullIndicators && nullIndicators[i]) {
+ values[i] = null;
+ continue;
+ }
+ values[i] = parseIntToDate(buffer.readInt32BE(currentOffset));
+ currentOffset += 4;
+ }
+ break;
+
case 3: // FLOAT
for (let i = 0; i < positionCount; i++) {
if (nullIndicators && nullIndicators[i]) {
diff --git a/src/client/Session.ts b/src/client/Session.ts
index 879f62e..3df82e4 100644
--- a/src/client/Session.ts
+++ b/src/client/Session.ts
@@ -35,6 +35,7 @@ import {
serializeTimestamps
} from "../utils/FastSerializer";
import { globalBufferPool } from "../utils/BufferPool";
+import { parseDateToInt, parseIntToDate } from "../utils/DataTypes";
const ttypes = require("../thrift/generated/client_types");
@@ -741,18 +742,11 @@ export class Session {
return buffer;
}
case 9: {
- // DATE (stored as INT32 - days since epoch) - Use big-endian
+ // DATE (stored as INT32 - yyyyMMdd, e.g. 20240101) - Use big-endian
const buffer = Buffer.alloc(values.length * 4);
values.forEach((v, i) => {
- let days = 0;
- if (v !== null && v !== undefined) {
- if (v instanceof Date) {
- days = Math.floor(v.getTime() / (24 * 60 * 60 * 1000));
- } else {
- days = v;
- }
- }
- buffer.writeInt32BE(days, i * 4);
+ const encoded = v === null || v === undefined ? 0 : parseDateToInt(v);
+ buffer.writeInt32BE(encoded, i * 4);
});
return buffer;
}
@@ -1021,6 +1015,23 @@ export class Session {
);
}
+ // DATE columns arrive in TsBlock with wire type INT32 (1); the real DATE
+ // type is only present in the query metadata. Convert those columns'
+ // yyyyMMdd integers (e.g. 20240101) to Date objects here.
+ for (let i = 0; i < valueColumns.length; i++) {
+ if (
+ valueColumnTypes[i] === 1 &&
+ this.getDataTypeCode(dataTypes[i]) === 9
+ ) {
+ const colValues = valueColumns[i].values;
+ for (let j = 0; j < colValues.length; j++) {
+ if (colValues[j] !== null) {
+ colValues[j] = parseIntToDate(colValues[j]);
+ }
+ }
+ }
+ }
+
// Build rows from columns
const rows: any[][] = [];
for (let rowIndex = 0; rowIndex < positionCount; rowIndex++) {
@@ -1313,15 +1324,14 @@ export class Session {
break;
}
case 9: {
- // DATE (stored as INT32 - days since epoch) - TSQueryDataSet uses BIG ENDIAN
+ // DATE (stored as INT32 - yyyyMMdd) - TSQueryDataSet uses BIG ENDIAN
for (let i = 0; i < rowCount; i++) {
if (this.isNull(bitmap, i)) {
values.push(null);
} else {
- // Convert days since epoch to Date object
- const days = buffer.readInt32BE(i * 4);
- const date = new Date(days * 24 * 60 * 60 * 1000);
- values.push(date);
+ // Convert yyyyMMdd integer to Date object
+ const encoded = buffer.readInt32BE(i * 4);
+ values.push(parseIntToDate(encoded));
}
}
break;
diff --git a/src/index.ts b/src/index.ts
index 54fdee4..76a2965 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -39,7 +39,12 @@ export {
parseNodeUrls,
} from "./utils/Config";
export { logger, LogLevel } from "./utils/Logger";
-export { TSDataType, getDataTypeName } from "./utils/DataTypes";
+export {
+ TSDataType,
+ getDataTypeName,
+ parseDateToInt,
+ parseIntToDate,
+} from "./utils/DataTypes";
export { RedirectException, TSStatusCode } from "./utils/Errors";
export { RedirectCache } from "./client/RedirectCache";
export { enableGlobalCleanup } from "./utils/ProcessCleanup";
diff --git a/src/utils/DataTypes.ts b/src/utils/DataTypes.ts
index a497e8b..166d192 100644
--- a/src/utils/DataTypes.ts
+++ b/src/utils/DataTypes.ts
@@ -77,8 +77,8 @@ export enum TSDataType {
/**
* Date with day precision (no time component)
- * JavaScript type: Date or number (days since epoch)
- * Storage size: 4 bytes (stored as INT32)
+ * JavaScript type: Date or number (yyyyMMdd integer, e.g. 20240101)
+ * Storage size: 4 bytes (stored as INT32, encoded as year*10000 + month*100 + day)
*/
DATE = 9,
@@ -99,6 +99,51 @@ export enum TSDataType {
// OBJECT = 12, // Reserved - not yet implemented
}
+/**
+ * Convert a JavaScript Date (or an already-encoded yyyyMMdd number) to the
+ * IoTDB DATE wire format: an INT32 encoded as year*10000 + month*100 + day
+ * (e.g. 2024-01-01 -> 20240101).
+ *
+ * This matches the Java client's DateUtils.parseDateExpressionToInt and the
+ * C# client. The calendar date is taken from the Date's UTC components,
+ * consistent with `new Date("2024-01-01")` which parses as UTC midnight.
+ *
+ * @param value - Date object or a yyyyMMdd integer (passed through unchanged)
+ * @returns The yyyyMMdd integer encoding
+ */
+export function parseDateToInt(value: Date | number): number {
+ if (value instanceof Date) {
+ return (
+ value.getUTCFullYear() * 10000 +
+ (value.getUTCMonth() + 1) * 100 +
+ value.getUTCDate()
+ );
+ }
+ return value;
+}
+
+/**
+ * Convert an IoTDB DATE wire value (INT32, year*10000 + month*100 + day)
+ * back to a JavaScript Date at UTC midnight of that calendar date.
+ *
+ * Inverse of {@link parseDateToInt}; matches the Java client's
+ * DateUtils.parseIntToDate.
+ *
+ * @param value - The yyyyMMdd integer (e.g. 20240101)
+ * @returns Date at UTC midnight of the encoded calendar date
+ */
+export function parseIntToDate(value: number): Date {
+ const year = Math.trunc(value / 10000);
+ const month = Math.trunc(value / 100) % 100;
+ const day = value % 100;
+ const date = new Date(Date.UTC(year, month - 1, day));
+ // Date.UTC maps years 0-99 to 1900-1999; correct that explicitly
+ if (year >= 0 && year < 100) {
+ date.setUTCFullYear(year);
+ }
+ return date;
+}
+
/**
* Get the name of a TSDataType from its numeric code
* @param typeCode - The numeric type code
diff --git a/src/utils/FastSerializer.ts b/src/utils/FastSerializer.ts
index a87f9b3..76ae9ed 100644
--- a/src/utils/FastSerializer.ts
+++ b/src/utils/FastSerializer.ts
@@ -18,6 +18,7 @@
*/
import { globalBufferPool } from "./BufferPool";
+import { parseDateToInt } from "./DataTypes";
/**
* Fast serialization utilities for IoTDB data types
@@ -167,26 +168,19 @@ export function serializeTimestampColumn(values: any[]): Buffer {
}
/**
- * Serialize DATE column (4 bytes per value, days since epoch)
+ * Serialize DATE column (4 bytes per value, INT32 yyyyMMdd encoding)
* Optimized: Single buffer allocation with direct writes
*/
export function serializeDateColumn(values: any[]): Buffer {
const size = values.length * 4;
const buffer = size >= 1024 ? globalBufferPool.acquire(size) : Buffer.allocUnsafe(size);
-
+
for (let i = 0; i < values.length; i++) {
const v = values[i];
- let days = 0;
- if (v !== null && v !== undefined) {
- if (v instanceof Date) {
- days = Math.floor(v.getTime() / (24 * 60 * 60 * 1000));
- } else {
- days = v;
- }
- }
- buffer.writeInt32BE(days, i * 4);
+ const encoded = v === null || v === undefined ? 0 : parseDateToInt(v);
+ buffer.writeInt32BE(encoded, i * 4);
}
-
+
return buffer.subarray(0, size);
}
diff --git a/tests/unit/DataTypes.test.ts b/tests/unit/DataTypes.test.ts
new file mode 100644
index 0000000..6ab66a9
--- /dev/null
+++ b/tests/unit/DataTypes.test.ts
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+ parseDateToInt,
+ parseIntToDate,
+} from "../../src/utils/DataTypes";
+import {
+ BaseColumnDecoder,
+ ColumnEncoding,
+} from "../../src/client/ColumnDecoder";
+
+describe("DATE yyyyMMdd conversion", () => {
+ describe("parseDateToInt", () => {
+ it("should encode a Date as INT32 yyyyMMdd (year*10000 + month*100 + day)", () => {
+ expect(parseDateToInt(new Date("2026-07-13"))).toBe(20260713);
+ expect(parseDateToInt(new Date("2024-01-01"))).toBe(20240101);
+ expect(parseDateToInt(new Date("2024-12-31"))).toBe(20241231);
+ // Leap day
+ expect(parseDateToInt(new Date("2024-02-29"))).toBe(20240229);
+ });
+
+ it("should pass through already-encoded numbers unchanged", () => {
+ expect(parseDateToInt(20260713)).toBe(20260713);
+ expect(parseDateToInt(0)).toBe(0);
+ });
+ });
+
+ describe("parseIntToDate", () => {
+ it("should decode INT32 yyyyMMdd to a Date at UTC midnight", () => {
+ const date = parseIntToDate(20260713);
+ expect(date.getUTCFullYear()).toBe(2026);
+ expect(date.getUTCMonth()).toBe(6); // July (0-based)
+ expect(date.getUTCDate()).toBe(13);
+ expect(date.getUTCHours()).toBe(0);
+ expect(date.getUTCMinutes()).toBe(0);
+ });
+ });
+
+ describe("round-trip", () => {
+ it("should round-trip Date -> yyyyMMdd -> Date", () => {
+ const dates = [
+ new Date("1970-01-01"),
+ new Date("2000-02-29"),
+ new Date("2024-01-01"),
+ new Date("2026-07-13"),
+ new Date("9999-12-31"),
+ ];
+ for (const original of dates) {
+ const encoded = parseDateToInt(original);
+ const decoded = parseIntToDate(encoded);
+ expect(decoded.getTime()).toBe(original.getTime());
+ // And the integer round-trips too
+ expect(parseDateToInt(decoded)).toBe(encoded);
+ }
+ });
+ });
+
+ describe("TsBlock column decode (Int32ArrayColumnDecoder)", () => {
+ it("should decode a DATE column value as a Date from yyyyMMdd wire bytes", () => {
+ // Column layout: 1 byte null flag (0 = no nulls) + INT32 BE values
+ // 20260713 = 0x01352769
+ const buffer = Buffer.from([0x00, 0x01, 0x35, 0x27, 0x69]);
+ const decoder = BaseColumnDecoder.getDecoder(ColumnEncoding.Int32Array);
+ const { column, bytesRead } = decoder.readColumn(buffer, 0, 9, 1);
+
+ expect(bytesRead).toBe(5);
+ const value = column.values[0];
+ expect(value).toBeInstanceOf(Date);
+ expect(value.getUTCFullYear()).toBe(2026);
+ expect(value.getUTCMonth()).toBe(6);
+ expect(value.getUTCDate()).toBe(13);
+ });
+
+ it("should still decode plain INT32 columns as numbers", () => {
+ const buffer = Buffer.from([0x00, 0x01, 0x35, 0x27, 0x69]);
+ const decoder = BaseColumnDecoder.getDecoder(ColumnEncoding.Int32Array);
+ const { column } = decoder.readColumn(buffer, 0, 1, 1);
+ expect(column.values[0]).toBe(20260713);
+ });
+ });
+});
diff --git a/tests/unit/FastSerializer.test.ts b/tests/unit/FastSerializer.test.ts
index 4f2b7ba..eb6d686 100644
--- a/tests/unit/FastSerializer.test.ts
+++ b/tests/unit/FastSerializer.test.ts
@@ -155,18 +155,24 @@ describe("FastSerializer", () => {
});
describe("DATE Serialization", () => {
- it("should serialize DATE values correctly", () => {
+ it("should serialize DATE values as INT32 yyyyMMdd", () => {
const date1 = new Date("2024-01-01");
- const days1 = Math.floor(date1.getTime() / (24 * 60 * 60 * 1000));
- const values = [date1, 100, null, undefined];
+ const values = [date1, 20241231, null, undefined];
const buffer = serializeDateColumn(values);
expect(buffer.length).toBe(16); // 4 * 4 bytes
- expect(buffer.readInt32BE(0)).toBe(days1);
- expect(buffer.readInt32BE(4)).toBe(100);
+ expect(buffer.readInt32BE(0)).toBe(20240101); // yyyyMMdd, NOT days since epoch
+ expect(buffer.readInt32BE(4)).toBe(20241231); // numbers pass through unchanged
expect(buffer.readInt32BE(8)).toBe(0); // null -> 0
expect(buffer.readInt32BE(12)).toBe(0); // undefined -> 0
});
+
+ it("should produce exact wire bytes for 2026-07-13", () => {
+ // IoTDB DATE wire format: INT32 yyyyMMdd, big-endian
+ // 20260713 = 0x01352769
+ const buffer = serializeDateColumn([new Date("2026-07-13")]);
+ expect(Array.from(buffer)).toEqual([0x01, 0x35, 0x27, 0x69]);
+ });
});
describe("BLOB Serialization", () => {
diff --git a/tests/unit/TabletSerialization.test.ts b/tests/unit/TabletSerialization.test.ts
index fb86706..65278cd 100644
--- a/tests/unit/TabletSerialization.test.ts
+++ b/tests/unit/TabletSerialization.test.ts
@@ -342,20 +342,28 @@ describe('Tablet Serialization', () => {
expect(buffer.readDoubleBE(16)).toBe(0.0);
});
- test('should serialize DATE column', () => {
+ test('should serialize DATE column as INT32 yyyyMMdd', () => {
const date1 = new Date('2024-01-01');
const date2 = new Date('2024-12-31');
const values = [date1, date2, 0];
const buffer = (session as any).serializeColumn(values, TSDataType.DATE);
expect(buffer.length).toBe(12); // 3 values * 4 bytes
- const days1 = Math.floor(date1.getTime() / (24 * 60 * 60 * 1000));
- const days2 = Math.floor(date2.getTime() / (24 * 60 * 60 * 1000));
- expect(buffer.readInt32BE(0)).toBe(days1);
- expect(buffer.readInt32BE(4)).toBe(days2);
+ expect(buffer.readInt32BE(0)).toBe(20240101); // yyyyMMdd, NOT days since epoch
+ expect(buffer.readInt32BE(4)).toBe(20241231);
expect(buffer.readInt32BE(8)).toBe(0);
});
+ test('should serialize DATE 2026-07-13 with exact wire bytes', () => {
+ // IoTDB DATE wire format: INT32 yyyyMMdd, big-endian
+ // 20260713 = 0x01352769
+ const buffer = (session as any).serializeColumn(
+ [new Date('2026-07-13')],
+ TSDataType.DATE,
+ );
+ expect(Array.from(buffer)).toEqual([0x01, 0x35, 0x27, 0x69]);
+ });
+
test('should serialize TIMESTAMP column', () => {
const ts1 = Date.now();
const ts2 = ts1 + 1000;