You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ra...@apache.org on 2024/04/29 15:50:12 UTC
(arrow) 04/24: GH-40959: [JS] Store Timestamps in 64 bits (#40960)
This is an automated email from the ASF dual-hosted git repository.
raulcd pushed a commit to branch maint-16.x.x
in repository https://gitbox.apache.org/repos/asf/arrow.git
commit 1c430be8b0cdf9621766d73610b90a6e908e2626
Author: Dominik Moritz <do...@apache.org>
AuthorDate: Tue Apr 16 17:01:13 2024 -0400
GH-40959: [JS] Store Timestamps in 64 bits (#40960)
Merge after #40892.
This pull request also changes Dates to return timestamps instead of
Date instances (similar to Timestamps and for the same reason.
* GitHub Issue: #40959
---
js/src/type.ts | 26 +++++++++++-----
js/src/util/bigint.ts | 13 ++++++++
js/src/visitor/get.ts | 21 +++++--------
js/src/visitor/iterator.ts | 9 +++---
js/src/visitor/set.ts | 25 +++------------
js/test/generate-test-data.ts | 53 +++++++++++++++-----------------
js/test/unit/builders/date-tests.ts | 20 ++++++------
js/test/unit/builders/utils.ts | 20 +++---------
js/test/unit/vector/date-vector-tests.ts | 36 +++++++++++++++++++---
js/test/unit/vector/vector-tests.ts | 17 ++--------
10 files changed, 119 insertions(+), 121 deletions(-)
diff --git a/js/src/type.ts b/js/src/type.ts
index a42552d65a..88aadb864e 100644
--- a/js/src/type.ts
+++ b/js/src/type.ts
@@ -333,16 +333,28 @@ export class Decimal extends DataType<Type.Decimal> {
/** @ignore */
export type Dates = Type.Date | Type.DateDay | Type.DateMillisecond;
/** @ignore */
-export interface Date_<T extends Dates = Dates> extends DataType<T> { TArray: Int32Array; TValue: Date; ArrayType: TypedArrayConstructor<Int32Array> }
+type DateType = {
+ [Type.Date]: { TArray: Int32Array | BigInt64Array };
+ [Type.DateDay]: { TArray: Int32Array };
+ [Type.DateMillisecond]: { TArray: BigInt64Array };
+};
+/** @ignore */
+export interface Date_<T extends Dates = Dates> extends DataType<T> {
+ TArray: DateType[T]['TArray'];
+ TValue: number;
+}
/** @ignore */
export class Date_<T extends Dates = Dates> extends DataType<T> {
constructor(public readonly unit: DateUnit) {
super(Type.Date as T);
}
public toString() { return `Date${(this.unit + 1) * 32}<${DateUnit[this.unit]}>`; }
+
+ public get ArrayType() {
+ return this.unit === DateUnit.DAY ? Int32Array : BigInt64Array;
+ }
protected static [Symbol.toStringTag] = ((proto: Date_) => {
(<any>proto).unit = null;
- (<any>proto).ArrayType = Int32Array;
return proto[Symbol.toStringTag] = 'Date';
})(Date_.prototype);
}
@@ -417,9 +429,9 @@ export class TimeNanosecond extends Time_<Type.TimeNanosecond> { constructor() {
type Timestamps = Type.Timestamp | Type.TimestampSecond | Type.TimestampMillisecond | Type.TimestampMicrosecond | Type.TimestampNanosecond;
/** @ignore */
interface Timestamp_<T extends Timestamps = Timestamps> extends DataType<T> {
- TArray: Int32Array;
+ TArray: BigInt64Array;
TValue: number;
- ArrayType: TypedArrayConstructor<Int32Array>;
+ ArrayType: BigIntArrayConstructor<BigInt64Array>;
}
/** @ignore */
@@ -432,7 +444,7 @@ class Timestamp_<T extends Timestamps = Timestamps> extends DataType<T> {
protected static [Symbol.toStringTag] = ((proto: Timestamp_) => {
(<any>proto).unit = null;
(<any>proto).timezone = null;
- (<any>proto).ArrayType = Int32Array;
+ (<any>proto).ArrayType = BigInt64Array;
return proto[Symbol.toStringTag] = 'Timestamp';
})(Timestamp_.prototype);
}
@@ -483,7 +495,7 @@ type Durations = Type.Duration | Type.DurationSecond | Type.DurationMillisecond
export interface Duration<T extends Durations = Durations> extends DataType<T> {
TArray: BigInt64Array;
TValue: bigint;
- ArrayType: BigInt64Array;
+ ArrayType: BigIntArrayConstructor<BigInt64Array>;
}
/** @ignore */
@@ -737,8 +749,6 @@ export function strideForType(type: DataType) {
const t: any = type;
switch (type.typeId) {
case Type.Decimal: return (type as Decimal).bitWidth / 32;
- case Type.Timestamp: return 2;
- case Type.Date: return 1 + (t as Date_).unit;
case Type.Interval: return 1 + (t as Interval_).unit;
// case Type.Int: return 1 + +((t as Int_).bitWidth > 32);
// case Type.Time: return 1 + +((t as Time_).bitWidth > 32);
diff --git a/js/src/util/bigint.ts b/js/src/util/bigint.ts
index 5af2f7f052..470b83f5fb 100644
--- a/js/src/util/bigint.ts
+++ b/js/src/util/bigint.ts
@@ -24,3 +24,16 @@ export function bigIntToNumber(number: bigint | number): number {
}
return Number(number);
}
+
+/**
+ * Duivides the bigint number by the divisor and returns the result as a number.
+ * Dividing bigints always results in bigints so we don't get the remainder.
+ * This function gives us the remainder but assumes that the result fits into a number.
+ *
+ * @param number The number to divide.
+ * @param divisor The divisor.
+ * @returns The result of the division as a number.
+ */
+export function divideBigInts(number: bigint, divisor: bigint): number {
+ return bigIntToNumber(number / divisor) + bigIntToNumber(number % divisor) / bigIntToNumber(divisor);
+}
diff --git a/js/src/visitor/get.ts b/js/src/visitor/get.ts
index 3ab3bcb68c..ddfc04884f 100644
--- a/js/src/visitor/get.ts
+++ b/js/src/visitor/get.ts
@@ -21,7 +21,7 @@ import { Vector } from '../vector.js';
import { Visitor } from '../visitor.js';
import { MapRow } from '../row/map.js';
import { StructRow, StructRowProxy } from '../row/struct.js';
-import { bigIntToNumber } from '../util/bigint.js';
+import { bigIntToNumber, divideBigInts } from '../util/bigint.js';
import { decodeUtf8 } from '../util/utf8.js';
import { TypeToDataType } from '../interfaces.js';
import { uint16ToFloat64 } from '../util/math.js';
@@ -106,13 +106,6 @@ function wrapGet<T extends DataType>(fn: (data: Data<T>, _1: any) => any) {
}
/** @ignore */const epochDaysToMs = (data: Int32Array, index: number) => 86400000 * data[index];
-/** @ignore */const epochMillisecondsLongToMs = (data: Int32Array, index: number) => 4294967296 * (data[index + 1]) + (data[index] >>> 0);
-/** @ignore */const epochMicrosecondsLongToMs = (data: Int32Array, index: number) => 4294967296 * (data[index + 1] / 1000) + ((data[index] >>> 0) / 1000);
-/** @ignore */const epochNanosecondsLongToMs = (data: Int32Array, index: number) => 4294967296 * (data[index + 1] / 1000000) + ((data[index] >>> 0) / 1000000);
-
-/** @ignore */const epochMillisecondsToDate = (epochMs: number) => new Date(epochMs);
-/** @ignore */const epochDaysToDate = (data: Int32Array, index: number) => epochMillisecondsToDate(epochDaysToMs(data, index));
-/** @ignore */const epochMillisecondsLongToDate = (data: Int32Array, index: number) => epochMillisecondsToDate(epochMillisecondsLongToMs(data, index));
/** @ignore */
const getNull = <T extends Null>(_data: Data<T>, _index: number): T['TValue'] => null;
@@ -139,9 +132,9 @@ type Numeric1X = Int8 | Int16 | Int32 | Uint8 | Uint16 | Uint32 | Float32 | Floa
type Numeric2X = Int64 | Uint64;
/** @ignore */
-const getDateDay = <T extends DateDay>({ values }: Data<T>, index: number): T['TValue'] => epochDaysToDate(values, index);
+const getDateDay = <T extends DateDay>({ values }: Data<T>, index: number): T['TValue'] => epochDaysToMs(values, index);
/** @ignore */
-const getDateMillisecond = <T extends DateMillisecond>({ values }: Data<T>, index: number): T['TValue'] => epochMillisecondsLongToDate(values, index * 2);
+const getDateMillisecond = <T extends DateMillisecond>({ values }: Data<T>, index: number): T['TValue'] => bigIntToNumber(values[index]);
/** @ignore */
const getNumeric = <T extends Numeric1X>({ stride, values }: Data<T>, index: number): T['TValue'] => values[stride * index];
/** @ignore */
@@ -178,13 +171,13 @@ const getDate = <T extends Date_>(data: Data<T>, index: number): T['TValue'] =>
);
/** @ignore */
-const getTimestampSecond = <T extends TimestampSecond>({ values }: Data<T>, index: number): T['TValue'] => 1000 * epochMillisecondsLongToMs(values, index * 2);
+const getTimestampSecond = <T extends TimestampSecond>({ values }: Data<T>, index: number): T['TValue'] => 1000 * bigIntToNumber(values[index]);
/** @ignore */
-const getTimestampMillisecond = <T extends TimestampMillisecond>({ values }: Data<T>, index: number): T['TValue'] => epochMillisecondsLongToMs(values, index * 2);
+const getTimestampMillisecond = <T extends TimestampMillisecond>({ values }: Data<T>, index: number): T['TValue'] => bigIntToNumber(values[index]);
/** @ignore */
-const getTimestampMicrosecond = <T extends TimestampMicrosecond>({ values }: Data<T>, index: number): T['TValue'] => epochMicrosecondsLongToMs(values, index * 2);
+const getTimestampMicrosecond = <T extends TimestampMicrosecond>({ values }: Data<T>, index: number): T['TValue'] => divideBigInts(values[index], BigInt(1000));
/** @ignore */
-const getTimestampNanosecond = <T extends TimestampNanosecond>({ values }: Data<T>, index: number): T['TValue'] => epochNanosecondsLongToMs(values, index * 2);
+const getTimestampNanosecond = <T extends TimestampNanosecond>({ values }: Data<T>, index: number): T['TValue'] => divideBigInts(values[index], BigInt(1000000));
/* istanbul ignore next */
/** @ignore */
const getTimestamp = <T extends Timestamp>(data: Data<T>, index: number): T['TValue'] => {
diff --git a/js/src/visitor/iterator.ts b/js/src/visitor/iterator.ts
index bf7e9d1591..2a55f74367 100644
--- a/js/src/visitor/iterator.ts
+++ b/js/src/visitor/iterator.ts
@@ -101,10 +101,11 @@ function vectorIterator<T extends DataType>(vector: Vector<T>): IterableIterator
// Fast case, defer to native iterators if possible
if (vector.nullCount === 0 && vector.stride === 1 && (
- (type.typeId === Type.Timestamp) ||
- (type instanceof Int && (type as Int).bitWidth !== 64) ||
- (type instanceof Time && (type as Time).bitWidth !== 64) ||
- (type instanceof Float && (type as Float).precision !== Precision.HALF)
+ // Don't defer to native iterator for timestamps since Numbers are expected
+ // (DataType.isTimestamp(type)) && type.unit === TimeUnit.MILLISECOND ||
+ (DataType.isInt(type) && type.bitWidth !== 64) ||
+ (DataType.isTime(type) && type.bitWidth !== 64) ||
+ (DataType.isFloat(type) && type.precision !== Precision.HALF)
)) {
return new ChunkedIterator(vector.data.length, (chunkIndex) => {
const data = vector.data[chunkIndex];
diff --git a/js/src/visitor/set.ts b/js/src/visitor/set.ts
index 5dc42283c3..5f2a3af4df 100644
--- a/js/src/visitor/set.ts
+++ b/js/src/visitor/set.ts
@@ -109,21 +109,6 @@ function wrapSet<T extends DataType>(fn: (data: Data<T>, _1: any, _2: any) => vo
/** @ignore */
export const setEpochMsToDays = (data: Int32Array, index: number, epochMs: number) => { data[index] = Math.floor(epochMs / 86400000); };
-/** @ignore */
-export const setEpochMsToMillisecondsLong = (data: Int32Array, index: number, epochMs: number) => {
- data[index] = Math.floor(epochMs % 4294967296);
- data[index + 1] = Math.floor(epochMs / 4294967296);
-};
-/** @ignore */
-export const setEpochMsToMicrosecondsLong = (data: Int32Array, index: number, epochMs: number) => {
- data[index] = Math.floor((epochMs * 1000) % 4294967296);
- data[index + 1] = Math.floor((epochMs * 1000) / 4294967296);
-};
-/** @ignore */
-export const setEpochMsToNanosecondsLong = (data: Int32Array, index: number, epochMs: number) => {
- data[index] = Math.floor((epochMs * 1000000) % 4294967296);
- data[index + 1] = Math.floor((epochMs * 1000000) / 4294967296);
-};
/** @ignore */
export const setVariableWidthBytes = <T extends Int32Array | BigInt64Array>(values: Uint8Array, valueOffsets: T, index: number, value: Uint8Array) => {
@@ -161,7 +146,7 @@ export const setAnyFloat = <T extends Float>(data: Data<T>, index: number, value
/** @ignore */
export const setDateDay = <T extends DateDay>({ values }: Data<T>, index: number, value: T['TValue']): void => { setEpochMsToDays(values, index, value.valueOf()); };
/** @ignore */
-export const setDateMillisecond = <T extends DateMillisecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { setEpochMsToMillisecondsLong(values, index * 2, value.valueOf()); };
+export const setDateMillisecond = <T extends DateMillisecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { values[index] = BigInt(value); };
/** @ignore */
export const setFixedSizeBinary = <T extends FixedSizeBinary>({ stride, values }: Data<T>, index: number, value: T['TValue']): void => { values.set(value.subarray(0, stride), stride * index); };
@@ -178,13 +163,13 @@ export const setDate = <T extends Date_>(data: Data<T>, index: number, value: T[
};
/** @ignore */
-export const setTimestampSecond = <T extends TimestampSecond>({ values }: Data<T>, index: number, value: T['TValue']): void => setEpochMsToMillisecondsLong(values, index * 2, value / 1000);
+export const setTimestampSecond = <T extends TimestampSecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { values[index] = BigInt(value / 1000); };
/** @ignore */
-export const setTimestampMillisecond = <T extends TimestampMillisecond>({ values }: Data<T>, index: number, value: T['TValue']): void => setEpochMsToMillisecondsLong(values, index * 2, value);
+export const setTimestampMillisecond = <T extends TimestampMillisecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { values[index] = BigInt(value); };
/** @ignore */
-export const setTimestampMicrosecond = <T extends TimestampMicrosecond>({ values }: Data<T>, index: number, value: T['TValue']): void => setEpochMsToMicrosecondsLong(values, index * 2, value);
+export const setTimestampMicrosecond = <T extends TimestampMicrosecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { values[index] = BigInt(value * 1000); };
/** @ignore */
-export const setTimestampNanosecond = <T extends TimestampNanosecond>({ values }: Data<T>, index: number, value: T['TValue']): void => setEpochMsToNanosecondsLong(values, index * 2, value);
+export const setTimestampNanosecond = <T extends TimestampNanosecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { values[index] = BigInt(value * 1000000); };
/* istanbul ignore next */
/** @ignore */
export const setTimestamp = <T extends Timestamp>(data: Data<T>, index: number, value: T['TValue']): void => {
diff --git a/js/test/generate-test-data.ts b/js/test/generate-test-data.ts
index 8e6e47de83..65719f875c 100644
--- a/js/test/generate-test-data.ts
+++ b/js/test/generate-test-data.ts
@@ -402,10 +402,7 @@ function generateDate<T extends Date_>(this: TestDataVectorGenerator, type: T, l
const data = type.unit === DateUnit.DAY
? createDate32(length, nullBitmap, values)
: createDate64(length, nullBitmap, values);
- return {
- values: () => values.map((x) => x == null ? null : new Date(x)),
- vector: new Vector([makeData({ type, length, nullCount, nullBitmap, data })])
- };
+ return { values: () => values, vector: new Vector([makeData({ type, length, nullCount, nullBitmap, data })]) };
}
function generateTimestamp<T extends Timestamp>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = Math.trunc(length * 0.2)): GeneratedVector<T> {
@@ -649,6 +646,7 @@ type TypedArrayConstructor =
const rand = Math.random.bind(Math);
+const randSign = () => rand() > 0.5 ? -1 : 1;
const randomBytes = (length: number) => fillRandom(Uint8Array, length);
const memoize = (fn: () => any) => ((x?: any) => () => x || (x = fn()))();
@@ -661,7 +659,7 @@ function fillRandom<T extends TypedArrayConstructor>(ArrayType: T, length: numbe
const BPE = ArrayType.BYTES_PER_ELEMENT;
const array = new ArrayType(length);
const max = (2 ** (8 * BPE)) - 1;
- for (let i = -1; ++i < length; array[i] = rand() * max * (rand() > 0.5 ? -1 : 1));
+ for (let i = -1; ++i < length; array[i] = rand() * max * randSign());
return array as InstanceType<T>;
}
@@ -669,7 +667,7 @@ function fillRandomBigInt<T extends (typeof BigInt64Array) | (typeof BigUint64Ar
const BPE = ArrayType.BYTES_PER_ELEMENT;
const array = new ArrayType(length);
const max = (2 ** (8 * BPE)) - 1;
- for (let i = -1; ++i < length; array[i] = BigInt(rand() * max * (rand() > 0.5 ? -1 : 1)));
+ for (let i = -1; ++i < length; array[i] = BigInt(rand() * max * randSign()));
return array as InstanceType<T>;
}
@@ -735,6 +733,9 @@ function createVariableWidthBytes(length: number, nullBitmap: Uint8Array, offset
return bytes;
}
+/**
+ * Creates timestamps with the accuracy of days (86400000 millisecond).
+ */
function createDate32(length: number, nullBitmap: Uint8Array, values: (number | null)[] = []) {
const data = new Int32Array(length).fill(Math.trunc(Date.now() / 86400000));
iterateBitmap(length, nullBitmap, (i, valid) => {
@@ -742,7 +743,7 @@ function createDate32(length: number, nullBitmap: Uint8Array, values: (number |
data[i] = 0;
values[i] = null;
} else {
- data[i] = Math.trunc(data[i] + (rand() * 10000 * (rand() > 0.5 ? -1 : 1)));
+ data[i] = Math.trunc(data[i] + (rand() * 10000 * randSign()));
values[i] = data[i] * 86400000;
}
});
@@ -750,32 +751,26 @@ function createDate32(length: number, nullBitmap: Uint8Array, values: (number |
}
function createDate64(length: number, nullBitmap: Uint8Array, values: (number | null)[] = []) {
- const data = new Int32Array(length * 2).fill(0);
const data32 = createDate32(length, nullBitmap, values);
- iterateBitmap(length, nullBitmap, (i, valid) => {
- if (valid) {
- const value = data32[i] * 86400000;
- const hi = Math.trunc(value / 4294967296);
- const lo = Math.trunc(value - 4294967296 * hi);
- values[i] = value;
- data[i * 2 + 0] = lo;
- data[i * 2 + 1] = hi;
- }
- });
- return data;
+ return BigInt64Array.from(data32, x => BigInt(x * 86400000));
+}
+
+function divideBigInts(number: bigint, divisor: bigint): number {
+ return Number(number / divisor) + Number(number % divisor) / Number(divisor);
}
function createTimestamp(length: number, nullBitmap: Uint8Array, multiple: number, values: (number | null)[] = []) {
- const mult = 86400 * multiple;
- const data = new Int32Array(length * 2).fill(0);
- const data32 = createDate32(length, nullBitmap, values);
+ const data = new BigInt64Array(length).fill(0n);
+ const tenYears = 10 * 365 * 24 * 60 * 60 * multiple;
+ const now = Math.trunc(Date.now() / 1000 * multiple);
iterateBitmap(length, nullBitmap, (i, valid) => {
- if (valid) {
- const value = data32[i] * mult;
- const hi = Math.trunc(value / 4294967296);
- const lo = Math.trunc(value - 4294967296 * hi);
- data[i * 2 + 0] = lo;
- data[i * 2 + 1] = hi;
+ if (!valid) {
+ data[i] = 0n;
+ values[i] = null;
+ } else {
+ const value = BigInt(now + Math.trunc(rand() * randSign() * tenYears));
+ data[i] = value;
+ values[i] = divideBigInts(value * 1000n, BigInt(multiple));
}
});
return data;
@@ -788,7 +783,7 @@ function createTime32(length: number, nullBitmap: Uint8Array, multiple: number,
data[i] = 0;
values[i] = null;
} else {
- values[i] = data[i] = ((1000 * rand()) | 0 * multiple) * (rand() > 0.5 ? -1 : 1);
+ values[i] = data[i] = ((1000 * rand()) | 0 * multiple) * randSign();
}
});
return data;
diff --git a/js/test/unit/builders/date-tests.ts b/js/test/unit/builders/date-tests.ts
index 318bb7dfc4..3fd3f7832f 100644
--- a/js/test/unit/builders/date-tests.ts
+++ b/js/test/unit/builders/date-tests.ts
@@ -17,10 +17,8 @@
import 'web-streams-polyfill';
import {
- date32sNoNulls,
- date32sWithNulls,
- date64sNoNulls,
- date64sWithNulls,
+ dateNoNulls,
+ dateWithNulls,
encodeAll,
encodeEach,
encodeEachDOM,
@@ -41,14 +39,14 @@ describe('DateDayBuilder', () => {
testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(() => new DateDay(), 25));
testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(() => new DateDay(), 25));
- function runTestsWithEncoder(name: string, encode: (vals: (Date | null)[], nullVals?: any[]) => Promise<Vector<DateDay>>) {
+ function runTestsWithEncoder(name: string, encode: (vals: (number | null)[], nullVals?: any[]) => Promise<Vector<DateDay>>) {
describe(`${encode.name} ${name}`, () => {
it(`encodes dates no nulls`, async () => {
- const vals = date32sNoNulls(20);
+ const vals = dateNoNulls(20);
validateVector(vals, await encode(vals, []), []);
});
it(`encodes dates with nulls`, async () => {
- const vals = date32sWithNulls(20);
+ const vals = dateWithNulls(20);
validateVector(vals, await encode(vals, [null]), [null]);
});
});
@@ -63,14 +61,14 @@ describe('DateMillisecondBuilder', () => {
testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(() => new DateMillisecond(), 25));
testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(() => new DateMillisecond(), 25));
- function runTestsWithEncoder(name: string, encode: (vals: (Date | null)[], nullVals?: any[]) => Promise<Vector<DateMillisecond>>) {
+ function runTestsWithEncoder(name: string, encode: (vals: (number | null)[], nullVals?: any[]) => Promise<Vector<DateMillisecond>>) {
describe(`${encode.name} ${name}`, () => {
it(`encodes dates no nulls`, async () => {
- const vals = date64sNoNulls(20);
+ const vals = dateNoNulls(20);
validateVector(vals, await encode(vals, []), []);
});
it(`encodes dates with nulls`, async () => {
- const vals = date64sWithNulls(20);
+ const vals = dateWithNulls(20);
validateVector(vals, await encode(vals, [null]), [null]);
});
});
@@ -100,7 +98,7 @@ describe('DateMillisecondBuilder with nulls', () => {
'2019-03-10T21:15:32.237Z',
'2019-03-21T07:25:34.864Z',
null
- ].map((x) => x === null ? x : new Date(x));
+ ].map((x) => x === null ? x : new Date(x).getTime());
it(`encodes dates with nulls`, async () => {
const vals = dates.slice();
validateVector(vals, await encode(vals, [null]), [null]);
diff --git a/js/test/unit/builders/utils.ts b/js/test/unit/builders/utils.ts
index db4e80d002..1d0707a6ca 100644
--- a/js/test/unit/builders/utils.ts
+++ b/js/test/unit/builders/utils.ts
@@ -32,22 +32,11 @@ const randnulls = <T, TNull = null>(values: T[], n: TNull = <any>null) => values
export const randomBytes = (length: number) => fillRandom(Uint8Array, length);
export const stringsNoNulls = (length = 20) => Array.from({ length }, (_) => randomString(1 + (Math.trunc(Math.random() * 19))));
-export const timestamp32sNoNulls = (length = 20, now = Math.trunc(Date.now() / 86400000)) =>
- Array.from({ length }, (_) => (Math.trunc(now + (rand() * 10000 * (rand() > 0.5 ? -1 : 1)))) * 86400000);
-
-export const timestamp64sNoNulls = (length = 20, now = Date.now()) => Array.from({ length }, (_) => {
- const ms = now + (Math.trunc(rand() * 31557600000 * (rand() > 0.5 ? -1 : 1)));
- return new Int32Array([Math.trunc(ms % 4294967296), Math.trunc(ms / 4294967296)]);
-});
-
-export const timestamp32sWithNulls = (length = 20) => randnulls(timestamp32sNoNulls(length), null);
-export const timestamp64sWithNulls = (length = 20) => randnulls(timestamp64sNoNulls(length), null);
-export const timestamp32sWithMaxInts = (length = 20) => randnulls(timestamp32sNoNulls(length), 0x7FFFFFFF);
-export const timestamp64sWithMaxInts = (length = 20) => randnulls(timestamp64sNoNulls(length), 9223372034707292159n);
export const boolsNoNulls = (length = 20) => Array.from({ length }, () => rand() > 0.5);
-export const date32sNoNulls = (length = 20) => timestamp32sNoNulls(length).map((x) => new Date(x));
-export const date64sNoNulls = (length = 20) => timestamp64sNoNulls(length).map((x) => new Date(4294967296 * x[1] + (x[0] >>> 0)));
+
+export const dateNoNulls = (length = 20, now = Math.trunc(Date.now() / 86400000)) =>
+ Array.from({ length }, (_) => (Math.trunc(now + (rand() * 100000 * (rand() > 0.5 ? -1 : 1)))) * 86400000);
export const int8sNoNulls = (length = 20) => Array.from(new Int8Array(randomBytes(length * Int8Array.BYTES_PER_ELEMENT).buffer));
export const int16sNoNulls = (length = 20) => Array.from(new Int16Array(randomBytes(length * Int16Array.BYTES_PER_ELEMENT).buffer));
export const int32sNoNulls = (length = 20) => Array.from(new Int32Array(randomBytes(length * Int32Array.BYTES_PER_ELEMENT).buffer));
@@ -66,8 +55,7 @@ export const stringsWithNulls = (length = 20) => randnulls(stringsNoNulls(length
export const stringsWithEmpties = (length = 20) => randnulls(stringsNoNulls(length), '\0');
export const boolsWithNulls = (length = 20) => randnulls(boolsNoNulls(length), null);
-export const date32sWithNulls = (length = 20) => randnulls(date32sNoNulls(length), null);
-export const date64sWithNulls = (length = 20) => randnulls(date64sNoNulls(length), null);
+export const dateWithNulls = (length = 20) => randnulls(dateNoNulls(length), null);
export const int8sWithNulls = (length = 20) => randnulls(int8sNoNulls(length), null);
export const int16sWithNulls = (length = 20) => randnulls(int16sNoNulls(length), null);
export const int32sWithNulls = (length = 20) => randnulls(int32sNoNulls(length), null);
diff --git a/js/test/unit/vector/date-vector-tests.ts b/js/test/unit/vector/date-vector-tests.ts
index e5cd49933e..4a6ffc1a82 100644
--- a/js/test/unit/vector/date-vector-tests.ts
+++ b/js/test/unit/vector/date-vector-tests.ts
@@ -15,14 +15,40 @@
// specific language governing permissions and limitations
// under the License.
-import { DateDay, DateMillisecond, TimestampMillisecond, RecordBatchReader, Table, vectorFromArray } from 'apache-arrow';
+import {
+ DateDay, TimestampSecond, DateMillisecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, RecordBatchReader,
+ Table, vectorFromArray
+} from 'apache-arrow';
describe(`TimestampVector`, () => {
test(`Dates are stored in TimestampMillisecond`, () => {
const date = new Date('2023-02-01T12:34:56Z');
const vec = vectorFromArray([date]);
expect(vec.type).toBeInstanceOf(TimestampMillisecond);
- expect(vec.get(0)).toBe(date.valueOf());
+ expect(vec.get(0)).toBe(date.getTime());
+ });
+
+ test(`Correctly get back TimestampSecond from Date`, () => {
+ const date = new Date('2023-02-01T12:34:56Z');
+ const vec = vectorFromArray([date], new TimestampSecond);
+ expect(vec.type).toBeInstanceOf(TimestampSecond);
+ expect(vec.get(0)).toBe(date.getTime());
+ });
+
+ test(`Correctly get back TimestampMicrosecond from Date`, () => {
+ const date = new Date('2023-02-01T12:34:56Z');
+ const vec = vectorFromArray([date, 0.5], new TimestampMicrosecond);
+ expect(vec.type).toBeInstanceOf(TimestampMicrosecond);
+ expect(vec.get(0)).toBe(date.getTime());
+ expect(vec.get(1)).toBe(0.5);
+ });
+
+ test(`Correctly get back TimestampNanosecond from Date`, () => {
+ const date = new Date('2023-02-01T12:34:56Z');
+ const vec = vectorFromArray([date, 0.5], new TimestampNanosecond);
+ expect(vec.type).toBeInstanceOf(TimestampNanosecond);
+ expect(vec.get(0)).toBe(date.getTime());
+ expect(vec.get(1)).toBe(0.5);
});
});
@@ -33,7 +59,7 @@ describe(`DateVector`, () => {
const date32 = table.getChildAt<DateDay>(0)!;
for (const date of date32) {
const millis = expectedMillis.shift();
- expect(date).toEqual(millis === null ? null : new Date(millis!));
+ expect(date).toEqual(millis);
}
});
@@ -43,7 +69,7 @@ describe(`DateVector`, () => {
const date64 = table.getChildAt<DateMillisecond>(1)!;
for (const date of date64) {
const millis = expectedMillis.shift();
- expect(date).toEqual(millis === null ? null : new Date(millis!));
+ expect(date).toEqual(millis);
}
});
@@ -51,7 +77,7 @@ describe(`DateVector`, () => {
const dates = [new Date(1950, 1, 0)];
const vec = vectorFromArray(dates, new DateMillisecond());
for (const date of vec) {
- expect(date).toEqual(dates.shift());
+ expect(date).toEqual(dates.shift()?.getTime());
}
});
});
diff --git a/js/test/unit/vector/vector-tests.ts b/js/test/unit/vector/vector-tests.ts
index 881bf987b5..c05838d7a9 100644
--- a/js/test/unit/vector/vector-tests.ts
+++ b/js/test/unit/vector/vector-tests.ts
@@ -16,7 +16,7 @@
// under the License.
import {
- Bool, DateDay, DateMillisecond, Dictionary, Float64, Int32, List, makeVector, Struct, Timestamp, TimeUnit, Utf8, LargeUtf8, util, Vector, vectorFromArray, makeData
+ Bool, DateDay, DateMillisecond, Dictionary, Float64, Int32, List, makeVector, Struct, Utf8, LargeUtf8, util, Vector, vectorFromArray, makeData
} from 'apache-arrow';
describe(`makeVectorFromArray`, () => {
@@ -130,7 +130,7 @@ describe(`DateVector`, () => {
new Date(1988, 3, 25, 4, 5, 6),
new Date(1987, 2, 24, 7, 8, 9),
new Date(2018, 4, 12, 17, 30, 0)
- ];
+ ].map(v => v.getTime());
const vector = vectorFromArray(values, new DateMillisecond);
basicVectorTests(vector, values, extras);
});
@@ -141,7 +141,7 @@ describe(`DateVector`, () => {
new Date(Date.UTC(1988, 3, 25)),
new Date(Date.UTC(1987, 2, 24)),
new Date(Date.UTC(2018, 4, 12))
- ];
+ ].map(v => v.getTime());
const vector = vectorFromArray(values, new DateDay);
basicVectorTests(vector, values, extras);
@@ -288,17 +288,6 @@ describe(`toArray()`, () => {
const array = vector.toArray();
expect(array).toHaveLength(26);
});
-
- test(`when stride is 2`, () => {
- let d1 = vectorFromArray([0, 1, 2], new Timestamp(TimeUnit.MILLISECOND)).data[0];
- let d2 = vectorFromArray([3, 4, 5], new Timestamp(TimeUnit.MILLISECOND)).data[0];
-
- const vector = new Vector([d1, d2]);
-
- let array = Array.from(vector.toArray());
- expect(array).toHaveLength(6 * 2);
- expect(Array.from(array)).toMatchObject([0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0]);
- });
});
// Creates some basic tests for the given vector.