You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by pt...@apache.org on 2019/08/14 19:09:59 UTC
[arrow] branch master updated: ARROW-5741: [JS] Make numeric vector
from functions consistent with TypedArray.from
This is an automated email from the ASF dual-hosted git repository.
ptaylor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new e523db4 ARROW-5741: [JS] Make numeric vector from functions consistent with TypedArray.from
e523db4 is described below
commit e523db4585eb1dc4a7c3eb55870fd1589b8e4cd7
Author: ptaylor <pa...@me.com>
AuthorDate: Wed Aug 14 12:09:45 2019 -0700
ARROW-5741: [JS] Make numeric vector from functions consistent with TypedArray.from
This PR updates `IntVector.from()` and `FloatVector.from()` to use the new Builders to cast input values that don't exactly match the expected numeric type, per [this discussion](https://lists.apache.org/thread.html/b648a781cba7f10d5a6072ff2e7dab6c03e2d1f12e359d9261891486@%3Cdev.arrow.apache.org%3E) on the mailing list.
Previously, these methods behaved like `reinterpret_cast` in C++, but now they behave like numpy's `ndarray.astype()`. The new behavior of zero-copy vs. copied is:
* If the input type is the same as the Vector we want to create, the Vector is created zero-copy
* If if the input type is different (or the input isn't a TypedArray), we use the Builders to enumerate the values, cast each one to the desired type, and construct a new Vector from the results.
Closes [ARROW-5741](https://issues.apache.org/jira/browse/ARROW-5741)
Closes #4746 from trxcllnt/js/update-int-float-from-methods and squashes the following commits:
4b606a6c2 <ptaylor> fix lint
0ac754db2 <ptaylor> update half float conversions
96d981a56 <ptaylor> Rebased from master and squashed:
Authored-by: ptaylor <pa...@me.com>
Signed-off-by: ptaylor <pa...@me.com>
---
js/.vscode/launch.json | 3 +-
js/package-lock.json | 14 +--
js/package.json | 2 +-
js/src/Arrow.ts | 2 +
js/src/builder/float.ts | 8 +-
js/src/util/buffer.ts | 21 ----
js/src/util/math.ts | 105 ++++++++++++++++
js/src/vector/chunked.ts | 4 +-
js/src/vector/float.ts | 120 ++++++++++++++----
js/src/vector/int.ts | 184 ++++++++++++++++++++--------
js/src/visitor/get.ts | 5 +-
js/src/visitor/set.ts | 5 +-
js/test/generate-test-data.ts | 6 +-
js/test/unit/builders/primitive-tests.ts | 8 ++
js/test/unit/builders/utils.ts | 2 +-
js/test/unit/math-tests.ts | 47 +++++++
js/test/unit/vector/numeric-vector-tests.ts | 130 ++++++++++++++++++--
17 files changed, 537 insertions(+), 129 deletions(-)
diff --git a/js/.vscode/launch.json b/js/.vscode/launch.json
index ec825d2..43851ba 100644
--- a/js/.vscode/launch.json
+++ b/js/.vscode/launch.json
@@ -41,6 +41,7 @@
"test/unit/",
// "test/unit/builders/",
+ // Uncomment any of these to run individual test suites
// "test/unit/builders/builder-tests.ts",
// "test/unit/builders/int64-tests.ts",
// "test/unit/builders/uint64-tests.ts",
@@ -49,8 +50,8 @@
// "test/unit/builders/dictionary-tests.ts",
// "test/unit/builders/utf8-tests.ts",
- // Uncomment any of these to run individual test suites
// "test/unit/int-tests.ts",
+ // "test/unit/math-tests.ts",
// "test/unit/table-tests.ts",
// "test/unit/generated-data-tests.ts",
diff --git a/js/package-lock.json b/js/package-lock.json
index 153fbfd..9472895 100644
--- a/js/package-lock.json
+++ b/js/package-lock.json
@@ -1,6 +1,6 @@
{
"name": "apache-arrow",
- "version": "0.14.0-SNAPSHOT",
+ "version": "1.0.0-SNAPSHOT",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@@ -13411,8 +13411,8 @@
"dev": true
},
"typedoc": {
- "version": "github:TypeStrong/typedoc#f781b20dda3dfafd453cf540f58fcd8cf4473316",
- "from": "github:TypeStrong/typedoc",
+ "version": "0.15.0-0",
+ "resolved": "github:TypeStrong/typedoc#f781b20dda3dfafd453cf540f58fcd8cf4473316",
"dev": true,
"requires": {
"@types/minimatch": "3.0.3",
@@ -13425,7 +13425,7 @@
"progress": "^2.0.3",
"shelljs": "^0.8.3",
"typedoc-default-themes": "^0.6.0-0",
- "typescript": "3.5.x"
+ "typescript": "3.4.x"
},
"dependencies": {
"handlebars": {
@@ -13447,9 +13447,9 @@
"dev": true
},
"typescript": {
- "version": "3.5.2",
- "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.5.2.tgz",
- "integrity": "sha512-7KxJovlYhTX5RaRbUdkAXN1KUZ8PwWlTzQdHV6xNqvuFOs7+WBo10TQUqT19Q/Jz2hk5v9TQDIhyLhhJY4p5AA==",
+ "version": "3.4.5",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.4.5.tgz",
+ "integrity": "sha512-YycBxUb49UUhdNMU5aJ7z5Ej2XGmaIBL0x34vZ82fn3hGvD+bgrMrVDpatgz2f7YxUMJxMkbWxJZeAvDxVe7Vw==",
"dev": true
}
}
diff --git a/js/package.json b/js/package.json
index 11c7c89..24eee1a 100644
--- a/js/package.json
+++ b/js/package.json
@@ -103,7 +103,7 @@
"ts-jest": "24.0.2",
"ts-node": "8.2.0",
"tslint": "5.12.1",
- "typedoc": "TypeStrong/typedoc",
+ "typedoc": "0.15.0-0",
"typescript": "3.5.1",
"web-stream-tools": "0.0.1",
"web-streams-polyfill": "2.0.3",
diff --git a/js/src/Arrow.ts b/js/src/Arrow.ts
index d197169..88ebc5a 100644
--- a/js/src/Arrow.ts
+++ b/js/src/Arrow.ts
@@ -102,6 +102,7 @@ export { DataFrame, FilteredDataFrame, CountByResult, BindFunc, NextFunc } from
import * as util_bn_ from './util/bn';
import * as util_int_ from './util/int';
import * as util_bit_ from './util/bit';
+import * as util_math_ from './util/math';
import * as util_buffer_ from './util/buffer';
import * as util_vector_ from './util/vector';
import * as predicate from './compute/predicate';
@@ -112,6 +113,7 @@ export const util = {
...util_bn_,
...util_int_,
...util_bit_,
+ ...util_math_,
...util_buffer_,
...util_vector_
};
diff --git a/js/src/builder/float.ts b/js/src/builder/float.ts
index 6b0fb6b..dbf4c0d 100644
--- a/js/src/builder/float.ts
+++ b/js/src/builder/float.ts
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+import { float64ToUint16 } from '../util/math';
import { FixedWidthBuilder } from '../builder';
import { Float, Float16, Float32, Float64 } from '../type';
@@ -22,7 +23,12 @@ import { Float, Float16, Float32, Float64 } from '../type';
export class FloatBuilder<T extends Float = Float, TNull = any> extends FixedWidthBuilder<T, TNull> {}
/** @ignore */
-export class Float16Builder<TNull = any> extends FloatBuilder<Float16, TNull> {}
+export class Float16Builder<TNull = any> extends FloatBuilder<Float16, TNull> {
+ public setValue(index: number, value: number) {
+ // convert JS float64 to a uint16
+ this._values.set(index, float64ToUint16(value));
+ }
+}
/** @ignore */
export class Float32Builder<TNull = any> extends FloatBuilder<Float32, TNull> {
diff --git a/js/src/util/buffer.ts b/js/src/util/buffer.ts
index a84aa3e..9c98370 100644
--- a/js/src/util/buffer.ts
+++ b/js/src/util/buffer.ts
@@ -125,27 +125,6 @@ export function toArrayBufferView(ArrayBufferViewCtor: any, input: ArrayBufferVi
/** @ignore */ export const toUint8ClampedArray = (input: ArrayBufferViewInput) => toArrayBufferView(Uint8ClampedArray, input);
/** @ignore */
-export const toFloat16Array = (input: ArrayBufferViewInput) => {
- let floats: Float32Array | Float64Array | null = null;
- if (ArrayBuffer.isView(input)) {
- switch (input.constructor) {
- case Float32Array: floats = input as Float32Array; break;
- case Float64Array: floats = input as Float64Array; break;
- }
- } else if (isIterable(input)) {
- floats = toFloat64Array(input);
- }
- if (floats) {
- const u16s = new Uint16Array(floats.length);
- for (let i = -1, n = u16s.length; ++i < n;) {
- u16s[i] = (floats[i] * 32767) + 32767;
- }
- return u16s;
- }
- return toUint16Array(input);
-};
-
-/** @ignore */
type ArrayBufferViewIteratorInput = Iterable<ArrayBufferViewInput> | ArrayBufferViewInput;
/** @ignore */
diff --git a/js/src/util/math.ts b/js/src/util/math.ts
new file mode 100644
index 0000000..e9b600a
--- /dev/null
+++ b/js/src/util/math.ts
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const f64 = new Float64Array(1);
+const u32 = new Uint32Array(f64.buffer);
+
+/**
+ * Convert uint16 (logically a float16) to a JS float64. Inspired by numpy's `npy_half_to_double`:
+ * https://github.com/numpy/numpy/blob/5a5987291dc95376bb098be8d8e5391e89e77a2c/numpy/core/src/npymath/halffloat.c#L29
+ * @param h {number} the uint16 to convert
+ * @private
+ * @ignore
+ */
+export function uint16ToFloat64(h: number) {
+ let expo = (h & 0x7C00) >> 10;
+ let sigf = (h & 0x03FF) / 1024;
+ let sign = (-1) ** ((h & 0x8000) >> 15);
+ switch (expo) {
+ case 0x1F: return sign * (sigf ? NaN : 1 / 0);
+ case 0x00: return sign * (sigf ? 6.103515625e-5 * sigf : 0);
+ }
+ return sign * (2 ** (expo - 15)) * (1 + sigf);
+}
+
+/**
+ * Convert a float64 to uint16 (assuming the float64 is logically a float16). Inspired by numpy's `npy_double_to_half`:
+ * https://github.com/numpy/numpy/blob/5a5987291dc95376bb098be8d8e5391e89e77a2c/numpy/core/src/npymath/halffloat.c#L43
+ * @param d {number} The float64 to convert
+ * @private
+ * @ignore
+ */
+export function float64ToUint16(d: number) {
+
+ if (d !== d) { return 0x7E00; } // NaN
+
+ f64[0] = d;
+
+ // Magic numbers:
+ // 0x80000000 = 10000000 00000000 00000000 00000000 -- masks the 32nd bit
+ // 0x7ff00000 = 01111111 11110000 00000000 00000000 -- masks the 21st-31st bits
+ // 0x000fffff = 00000000 00001111 11111111 11111111 -- masks the 1st-20th bit
+
+ let sign = (u32[1] & 0x80000000) >> 16 & 0xFFFF;
+ let expo = (u32[1] & 0x7ff00000), sigf = 0x0000;
+
+ if (expo >= 0x40f00000) {
+ //
+ // If exponent overflowed, the float16 is either NaN or Infinity.
+ // Rules to propagate the sign bit: mantissa > 0 ? NaN : +/-Infinity
+ //
+ // Magic numbers:
+ // 0x40F00000 = 01000000 11110000 00000000 00000000 -- 6-bit exponent overflow
+ // 0x7C000000 = 01111100 00000000 00000000 00000000 -- masks the 27th-31st bits
+ //
+ // returns:
+ // qNaN, aka 32256 decimal, 0x7E00 hex, or 01111110 00000000 binary
+ // sNaN, aka 32000 decimal, 0x7D00 hex, or 01111101 00000000 binary
+ // +inf, aka 31744 decimal, 0x7C00 hex, or 01111100 00000000 binary
+ // -inf, aka 64512 decimal, 0xFC00 hex, or 11111100 00000000 binary
+ //
+ // If mantissa is greater than 23 bits, set to +Infinity like numpy
+ if (u32[0] > 0) {
+ expo = 0x7C00;
+ } else {
+ expo = (expo & 0x7C000000) >> 16;
+ sigf = (u32[1] & 0x000fffff) >> 10;
+ }
+ } else if (expo <= 0x3f000000) {
+ //
+ // If exponent underflowed, the float is either signed zero or subnormal.
+ //
+ // Magic numbers:
+ // 0x3F000000 = 00111111 00000000 00000000 00000000 -- 6-bit exponent underflow
+ //
+ sigf = 0x100000 + (u32[1] & 0x000fffff);
+ sigf = 0x100000 + (sigf << ((expo >> 20) - 998)) >> 21;
+ expo = 0;
+ } else {
+ //
+ // No overflow or underflow, rebase the exponent and round the mantissa
+ // Magic numbers:
+ // 0x200 = 00000010 00000000 -- masks off the 10th bit
+ //
+
+ // Ensure the first mantissa bit (the 10th one) is 1 and round
+ expo = (expo - 0x3f000000) >> 10;
+ sigf = ((u32[1] & 0x000fffff) + 0x200) >> 10;
+ }
+
+ return sign | expo | sigf & 0xFFFF;
+}
diff --git a/js/src/vector/chunked.ts b/js/src/vector/chunked.ts
index 5a0a483..f594e6b 100644
--- a/js/src/vector/chunked.ts
+++ b/js/src/vector/chunked.ts
@@ -267,9 +267,9 @@ const typedSet = (src: TypedArray, dst: TypedArray, offset: number) => {
/** @ignore */
const arraySet = (src: any[], dst: any[], offset: number) => {
- let idx = offset - 1;
+ let idx = offset;
for (let i = -1, n = src.length; ++i < n;) {
- dst[++idx] = src[i];
+ dst[idx++] = src[i];
}
return idx;
};
diff --git a/js/src/vector/float.ts b/js/src/vector/float.ts
index 87296b0..cb15d15 100644
--- a/js/src/vector/float.ts
+++ b/js/src/vector/float.ts
@@ -17,38 +17,88 @@
import { Data } from '../data';
import { Vector } from '../vector';
+import { Chunked } from './chunked';
import { BaseVector } from './base';
-import { VectorType as V } from '../interfaces';
-import { Float, Float16, Float32, Float64 } from '../type';
-import { toFloat16Array, toFloat32Array, toFloat64Array } from '../util/buffer';
+import { VectorBuilderOptions } from './index';
+import { vectorFromValuesWithType } from './index';
+import { VectorBuilderOptionsAsync } from './index';
+import { Float, Float16, Float32, Float64, FloatArray } from '../type';
+import { VectorType as V, TypedArrayConstructor } from '../interfaces';
+
+/** @ignore */
+type FloatVectorConstructors =
+ typeof FloatVector |
+ typeof Float16Vector |
+ typeof Float32Vector |
+ typeof Float64Vector ;
+
+/** @ignore */
+type FromInput<T extends Float, TNull = any> =
+ FloatArray |
+ Iterable<T['TValue'] | TNull> |
+ AsyncIterable<T['TValue'] | TNull> |
+ VectorBuilderOptions<T, TNull> |
+ VectorBuilderOptionsAsync<T, TNull> ;
+
+/** @ignore */
+type FloatArrayCtor = TypedArrayConstructor<FloatArray>;
/** @ignore */
export class FloatVector<T extends Float = Float> extends BaseVector<T> {
- public static from(this: typeof FloatVector, data: Float16['TArray']): Float16Vector;
- public static from(this: typeof FloatVector, data: Float32['TArray']): Float32Vector;
- public static from(this: typeof FloatVector, data: Float64['TArray']): Float64Vector;
- public static from<T extends Float>(this: typeof FloatVector, data: T['TArray']): V<T>;
+ // Guaranteed zero-copy variants
+ public static from(this: typeof FloatVector, input: Uint16Array): Float16Vector;
+ public static from(this: typeof FloatVector, input: Float32Array): Float32Vector;
+ public static from(this: typeof FloatVector, input: Float64Array): Float64Vector;
+
+ // Zero-copy if input is a TypedArray of the same type as the
+ // Vector that from is called on, otherwise uses the Builders
+ public static from<TNull = any>(this: typeof Float16Vector, input: FromInput<Float16, TNull>): Float16Vector;
+ public static from<TNull = any>(this: typeof Float32Vector, input: FromInput<Float32, TNull>): Float32Vector;
+ public static from<TNull = any>(this: typeof Float64Vector, input: FromInput<Float64, TNull>): Float64Vector;
- public static from(this: typeof Float16Vector, data: Float16['TArray'] | Iterable<number>): Float16Vector;
- public static from(this: typeof Float32Vector, data: Float32['TArray'] | Iterable<number>): Float32Vector;
- public static from(this: typeof Float64Vector, data: Float64['TArray'] | Iterable<number>): Float64Vector;
+ // Not zero-copy
+ public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: Iterable<T['TValue'] | TNull>): V<T>;
+ public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: AsyncIterable<T['TValue'] | TNull>): Promise<V<T>>;
+ public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: VectorBuilderOptions<T, TNull>): Chunked<T>;
+ public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>;
/** @nocollapse */
- public static from<T extends Float>(data: T['TArray']) {
- let type: Float | null = null;
- switch (this) {
- case Float16Vector: data = toFloat16Array(data); break;
- case Float32Vector: data = toFloat32Array(data); break;
- case Float64Vector: data = toFloat64Array(data); break;
+ public static from<T extends Float, TNull = any>(this: FloatVectorConstructors, input: FromInput<T, TNull>) {
+
+ let ArrowType = vectorTypeToDataType(this);
+
+ if ((input instanceof ArrayBuffer) || ArrayBuffer.isView(input)) {
+ let InputType = arrayTypeToDataType(input.constructor as FloatArrayCtor) || ArrowType;
+ // Special case, infer the Arrow DataType from the input if calling the base
+ // FloatVector.from with a TypedArray, e.g. `FloatVector.from(new Float32Array())`
+ if (ArrowType === null) {
+ ArrowType = InputType;
+ }
+ // If the DataType inferred from the Vector constructor matches the
+ // DataType inferred from the input arguments, return zero-copy view
+ if (ArrowType && ArrowType === InputType) {
+ let type = new ArrowType();
+ let length = input.byteLength / type.ArrayType.BYTES_PER_ELEMENT;
+ // If the ArrowType is Float16 but the input type isn't a Uint16Array,
+ // let the Float16Builder handle casting the input values to Uint16s.
+ if (!convertTo16Bit(ArrowType, input.constructor)) {
+ return Vector.new(Data.Float(type, 0, length, 0, null, input as FloatArray));
+ }
+ }
+ }
+
+ if (ArrowType) {
+ // If the DataType inferred from the Vector constructor is different than
+ // the DataType inferred from the input TypedArray, or if input isn't a
+ // TypedArray, use the Builders to construct the result Vector
+ return vectorFromValuesWithType(() => new ArrowType!() as T, input);
}
- switch (data.constructor) {
- case Uint16Array: type = new Float16(); break;
- case Float32Array: type = new Float32(); break;
- case Float64Array: type = new Float64(); break;
+
+ if ((input instanceof DataView) || (input instanceof ArrayBuffer)) {
+ throw new TypeError(`Cannot infer float type from instance of ${input.constructor.name}`);
}
- return type !== null
- ? Vector.new(Data.Float(type, 0, data.length, 0, null, data))
- : (() => { throw new TypeError('Unrecognized FloatVector input'); })();
+
+ throw new TypeError('Unrecognized FloatVector input');
}
}
@@ -68,3 +118,27 @@ export class Float16Vector extends FloatVector<Float16> {
export class Float32Vector extends FloatVector<Float32> {}
/** @ignore */
export class Float64Vector extends FloatVector<Float64> {}
+
+const convertTo16Bit = (typeCtor: any, dataCtor: any) => {
+ return (typeCtor === Float16) && (dataCtor !== Uint16Array);
+};
+
+/** @ignore */
+const arrayTypeToDataType = (ctor: FloatArrayCtor) => {
+ switch (ctor) {
+ case Uint16Array: return Float16;
+ case Float32Array: return Float32;
+ case Float64Array: return Float64;
+ default: return null;
+ }
+};
+
+/** @ignore */
+const vectorTypeToDataType = (ctor: FloatVectorConstructors) => {
+ switch (ctor) {
+ case Float16Vector: return Float16;
+ case Float32Vector: return Float32;
+ case Float64Vector: return Float64;
+ default: return null;
+ }
+};
diff --git a/js/src/vector/int.ts b/js/src/vector/int.ts
index ca57b83..74c284e 100644
--- a/js/src/vector/int.ts
+++ b/js/src/vector/int.ts
@@ -17,70 +17,111 @@
import { Data } from '../data';
import { Vector } from '../vector';
+import { Chunked } from './chunked';
import { BaseVector } from './base';
-import { VectorType as V } from '../interfaces';
-import { Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64 } from '../type';
-import {
- toInt8Array, toInt16Array, toInt32Array,
- toUint8Array, toUint16Array, toUint32Array,
- toBigInt64Array, toBigUint64Array
-} from '../util/buffer';
+import { VectorBuilderOptions } from './index';
+import { vectorFromValuesWithType } from './index';
+import { VectorBuilderOptionsAsync } from './index';
+import { BigInt64Array, BigUint64Array } from '../util/compat';
+import { toBigInt64Array, toBigUint64Array } from '../util/buffer';
+import { Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, IntArray } from '../type';
+import { VectorType as V, TypedArrayConstructor, BigIntArrayConstructor, BigIntArray } from '../interfaces';
+
+/** @ignore */
+type IntVectorConstructors =
+ typeof IntVector |
+ typeof Int8Vector |
+ typeof Int16Vector |
+ typeof Int32Vector |
+ typeof Uint8Vector |
+ typeof Uint16Vector |
+ typeof Uint32Vector |
+ typeof Int64Vector |
+ typeof Uint64Vector ;
+
+/** @ignore */
+type FromInput<T extends Int, TNull = any> =
+ IntArray | BigIntArray |
+ Iterable<T['TValue'] | TNull> |
+ AsyncIterable<T['TValue'] | TNull> |
+ VectorBuilderOptions<T, TNull> |
+ VectorBuilderOptionsAsync<T, TNull> ;
+
+/** @ignore */
+type FromArgs<T extends Int, TNull = any> = [FromInput<T, TNull>, boolean?];
+
+/** @ignore */
+type IntArrayCtor = TypedArrayConstructor<IntArray> | BigIntArrayConstructor<BigIntArray>;
/** @ignore */
export class IntVector<T extends Int = Int> extends BaseVector<T> {
- public static from(this: typeof IntVector, data: Int8Array): Int8Vector;
- public static from(this: typeof IntVector, data: Int16Array): Int16Vector;
- public static from(this: typeof IntVector, data: Int32Array): Int32Vector;
- public static from(this: typeof IntVector, data: Uint8Array): Uint8Vector;
- public static from(this: typeof IntVector, data: Uint16Array): Uint16Vector;
- public static from(this: typeof IntVector, data: Uint32Array): Uint32Vector;
+ // Guaranteed zero-copy variants
+ public static from(this: typeof IntVector, input: Int8Array): Int8Vector;
+ public static from(this: typeof IntVector, input: Int16Array): Int16Vector;
+ public static from(this: typeof IntVector, input: Int32Array): Int32Vector;
+ public static from(this: typeof IntVector, input: BigInt64Array): Int64Vector;
+ public static from(this: typeof IntVector, input: Int32Array, is64bit: true): Int64Vector;
+ public static from(this: typeof IntVector, input: Uint8Array): Uint8Vector;
+ public static from(this: typeof IntVector, input: Uint16Array): Uint16Vector;
+ public static from(this: typeof IntVector, input: Uint32Array): Uint32Vector;
+ public static from(this: typeof IntVector, input: BigUint64Array): Uint64Vector;
+ public static from(this: typeof IntVector, input: Uint32Array, is64bit: true): Uint64Vector;
- // @ts-ignore
- public static from(this: typeof IntVector, data: Int32Array, is64: true): Int64Vector;
- public static from(this: typeof IntVector, data: Uint32Array, is64: true): Uint64Vector;
- public static from<T extends Int>(this: typeof IntVector, data: T['TArray']): V<T>;
-
- public static from(this: typeof Int8Vector, data: Int8['TArray'] | Iterable<number>): Int8Vector;
- public static from(this: typeof Int16Vector, data: Int16['TArray'] | Iterable<number>): Int16Vector;
- public static from(this: typeof Int32Vector, data: Int32['TArray'] | Iterable<number>): Int32Vector;
- public static from(this: typeof Int64Vector, data: Int32['TArray'] | Iterable<number>): Int64Vector;
- public static from(this: typeof Uint8Vector, data: Uint8['TArray'] | Iterable<number>): Uint8Vector;
- public static from(this: typeof Uint16Vector, data: Uint16['TArray'] | Iterable<number>): Uint16Vector;
- public static from(this: typeof Uint32Vector, data: Uint32['TArray'] | Iterable<number>): Uint32Vector;
- public static from(this: typeof Uint64Vector, data: Uint32['TArray'] | Iterable<number>): Uint64Vector;
+ // Zero-copy if input is a TypedArray of the same type as the
+ // Vector that from is called on, otherwise uses the Builders
+ public static from<TNull = any>(this: typeof Int8Vector, input: FromInput<Int8, TNull>): Int8Vector;
+ public static from<TNull = any>(this: typeof Int16Vector, input: FromInput<Int16, TNull>): Int16Vector;
+ public static from<TNull = any>(this: typeof Int32Vector, input: FromInput<Int32, TNull>): Int32Vector;
+ public static from<TNull = any>(this: typeof Int64Vector, input: FromInput<Int64, TNull>): Int64Vector;
+ public static from<TNull = any>(this: typeof Uint8Vector, input: FromInput<Uint8, TNull>): Uint8Vector;
+ public static from<TNull = any>(this: typeof Uint16Vector, input: FromInput<Uint16, TNull>): Uint16Vector;
+ public static from<TNull = any>(this: typeof Uint32Vector, input: FromInput<Uint32, TNull>): Uint32Vector;
+ public static from<TNull = any>(this: typeof Uint64Vector, input: FromInput<Uint64, TNull>): Uint64Vector;
+ // Not zero-copy
+ public static from<T extends Int, TNull = any>(this: typeof IntVector, input: Iterable<T['TValue'] | TNull>): V<T>;
+ public static from<T extends Int, TNull = any>(this: typeof IntVector, input: AsyncIterable<T['TValue'] | TNull>): Promise<V<T>>;
+ public static from<T extends Int, TNull = any>(this: typeof IntVector, input: VectorBuilderOptions<T, TNull>): Chunked<T>;
+ public static from<T extends Int, TNull = any>(this: typeof IntVector, input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>;
/** @nocollapse */
- public static from<T extends Int>(data: T['TArray'], is64?: boolean) {
- let length: number = 0;
- let type: Int | null = null;
- switch (this) {
- case Int8Vector: data = toInt8Array(data); is64 = false; break;
- case Int16Vector: data = toInt16Array(data); is64 = false; break;
- case Int32Vector: data = toInt32Array(data); is64 = false; break;
- case Int64Vector: data = toInt32Array(data); is64 = true; break;
- case Uint8Vector: data = toUint8Array(data); is64 = false; break;
- case Uint16Vector: data = toUint16Array(data); is64 = false; break;
- case Uint32Vector: data = toUint32Array(data); is64 = false; break;
- case Uint64Vector: data = toUint32Array(data); is64 = true; break;
- }
- if (is64 === true) {
- length = data.length * 0.5;
- type = data instanceof Int32Array ? new Int64() : new Uint64();
- } else {
- length = data.length;
- switch (data.constructor) {
- case Int8Array: type = new Int8(); break;
- case Int16Array: type = new Int16(); break;
- case Int32Array: type = new Int32(); break;
- case Uint8Array: type = new Uint8(); break;
- case Uint16Array: type = new Uint16(); break;
- case Uint32Array: type = new Uint32(); break;
+ public static from<T extends Int, TNull = any>(this: IntVectorConstructors, ...args: FromArgs<T, TNull>) {
+
+ let [input, is64bit = false] = args;
+ let ArrowType = vectorTypeToDataType(this, is64bit);
+
+ if ((input instanceof ArrayBuffer) || ArrayBuffer.isView(input)) {
+ let InputType = arrayTypeToDataType(input.constructor as IntArrayCtor, is64bit) || ArrowType;
+ // Special case, infer the Arrow DataType from the input if calling the base
+ // IntVector.from with a TypedArray, e.g. `IntVector.from(new Int32Array())`
+ if (ArrowType === null) {
+ ArrowType = InputType;
+ }
+ // If the DataType inferred from the Vector constructor matches the
+ // DataType inferred from the input arguments, return zero-copy view
+ if (ArrowType && ArrowType === InputType) {
+ let type = new ArrowType();
+ let length = input.byteLength / type.ArrayType.BYTES_PER_ELEMENT;
+ // If the ArrowType is 64bit but the input type is 32bit pairs, update the logical length
+ if (convert32To64Bit(ArrowType, input.constructor)) {
+ length *= 0.5;
+ }
+ return Vector.new(Data.Int(type, 0, length, 0, null, input as IntArray));
}
}
- return type !== null
- ? Vector.new(Data.Int(type, 0, length, 0, null, data))
- : (() => { throw new TypeError('Unrecognized IntVector input'); })();
+
+ if (ArrowType) {
+ // If the DataType inferred from the Vector constructor is different than
+ // the DataType inferred from the input TypedArray, or if input isn't a
+ // TypedArray, use the Builders to construct the result Vector
+ return vectorFromValuesWithType(() => new ArrowType!() as T, input);
+ }
+
+ if ((input instanceof DataView) || (input instanceof ArrayBuffer)) {
+ throw new TypeError(`Cannot infer integer type from instance of ${input.constructor.name}`);
+ }
+
+ throw new TypeError('Unrecognized IntVector input');
}
}
@@ -119,3 +160,38 @@ export class Uint64Vector extends IntVector<Uint64> {
return this._values64 || (this._values64 = this.toBigUint64Array());
}
}
+
+const convert32To64Bit = (typeCtor: any, dataCtor: any) => {
+ return (typeCtor === Int64 || typeCtor === Uint64) &&
+ (dataCtor === Int32Array || dataCtor === Uint32Array);
+};
+
+/** @ignore */
+const arrayTypeToDataType = (ctor: IntArrayCtor, is64bit: boolean) => {
+ switch (ctor) {
+ case Int8Array: return Int8;
+ case Int16Array: return Int16;
+ case Int32Array: return is64bit ? Int64 : Int32;
+ case BigInt64Array: return Int64;
+ case Uint8Array: return Uint8;
+ case Uint16Array: return Uint16;
+ case Uint32Array: return is64bit ? Uint64 : Uint32;
+ case BigUint64Array: return Uint64;
+ default: return null;
+ }
+};
+
+/** @ignore */
+const vectorTypeToDataType = (ctor: IntVectorConstructors, is64bit: boolean) => {
+ switch (ctor) {
+ case Int8Vector: return Int8;
+ case Int16Vector: return Int16;
+ case Int32Vector: return is64bit ? Int64 : Int32;
+ case Int64Vector: return Int64;
+ case Uint8Vector: return Uint8;
+ case Uint16Vector: return Uint16;
+ case Uint32Vector: return is64bit ? Uint64 : Uint32;
+ case Uint64Vector: return Uint64;
+ default: return null;
+ }
+};
diff --git a/js/src/visitor/get.ts b/js/src/visitor/get.ts
index 9a09a8b..eea1409 100644
--- a/js/src/visitor/get.ts
+++ b/js/src/visitor/get.ts
@@ -18,8 +18,9 @@
import { Data } from '../data';
import { BN } from '../util/bn';
import { Visitor } from '../visitor';
-import { VectorType } from '../interfaces';
import { decodeUtf8 } from '../util/utf8';
+import { VectorType } from '../interfaces';
+import { uint16ToFloat64 } from '../util/math';
import { Type, UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum';
import {
DataType, Dictionary,
@@ -123,7 +124,7 @@ const getDateMillisecond = <T extends DateMillisecond>({ values }: Vecto
/** @ignore */
const getNumeric = <T extends Numeric1X> ({ stride, values }: VectorType<T>, index: number): T['TValue'] => values[stride * index];
/** @ignore */
-const getFloat16 = <T extends Float16> ({ stride, values }: VectorType<T>, index: number): T['TValue'] => (values[stride * index] - 32767) / 32767;
+const getFloat16 = <T extends Float16> ({ stride, values }: VectorType<T>, index: number): T['TValue'] => uint16ToFloat64(values[stride * index]);
/** @ignore */
const getBigInts = <T extends Numeric2X>({ stride, values, type }: VectorType<T>, index: number): T['TValue'] => <any> BN.new(values.subarray(stride * index, stride * (index + 1)), type.isSigned);
/** @ignore */
diff --git a/js/src/visitor/set.ts b/js/src/visitor/set.ts
index 460c186..c7adc65 100644
--- a/js/src/visitor/set.ts
+++ b/js/src/visitor/set.ts
@@ -17,8 +17,9 @@
import { Data } from '../data';
import { Visitor } from '../visitor';
-import { VectorType } from '../interfaces';
import { encodeUtf8 } from '../util/utf8';
+import { VectorType } from '../interfaces';
+import { float64ToUint16 } from '../util/math';
import { toArrayBufferView } from '../util/buffer';
import { Type, UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum';
import {
@@ -131,7 +132,7 @@ const setDateMillisecond = <T extends DateMillisecond>({ values }: Vecto
/** @ignore */
const setNumeric = <T extends Numeric1X> ({ stride, values }: VectorType<T>, index: number, value: T['TValue']): void => { values[stride * index] = value; };
/** @ignore */
-const setFloat16 = <T extends Float16> ({ stride, values }: VectorType<T>, index: number, value: T['TValue']): void => { values[stride * index] = (value * 32767) + 32767; };
+const setFloat16 = <T extends Float16> ({ stride, values }: VectorType<T>, index: number, value: T['TValue']): void => { values[stride * index] = float64ToUint16(value); };
/** @ignore */
const setNumericX2 = <T extends Numeric2X> (vector: VectorType<T>, index: number, value: T['TValue']): void => {
switch (typeof value) {
diff --git a/js/test/generate-test-data.ts b/js/test/generate-test-data.ts
index ff6056d..28edb20 100644
--- a/js/test/generate-test-data.ts
+++ b/js/test/generate-test-data.ts
@@ -41,7 +41,8 @@ import {
Interval, IntervalDayTime, IntervalYearMonth,
FixedSizeList,
Map_,
- DateUnit, TimeUnit, UnionMode
+ DateUnit, TimeUnit, UnionMode,
+ util
} from './Arrow';
type TKeys = Int8 | Int16 | Int32 | Uint8 | Uint16 | Uint32;
@@ -269,7 +270,8 @@ function generateFloat<T extends Float>(this: TestDataVectorGenerator, type: T,
const values = memoize(() => {
const values = [] as (number | null)[];
iterateBitmap(length, nullBitmap, (i, valid) => {
- values[i] = !valid ? null : precision > 0 ? data[i] : (data[i] - 32767) / 32767;
+ // values[i] = !valid ? null : precision > 0 ? data[i] : (data[i] - 32767) / 32767;
+ values[i] = !valid ? null : precision > 0 ? data[i] : util.uint16ToFloat64(data[i]);
});
return values;
});
diff --git a/js/test/unit/builders/primitive-tests.ts b/js/test/unit/builders/primitive-tests.ts
index 1734081..994d78e 100644
--- a/js/test/unit/builders/primitive-tests.ts
+++ b/js/test/unit/builders/primitive-tests.ts
@@ -144,3 +144,11 @@ type PrimitiveTypeOpts<T extends DataType> = [
}
});
});
+
+describe('Float16Builder', () => {
+ const encode = encodeAll(() => new Float16());
+ it(`encodes the weird values`, async () => {
+ const vals = [0, 5.960464477539063e-8, NaN, 65504, 2, -0];
+ validateVector(vals, await encode(vals, []), []);
+ });
+});
diff --git a/js/test/unit/builders/utils.ts b/js/test/unit/builders/utils.ts
index c6a29a7..2e68b78 100644
--- a/js/test/unit/builders/utils.ts
+++ b/js/test/unit/builders/utils.ts
@@ -83,7 +83,7 @@ export const uint64sNoNulls = (length = 20) => Array.from({ length }, (_, i) =>
default: return bn[0];
}
});
-export const float16sNoNulls = (length = 20) => Array.from(new Uint16Array(randomBytes(length * Uint16Array.BYTES_PER_ELEMENT).buffer)).map((x) => (x - 32767) / 32767);
+export const float16sNoNulls = (length = 20) => Array.from(new Uint16Array(randomBytes(length * Uint16Array.BYTES_PER_ELEMENT).buffer)).map(util.uint16ToFloat64);
export const float32sNoNulls = (length = 20) => Array.from(new Float32Array(randomBytes(length * Float32Array.BYTES_PER_ELEMENT).buffer));
export const float64sNoNulls = (length = 20) => Array.from(new Float64Array(randomBytes(length * Float64Array.BYTES_PER_ELEMENT).buffer));
diff --git a/js/test/unit/math-tests.ts b/js/test/unit/math-tests.ts
new file mode 100644
index 0000000..2baaa03
--- /dev/null
+++ b/js/test/unit/math-tests.ts
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as Arrow from '../Arrow';
+const { float64ToUint16, uint16ToFloat64 } = Arrow.util;
+
+describe('Float16', () => {
+ test('Uint16 to Float64 works', () => {
+
+ const uNaN = 0x7E00 /* NaN */;
+ const pInf = 0x7C00 /* 1/0 */;
+ const nInf = 0xFC00 /*-1/0 */;
+ let value = 0, expected = value;
+
+ do {
+
+ expected = value;
+
+ // if exponent is all 1s, either Infinity or NaN
+ if ((value & 0x7C00) === 0x7C00) {
+ // if significand, must be NaN
+ if (((value << 6) & 0xFFFF) !== 0) {
+ expected = uNaN;
+ } else {
+ // otherwise +/- Infinity
+ expected = (value >>> 15) !== 0 ? nInf : pInf;
+ }
+ }
+
+ expect(float64ToUint16(uint16ToFloat64(value))).toEqual(expected);
+ } while (++value < 65536);
+ });
+});
diff --git a/js/test/unit/vector/numeric-vector-tests.ts b/js/test/unit/vector/numeric-vector-tests.ts
index 3ada698..621e6b7 100644
--- a/js/test/unit/vector/numeric-vector-tests.ts
+++ b/js/test/unit/vector/numeric-vector-tests.ts
@@ -25,20 +25,41 @@ import {
Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector,
} from '../../Arrow';
+const { float64ToUint16, uint16ToFloat64 } = util;
import { VectorType as V } from '../../../src/interfaces';
import { TypedArray, TypedArrayConstructor } from '../../../src/interfaces';
import { BigIntArray, BigIntArrayConstructor } from '../../../src/interfaces';
-const { joinUint8Arrays } = util;
-const uint16ToFloat64 = (x: number) => (x - 32767) / 32767;
+const { joinUint8Arrays, BN } = util;
const uint16ToFloat64Array = (b: ArrayBuffer) => new Float64Array([...new Uint16Array(b)].map(uint16ToFloat64));
-const randomBytes = (n: number) => Uint16Array.from({ length: n / 2 }, () => (Math.random() * 65536) | 0).buffer;
+const randomBytes = (n: number) => new Uint16Array([
+ ...Uint16Array.from([0, 65535]),
+ ...Uint16Array.from({ length: (n / 2) - 2 }, () => (Math.random() * 65536) | 0),
+]).buffer;
+const toBigNumsArray = (values: Int32Array | Uint32Array) => {
+ const array = new Array(values.length * 0.5);
+ for (let i = -1, n = values.length * 0.5; ++i < n;) {
+ array[i] = BN.new(values.subarray(i * 2, i * 2 + 2))[Symbol.toPrimitive]();
+ }
+ return array;
+};
const testValueBuffers = Array.from({ length: 5 }, () => randomBytes(64));
const testValuesBuffer = joinUint8Arrays(testValueBuffers.map((b) => new Uint8Array(b)))[0].buffer;
const checkType = <T, R extends T>(Ctor: new (...args: any) => T, inst: R) => expect(inst).toBeInstanceOf(Ctor);
const valuesArray = <T extends TypedArray>(ArrayType: TypedArrayConstructor<T>) => [...valuesTyped<T>(ArrayType)];
+const valuesArray64 = <T extends TypedArray>(ArrayType: TypedArrayConstructor<T>) => {
+ const typed = valuesTyped<T>(ArrayType);
+ const array = new Array(typed.length * 0.5);
+ for (let i = -1, n = array.length; ++i < n;) {
+ // Interleave regular Arrays and TypedArrays to cover more surface area
+ array[i] = i % 2 === 0
+ ? [...typed.subarray(i * 2, (i + 1) * 2)]
+ : typed.subarray(i * 2, (i + 1) * 2);
+ }
+ return array;
+};
const valuesTyped = <T extends TypedArray>(ArrayType: TypedArrayConstructor<T>) => new ArrayType(testValuesBuffer);
const bigIntValuesTyped = <T extends BigIntArray>(ArrayType: BigIntArrayConstructor<T>) => new ArrayType(testValuesBuffer);
const bigIntValuesArray = <T extends BigIntArray>(ArrayType: BigIntArrayConstructor<T>) => [...bigIntValuesTyped<T>(ArrayType)];
@@ -47,7 +68,7 @@ describe(`FloatVector`, () => {
describe(`FloatVector.from infers the type from the input TypedArray`, () => {
- const u16s = valuesTyped(Uint16Array);
+ const u16s = valuesTyped(Uint16Array).map((x) => float64ToUint16(uint16ToFloat64(x)));
const f16s = valuesArray(Uint16Array).map(uint16ToFloat64);
const f32s = valuesTyped(Float32Array);
const f64s = valuesTyped(Float64Array);
@@ -68,17 +89,37 @@ describe(`FloatVector`, () => {
testAndValidateVector(f64Vec, f64s);
});
+ describe(`FloatVector.from casts the input values to the correct float type`, () => {
+
+ const u16s = valuesTyped(Uint16Array).map((x) => float64ToUint16(uint16ToFloat64(x)));
+ const f16s = valuesArray(Uint16Array).map(uint16ToFloat64);
+ const f16Vec_ = FloatVector.from(u16s);
+
+ const f16Vec = Float16Vector.from(f16Vec_);
+ const f32Vec = Float32Vector.from(f16Vec_);
+ const f64Vec = Float64Vector.from(f16Vec_);
+
+ // test strong typing at compile-time
+ test(`return type is correct`, () => checkType(Float16Vector, f16Vec));
+ test(`return type is correct`, () => checkType(Float32Vector, f32Vec));
+ test(`return type is correct`, () => checkType(Float64Vector, f64Vec));
+
+ testAndValidateVector(f16Vec, u16s, f16s);
+ testAndValidateVector(f32Vec, Float32Array.from(f16s));
+ testAndValidateVector(f64Vec, Float64Array.from(f16s));
+ });
+
describe(`Float16Vector`, () => {
testFloatVector(Float16, valuesArray(Uint16Array).map(uint16ToFloat64));
describe(`Float16Vector.from accepts regular Arrays`, () => {
- const u16s = valuesTyped(Uint16Array);
+ const u16s = valuesTyped(Uint16Array).map((x) => float64ToUint16(uint16ToFloat64(x)));
const f16s = valuesArray(Uint16Array).map(uint16ToFloat64);
const vector = Float16Vector.from(f16s);
test(`return type is correct`, () => checkType(Float16Vector, vector));
testAndValidateVector(vector, u16s, f16s);
});
describe(`Float16Vector.from accepts Uint16Arrays`, () => {
- const u16s = valuesTyped(Uint16Array);
+ const u16s = valuesTyped(Uint16Array).map((x) => float64ToUint16(uint16ToFloat64(x)));
const f16s = valuesArray(Uint16Array).map(uint16ToFloat64);
const vector = Float16Vector.from(u16s);
test(`return type is correct`, () => checkType(Float16Vector, vector));
@@ -139,14 +180,81 @@ describe(`IntVector`, () => {
expect(() => IntVector.from(<any> {})).toThrow('Unrecognized IntVector input');
});
+ const bigI64s = BigInt64Array.from(toBigNumsArray(i64s));
+ const bigU64s = BigUint64Array.from(toBigNumsArray(u64s));
+
testAndValidateVector(i8Vec, i8s);
testAndValidateVector(i16Vec, i16s);
testAndValidateVector(i32Vec, i32s);
+ // This tests when values are represented as pairs of lo, hi
testAndValidateVector(i64Vec, i64s);
+ // This tests when values are represented as native JS bigints
+ testAndValidateVector(i64Vec, i64s, [...bigI64s]);
testAndValidateVector(u8Vec, u8s);
testAndValidateVector(u16Vec, u16s);
testAndValidateVector(u32Vec, u32s);
+ // This tests when values are represented as pairs of lo, hi
testAndValidateVector(u64Vec, u64s);
+ // This tests when values are represented as native JS bigints
+ testAndValidateVector(u64Vec, u64s, [...bigU64s]);
+ });
+
+ describe('IntVector.from casts the input values to the correct integer type', () => {
+
+ const i8s = valuesTyped(Int8Array);
+ const i16s = valuesTyped(Int16Array);
+ const i32s = valuesTyped(Int32Array);
+ const i64s = valuesTyped(Int32Array);
+ const u8s = valuesTyped(Uint8Array);
+ const u16s = valuesTyped(Uint16Array);
+ const u32s = valuesTyped(Uint32Array);
+ const u64s = valuesTyped(Uint32Array);
+ const i8Vec_ = IntVector.from(i8s);
+ const i16Vec_ = IntVector.from(i16s);
+ const i32Vec_ = IntVector.from(i32s);
+ const i64Vec_ = IntVector.from(i64s, true);
+ const u8Vec_ = IntVector.from(u8s);
+ const u16Vec_ = IntVector.from(u16s);
+ const u32Vec_ = IntVector.from(u32s);
+ const u64Vec_ = IntVector.from(u64s, true);
+
+ // Convert from a Vector of the opposite sign
+ const i8Vec = Int8Vector.from(u8Vec_);
+ const i16Vec = Int16Vector.from(u16Vec_);
+ const i32Vec = Int32Vector.from(u32Vec_);
+ const i64Vec = Int64Vector.from(u64Vec_);
+ const u8Vec = Uint8Vector.from(i8Vec_);
+ const u16Vec = Uint16Vector.from(i16Vec_);
+ const u32Vec = Uint32Vector.from(i32Vec_);
+ const u64Vec = Uint64Vector.from(i64Vec_);
+
+ // test strong typing at compile-time
+ test(`return type is correct`, () => checkType(Int8Vector, i8Vec));
+ test(`return type is correct`, () => checkType(Int16Vector, i16Vec));
+ test(`return type is correct`, () => checkType(Int32Vector, i32Vec));
+ test(`return type is correct`, () => checkType(Int64Vector, i64Vec));
+ test(`return type is correct`, () => checkType(Uint8Vector, u8Vec));
+ test(`return type is correct`, () => checkType(Uint16Vector, u16Vec));
+ test(`return type is correct`, () => checkType(Uint32Vector, u32Vec));
+ test(`return type is correct`, () => checkType(Uint64Vector, u64Vec));
+
+ const bigI64s = BigInt64Array.from(toBigNumsArray(u64s));
+ const bigU64s = BigUint64Array.from(toBigNumsArray(i64s));
+
+ testAndValidateVector(i8Vec, Int8Array.from(u8s));
+ testAndValidateVector(i16Vec, Int16Array.from(u16s));
+ testAndValidateVector(i32Vec, Int32Array.from(u32s));
+ // This tests when values are represented as pairs of lo, hi
+ testAndValidateVector(i64Vec, new Int32Array(bigI64s.buffer));
+ // This tests when values are represented as native JS bigints
+ testAndValidateVector(i64Vec, new Int32Array(bigI64s.buffer), [...bigI64s]);
+ testAndValidateVector(u8Vec, Uint8Array.from(i8s));
+ testAndValidateVector(u16Vec, Uint16Array.from(i16s));
+ testAndValidateVector(u32Vec, Uint32Array.from(i32s));
+ // This tests when values are represented as pairs of lo, hi
+ testAndValidateVector(u64Vec, new Uint32Array(bigU64s.buffer));
+ // This tests when values are represented as native JS bigints
+ testAndValidateVector(u64Vec, new Uint32Array(bigU64s.buffer), [...bigU64s]);
});
describe(`Int8Vector`, () => {
@@ -180,7 +288,7 @@ describe(`IntVector`, () => {
testIntVector(Int64);
testIntVector(Int64, bigIntValuesArray(BigInt64Array));
describe(`Int64Vector.from accepts regular Arrays`, () => {
- const values = valuesArray(Int32Array);
+ const values = valuesArray64(Int32Array);
const vector = Int64Vector.from(values);
testAndValidateVector(vector, valuesTyped(Int32Array), values);
testAndValidateVector(vector, valuesTyped(Int32Array), bigIntValuesArray(BigInt64Array));
@@ -218,7 +326,7 @@ describe(`IntVector`, () => {
testIntVector(Uint64);
testIntVector(Uint64, bigIntValuesArray(BigUint64Array));
describe(`Uint64Vector.from accepts regular Arrays`, () => {
- const values = valuesArray(Uint32Array);
+ const values = valuesArray64(Uint32Array);
const vector = Uint64Vector.from(values);
testAndValidateVector(vector, valuesTyped(Uint32Array), values);
testAndValidateVector(vector, valuesTyped(Uint32Array), bigIntValuesArray(BigUint64Array));
@@ -337,8 +445,7 @@ function gets_expected_values<T extends Int | Float>(vector: Vector<T>, typed: T
}
} else {
const vector64 = vector as Vector<Int64 | Uint64>;
- const ArrayType = (vector as Vector<Int64 | Uint64>).ArrayType;
- const i64 = () => new ArrayType(values.slice(stride * i, stride * (i + 1)));
+ const i64 = (() => typed.subarray(stride * i, stride * (i + 1)));
while (++i < n) {
expect((vector64.get(i) as any).subarray(0, stride)).toEqual(i64());
}
@@ -367,8 +474,7 @@ function iterates_expected_values<T extends Int | Float>(vector: Vector<T>, type
}
} else {
const vector64 = vector as Vector<Int64 | Uint64>;
- const ArrayType = (vector as Vector<Int64 | Uint64>).ArrayType;
- const i64 = () => new ArrayType(values.slice(stride * i, stride * (i + 1)));
+ const i64 = (() => typed.subarray(stride * i, stride * (i + 1)));
for (let v of vector64) {
expect(++i).toBeLessThan(n);
expect((v as any).subarray(0, stride)).toEqual(i64());