You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/09/08 22:42:17 UTC
[2/7] arrow git commit: ARROW-1479: [JS] Expand JavaScript
implementation
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/src/reader/vector.ts
----------------------------------------------------------------------
diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts
new file mode 100644
index 0000000..a3cd798
--- /dev/null
+++ b/js/src/reader/vector.ts
@@ -0,0 +1,271 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { flatbuffers } from 'flatbuffers';
+import { MessageBatch } from './message';
+import { Vector } from '../vector/vector';
+import * as Schema_ from '../format/Schema_generated';
+import { StructVector } from '../vector/struct';
+import { IteratorState, Dictionaries } from './arrow';
+import { DictionaryVector } from '../vector/dictionary';
+import { Utf8Vector, ListVector, FixedSizeListVector } from '../vector/list';
+import {
+ TypedArray, TypedArrayCtor, IntArray, FloatArray,
+ Int8Vector, Int16Vector, Int32Vector, Int64Vector,
+ Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector,
+ Float32Vector, Float64Vector, IndexVector, DateVector,
+} from '../vector/typed';
+
+import Int = Schema_.org.apache.arrow.flatbuf.Int;
+import Type = Schema_.org.apache.arrow.flatbuf.Type;
+import Field = Schema_.org.apache.arrow.flatbuf.Field;
+import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
+import VectorType = Schema_.org.apache.arrow.flatbuf.VectorType;
+import VectorLayout = Schema_.org.apache.arrow.flatbuf.VectorLayout;
+import FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList;
+import FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint;
+import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding;
+
+export function readVector(field: Field, batch: MessageBatch, state: IteratorState, dictionaries: Dictionaries) {
+ return readDictionaryVector(field, batch, state, dictionaries) ||
+ readTypedVector(field, batch, state, dictionaries);
+}
+
+function readTypedVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) {
+ let typeType = field.typeType(), readTyped = typedVectorReaders[typeType];
+ if (!readTyped) {
+ throw new Error('Unrecognized vector name "' + Type[typeType] + '" type "' + typeType + '"');
+ }
+ return readTyped(field, batch, iterator, dictionaries);
+}
+
+function readDictionaryVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) {
+ let encoding: DictionaryEncoding;
+ if (dictionaries && (encoding = field.dictionary())) {
+ let id = encoding.id().toFloat64().toString();
+ let fieldType = encoding.indexType() ||
+ /* a dictionary index defaults to signed 32 bit int if unspecified */
+ { bitWidth: () => 32, isSigned: () => true };
+ let indexField = createSyntheticDictionaryIndexField(field, fieldType);
+ let index = readIntVector(indexField, batch, iterator, null, fieldType);
+ return DictionaryVector.create(field, index.length, index, dictionaries[id]);
+ }
+}
+
+const IntViews = [Int8Array, Int16Array, Int32Array, Int32Array ];
+const Int32Views = [Int32Array, Int32Array, Int32Array, Int32Array ];
+const UintViews = [Uint8Array, Uint16Array, Uint32Array, Uint32Array ];
+const Uint8Views = [Uint8Array, Uint8Array, Uint8Array, Uint8Array ];
+const Uint32Views = [Uint32Array, Uint32Array, Uint32Array, Uint32Array ];
+const FloatViews = [Int8Array, Int16Array, Float32Array, Float64Array];
+
+const createIntDataViews = createDataView.bind(null, IntViews, null);
+const createUintDataViews = createDataView.bind(null, UintViews, null);
+const createDateDataViews = createDataView.bind(null, Uint32Views, null);
+const createFloatDataViews = createDataView.bind(null, FloatViews, null);
+const createNestedDataViews = createDataView.bind(null, Uint32Views, null);
+const createValidityDataViews = createDataView.bind(null, Uint8Views, null);
+const createUtf8DataViews = createDataView.bind(null, Uint8Views, Int32Views);
+
+const floatVectors = {
+ [Precision.SINGLE]: Float32Vector,
+ [Precision.DOUBLE]: Float64Vector
+};
+const intVectors = [
+ [/* unsigned */ Uint8Vector, /* signed */ Int8Vector ],
+ [/* unsigned */ Uint16Vector, /* signed */ Int16Vector],
+ [/* unsigned */ Uint32Vector, /* signed */ Int32Vector],
+ [/* unsigned */ Uint64Vector, /* signed */ Int64Vector]
+];
+
+function readIntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, fieldType?: FieldType) {
+ let type = (fieldType || field.type(new Int()));
+ return type.isSigned() ?
+ read_IntVector(field, batch, iterator, dictionaries, type) :
+ readUintVector(field, batch, iterator, dictionaries, type);
+}
+
+const read_IntVector = readVectorLayout<number, IntArray>(createIntDataViews, createIntVector);
+const readUintVector = readVectorLayout<number, IntArray>(createUintDataViews, createIntVector);
+function createIntVector(field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) {
+ let type = fieldType || field.type(new Int()), bitWidth = type.bitWidth();
+ let Vector = valueForBitWidth(bitWidth, intVectors)[+type.isSigned()];
+ return Vector.create(field, length, validity, data || offsets);
+ // ---------------------- so this is kinda strange 👆:
+ // The dictionary encoded vectors I generated from sample mapd-core queries have the indicies' data buffers
+ // tagged as VectorType.OFFSET (0) in the field metadata. The current TS impl ignores buffers' layout type,
+ // and assumes the second buffer is the data for a NullableIntVector. Since we've been stricter about enforcing
+ // the Arrow spec while parsing, the IntVector's data buffer reads empty in this case. If so, fallback to using
+ // the offsets buffer as the data, since IntVectors don't have offsets.
+}
+
+const readFloatVector = readVectorLayout<number, FloatArray>(
+ createFloatDataViews,
+ (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+ let type = field.type(new FloatingPoint());
+ let Vector = floatVectors[type.precision()];
+ return Vector.create(field, length, validity, data);
+ }
+);
+
+const readDateVector = readVectorLayout<Date, Uint32Array>(
+ createDateDataViews,
+ (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+ return DateVector.create(field, length, validity, data);
+ }
+);
+
+const readUtf8Vector = readVectorLayout<string, Uint8Array>(
+ createUtf8DataViews,
+ (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+ let offsetsAdjusted = new Int32Array(offsets.buffer, offsets.byteOffset, length + 1);
+ return Utf8Vector.create(
+ field, length, validity,
+ Uint8Vector.create(field, data.length, null, data),
+ IndexVector.create(field, length + 1, null, offsetsAdjusted)
+ );
+ }
+);
+
+const readListVector = readVectorLayout<any[], Uint32Array>(
+ createNestedDataViews,
+ (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+ let offsetsAdjusted = new Int32Array(offsets.buffer, offsets.byteOffset, length + 1);
+ return ListVector.create(
+ field, length, validity,
+ readVector(field.children(0), batch, iterator, dictionaries),
+ IndexVector.create(field, length + 1, null, offsetsAdjusted)
+ );
+ }
+);
+
+const readFixedSizeListVector = readVectorLayout<any[], Uint32Array>(
+ createNestedDataViews,
+ (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+ let size = field.type(new FixedSizeList()).listSize();
+ return FixedSizeListVector.create(
+ field, length, size, validity,
+ readVector(field.children(0), batch, iterator, dictionaries)
+ );
+ }
+);
+
+const readStructVector = readVectorLayout<any[], ArrayLike<any>>(
+ createNestedDataViews,
+ (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => {
+ let vectors: Vector<any>[] = [];
+ for (let i = -1, n = field.childrenLength(); ++i < n;) {
+ vectors[i] = readVector(field.children(i), batch, iterator, dictionaries);
+ }
+ return StructVector.create(field, length, validity, ...vectors);
+ }
+);
+
+const typedVectorReaders = {
+ [Type.Int]: readIntVector,
+ [Type.Date]: readDateVector,
+ [Type.List]: readListVector,
+ [Type.Utf8]: readUtf8Vector,
+ [Type.Struct_]: readStructVector,
+ [Type.FloatingPoint]: readFloatVector,
+ [Type.FixedSizeList]: readFixedSizeListVector,
+};
+
+type FieldType = { bitWidth(): number; isSigned(): boolean };
+type dataViewFactory<V = TypedArray> = (batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number) => V;
+type vectorFactory<TList, V = Vector<any>> = (field: Field,
+ length: number,
+ data: TList,
+ nulls: Uint8Array,
+ offsets: TypedArray,
+ fieldType: FieldType,
+ chunk: MessageBatch,
+ iterable: IteratorState,
+ dictionaries: Dictionaries) => V;
+
+function readVectorLayout<T, TList>(createDataView: dataViewFactory<TList>, createVector: vectorFactory<TList, Vector<T>>) {
+ return function readLayout(
+ field: Field,
+ chunk: MessageBatch,
+ iterator: IteratorState,
+ dictionaries: Dictionaries,
+ integerFieldType?: FieldType
+ ) {
+ let batch = chunk.data;
+ let layoutLength = field.layoutLength();
+ let node = batch.nodes(iterator.nodeIndex++);
+ let data: TList, offsets: any, validity: Uint8Array;
+ let type, bitWidth, bufferLength, nodeLength = node.length().low;
+ for (let i = -1; ++i < layoutLength;) {
+ let layout = field.layout(i);
+ let buffer = batch.buffers(iterator.bufferIndex++);
+ if ((type = layout.type()) === VectorType.TYPE ||
+ (bufferLength = buffer.length().low) <= 0 ||
+ (bitWidth = layout.bitWidth()) <= 0) {
+ continue;
+ } else if (type === VectorType.DATA) {
+ data = createDataView(chunk, type, bitWidth, buffer.offset().low, bufferLength);
+ } else if (type === VectorType.OFFSET) {
+ offsets = createDataView(chunk, type, bitWidth, buffer.offset().low, bufferLength);
+ } else if (node.nullCount().low > 0) {
+ validity = createValidityDataViews(chunk, type, bitWidth, buffer.offset().low, nodeLength);
+ }
+ }
+ return createVector(field, nodeLength, data, validity, offsets, integerFieldType, chunk, iterator, dictionaries);
+ };
+}
+
+function createDataView(
+ dataViews: TypedArrayCtor<any>[], offsetViews: TypedArrayCtor<any>[] | null,
+ batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number
+) {
+ const buffer = batch.bytes.buffer;
+ const byteLength = buffer.byteLength;
+ const byteOffset = batch.offset + offset;
+ const DataViewType = valueForBitWidth(bitWidth, type === VectorType.OFFSET && offsetViews || dataViews);
+ const dataViewLength = ((byteOffset + length) <= byteLength
+ ? length
+ : byteLength - byteOffset
+ ) / DataViewType['BYTES_PER_ELEMENT'];
+ return new DataViewType(buffer, byteOffset, dataViewLength);
+}
+
+function valueForBitWidth(bitWidth: number, values: any[]) {
+ return values[bitWidth >> 4] || values[3];
+}
+
+function createSyntheticDictionaryIndexField(field: Field, type: FieldType) {
+ let layouts = [];
+ let builder = new flatbuffers.Builder();
+ if (field.nullable()) {
+ VectorLayout.startVectorLayout(builder);
+ VectorLayout.addBitWidth(builder, 8);
+ VectorLayout.addType(builder, VectorType.VALIDITY);
+ builder.finish(VectorLayout.endVectorLayout(builder));
+ layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer()));
+ builder = new flatbuffers.Builder();
+ }
+ VectorLayout.startVectorLayout(builder);
+ VectorLayout.addBitWidth(builder, type.bitWidth());
+ VectorLayout.addType(builder, VectorType.DATA);
+ builder.finish(VectorLayout.endVectorLayout(builder));
+ layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer()));
+ return Object.create(field, {
+ layout: { value(i) { return layouts[i]; } },
+ layoutLength: { value() { return layouts.length; } }
+ });
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/src/table.ts
----------------------------------------------------------------------
diff --git a/js/src/table.ts b/js/src/table.ts
new file mode 100644
index 0000000..999bb24
--- /dev/null
+++ b/js/src/table.ts
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { readBuffers } from './reader/arrow';
+import { StructVector } from './vector/struct';
+import { Vector, sliceToRangeArgs } from './vector/vector';
+
+export class Table implements Iterable<Map<string, any>> {
+ public length: number;
+ protected _columns: Vector<any>[];
+ protected _columnsMap: { [k: string]: Vector<any> };
+ static from(...bytes: Array<Uint8Array | Buffer | string>) {
+ let columns: Vector<any>[];
+ for (let vectors of readBuffers(...bytes)) {
+ columns = !columns ? vectors : columns.map((v, i) => v.concat(vectors[i]));
+ }
+ return new Table(columns);
+ }
+ static fromStruct(vector: StructVector) {
+ return new Table((<any> vector).vectors);
+ }
+ constructor(columns: Vector<any>[]) {
+ this._columns = columns || [];
+ this.length = Math.max(...this._columns.map((v) => v.length));
+ this._columnsMap = this._columns.reduce((map, vec) => {
+ return (map[vec.name] = vec) && map || map;
+ }, <any> {});
+ }
+ *[Symbol.iterator]() {
+ for (let cols = this._columns, i = -1, n = this.length; ++i < n;) {
+ yield rowAsMap(i, cols);
+ }
+ }
+ *rows(startRow?: number | boolean, endRow?: number | boolean, compact?: boolean) {
+ let start = startRow as number, end = endRow as number;
+ if (typeof startRow === 'boolean') {
+ compact = startRow;
+ start = end;
+ end = undefined;
+ } else if (typeof endRow === 'boolean') {
+ compact = endRow;
+ end = undefined;
+ }
+ let rowIndex = -1, { length } = this;
+ const [rowOffset, rowsTotal] = sliceToRangeArgs(length, start, end);
+ while (++rowIndex < rowsTotal) {
+ yield this.getRow((rowIndex + rowOffset) % length, compact);
+ }
+ }
+ *cols(startCol?: number, endCol?: number) {
+ for (const column of this._columns.slice(startCol, endCol)) {
+ yield column;
+ }
+ }
+ getRow(rowIndex: number, compact?: boolean) {
+ return (compact && rowAsArray || rowAsObject)(rowIndex, this._columns);
+ }
+ getCell(columnName: string, rowIndex: number) {
+ return this.getColumn(columnName).get(rowIndex);
+ }
+ getCellAt(columnIndex: number, rowIndex: number) {
+ return this.getColumnAt(columnIndex).get(rowIndex);
+ }
+ getColumn<T = any>(columnName: string) {
+ return this._columnsMap[columnName] as Vector<T>;
+ }
+ getColumnAt<T = any>(columnIndex: number) {
+ return this._columns[columnIndex] as Vector<T>;
+ }
+ toString({ index = false } = {}) {
+ const { length } = this;
+ if (length <= 0) { return ''; }
+ const maxColumnWidths = [];
+ const rows = new Array(length + 1);
+ rows[0] = this._columns.map((c) => c.name);
+ index && rows[0].unshift('Index');
+ for (let i = -1, n = rows.length - 1; ++i < n;) {
+ rows[i + 1] = this.getRow(i, true);
+ index && rows[i + 1].unshift(i);
+ }
+ // Pass one to convert to strings and count max column widths
+ for (let i = -1, n = rows.length; ++i < n;) {
+ const row = rows[i];
+ for (let j = -1, k = row.length; ++j < k;) {
+ const val = row[j] = `${row[j]}`;
+ maxColumnWidths[j] = !maxColumnWidths[j]
+ ? val.length
+ : Math.max(maxColumnWidths[j], val.length);
+ }
+ }
+ // Pass two to pad each one to max column width
+ for (let i = -1, n = rows.length; ++i < n;) {
+ const row = rows[i];
+ for (let j = -1, k = row.length; ++j < k;) {
+ row[j] = leftPad(row[j], ' ', maxColumnWidths[j]);
+ }
+ rows[i] = row.join(', ');
+ }
+ return rows.join('\n');
+ }
+}
+
+Table.prototype.length = 0;
+
+function leftPad(str, fill, n) {
+ return (new Array(n + 1).join(fill) + str).slice(-1 * n);
+}
+
+function rowAsMap(row: number, columns: Vector<any>[]) {
+ return columns.reduce((map, vector) => map.set(vector.name, vector.get(row)), new Map());
+}
+
+function rowAsObject(rowIndex: number, columns: Vector<any>[]) {
+ return columns.reduce((row, vector) => (row[vector.name] = vector.get(rowIndex)) && row || row, Object.create(null));
+}
+
+function rowAsArray(rowIndex: number, columns: Vector<any>[]) {
+ return columns.reduce((row, vector, columnIndex) => (row[columnIndex] = vector.get(rowIndex)) && row || row, new Array(columns.length));
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/src/types.ts
----------------------------------------------------------------------
diff --git a/js/src/types.ts b/js/src/types.ts
deleted file mode 100644
index c541098..0000000
--- a/js/src/types.ts
+++ /dev/null
@@ -1,597 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { org } from "./Arrow_generated";
-import { BitArray } from "./bitarray";
-
-import { TextDecoder } from "text-encoding";
-
-const Type = org.apache.arrow.flatbuf.Type;
-
-interface IArrayView {
- slice(start: number, end: number): IArrayView;
- toString(): string;
-}
-
-interface IViewConstructor<T extends IArrayView> {
- BYTES_PER_ELEMENT: number;
- new(buffer: any, offset: number, length: number): T;
-}
-
-export abstract class Vector {
- /**
- * Helper function for loading a VALIDITY buffer (for Nullable types)
- * bb: flatbuffers.ByteBuffer
- * buffer: org.apache.arrow.flatbuf.Buffer
- */
- public static loadValidityBuffer(bb, buffer): BitArray {
- const arrayBuffer = bb.bytes_.buffer;
- const offset = bb.bytes_.byteOffset + buffer.offset;
- return new BitArray(arrayBuffer, offset, buffer.length * 8);
- }
-
- /**
- * Helper function for loading an OFFSET buffer
- * buffer: org.apache.arrow.flatbuf.Buffer
- */
- public static loadOffsetBuffer(bb, buffer): Int32Array {
- const arrayBuffer = bb.bytes_.buffer;
- const offset = bb.bytes_.byteOffset + buffer.offset;
- const length = buffer.length / Int32Array.BYTES_PER_ELEMENT;
- return new Int32Array(arrayBuffer, offset, length);
- }
-
- public field: any;
- public name: string;
- public length: number;
- public nullCount: number;
-
- constructor(field) {
- this.field = field;
- this.name = field.name();
- }
-
- /* Access datum at index i */
- public abstract get(i);
- /* Return array representing data in the range [start, end) */
- public abstract slice(start: number, end: number);
- /* Return array of child vectors, for container types */
- public getChildVectors() {
- return [];
- }
-
- /**
- * Use recordBatch fieldNodes and Buffers to construct this Vector
- * bb: flatbuffers.ByteBuffer
- * node: org.apache.arrow.flatbuf.FieldNode
- * buffers: { offset: number, length: number }[]
- */
- public loadData(bb, node, buffers) {
- this.length = node.length().low;
- this.nullCount = node.nullCount().low;
- this.loadBuffers(bb, node, buffers);
- }
-
- protected abstract loadBuffers(bb, node, buffers);
-}
-
-class SimpleVector<T extends IArrayView> extends Vector {
- protected dataView: T;
- private TypedArray: IViewConstructor<T>;
-
- constructor(field, TypedArray: IViewConstructor<T>) {
- super(field);
- this.TypedArray = TypedArray;
- }
-
- public get(i) {
- return this.dataView[i];
- }
-
- public getDataView() {
- return this.dataView;
- }
-
- public toString() {
- return this.dataView.toString();
- }
-
- public slice(start, end) {
- return this.dataView.slice(start, end);
- }
-
- protected loadBuffers(bb, node, buffers) {
- this.loadDataBuffer(bb, buffers[0]);
- }
-
- /*
- * buffer: org.apache.arrow.flatbuf.Buffer
- */
- protected loadDataBuffer(bb, buffer) {
- const arrayBuffer = bb.bytes_.buffer;
- const offset = bb.bytes_.byteOffset + buffer.offset;
- const length = buffer.length / this.TypedArray.BYTES_PER_ELEMENT;
- this.dataView = new this.TypedArray(arrayBuffer, offset, length);
- }
-
-}
-
-class NullableSimpleVector<T extends IArrayView> extends SimpleVector<T> {
- protected validityView: BitArray;
-
- public get(i: number) {
- if (this.validityView.get(i)) {
- return this.dataView[i];
- } else {
- return null;
- }
- }
-
- public getValidityVector() {
- return this.validityView;
- }
-
- protected loadBuffers(bb, node, buffers) {
- this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
- this.loadDataBuffer(bb, buffers[1]);
- }
-}
-
-/* tslint:disable max-line-length */
-class Uint8Vector extends SimpleVector<Uint8Array> { constructor(field) { super(field, Uint8Array); } }
-class Uint16Vector extends SimpleVector<Uint16Array> { constructor(field) { super(field, Uint16Array); } }
-class Uint32Vector extends SimpleVector<Uint32Array> { constructor(field) { super(field, Uint32Array); } }
-class Int8Vector extends SimpleVector<Uint8Array> { constructor(field) { super(field, Uint8Array); } }
-class Int16Vector extends SimpleVector<Uint16Array> { constructor(field) { super(field, Uint16Array); } }
-class Int32Vector extends SimpleVector<Uint32Array> { constructor(field) { super(field, Uint32Array); } }
-class Float32Vector extends SimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); } }
-class Float64Vector extends SimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); } }
-
-class NullableUint8Vector extends NullableSimpleVector<Uint8Array> { constructor(field) { super(field, Uint8Array); } }
-class NullableUint16Vector extends NullableSimpleVector<Uint16Array> { constructor(field) { super(field, Uint16Array); } }
-class NullableUint32Vector extends NullableSimpleVector<Uint32Array> { constructor(field) { super(field, Uint32Array); } }
-class NullableInt8Vector extends NullableSimpleVector<Uint8Array> { constructor(field) { super(field, Uint8Array); } }
-class NullableInt16Vector extends NullableSimpleVector<Uint16Array> { constructor(field) { super(field, Uint16Array); } }
-class NullableInt32Vector extends NullableSimpleVector<Uint32Array> { constructor(field) { super(field, Uint32Array); } }
-class NullableFloat32Vector extends NullableSimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); } }
-class NullableFloat64Vector extends NullableSimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); } }
-/* tslint:enable max-line-length */
-
-class Uint64Vector extends SimpleVector<Uint32Array> {
- constructor(field) {
- super(field, Uint32Array);
- }
-
- public get(i: number) {
- return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
- }
-}
-
-class NullableUint64Vector extends NullableSimpleVector<Uint32Array> {
- constructor(field) {
- super(field, Uint32Array);
- }
-
- public get(i: number) {
- if (this.validityView.get(i)) {
- return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
- } else {
- return null;
- }
- }
-}
-
-class Int64Vector extends NullableSimpleVector<Uint32Array> {
- constructor(field) {
- super(field, Uint32Array);
- }
-
- public get(i: number) {
- return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
- }
-}
-
-class NullableInt64Vector extends NullableSimpleVector<Uint32Array> {
- constructor(field) {
- super(field, Uint32Array);
- }
-
- public get(i: number) {
- if (this.validityView.get(i)) {
- return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] };
- } else {
- return null;
- }
- }
-}
-
-class DateVector extends SimpleVector<Uint32Array> {
- constructor(field) {
- super(field, Uint32Array);
- }
-
- public get(i) {
- return new Date(super.get(2 * i + 1) * Math.pow(2, 32) + super.get(2 * i));
- }
-}
-
-class NullableDateVector extends DateVector {
- private validityView: BitArray;
-
- public get(i) {
- if (this.validityView.get(i)) {
- return super.get(i);
- } else {
- return null;
- }
- }
-
- public getValidityVector() {
- return this.validityView;
- }
-
- protected loadBuffers(bb, node, buffers) {
- this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
- this.loadDataBuffer(bb, buffers[1]);
- }
-}
-
-class Utf8Vector extends SimpleVector<Uint8Array> {
- private static decoder: TextDecoder = new TextDecoder("utf8");
-
- protected offsetView: Int32Array;
-
- constructor(field) {
- super(field, Uint8Array);
- }
-
- public get(i) {
- return Utf8Vector.decoder.decode(this.dataView.slice(this.offsetView[i], this.offsetView[i + 1]));
- }
-
- public slice(start: number, end: number) {
- const result: string[] = [];
- for (let i: number = start; i < end; i++) {
- result.push(this.get(i));
- }
- return result;
- }
-
- public getOffsetView() {
- return this.offsetView;
- }
-
- protected loadBuffers(bb, node, buffers) {
- this.offsetView = Vector.loadOffsetBuffer(bb, buffers[0]);
- this.loadDataBuffer(bb, buffers[1]);
- }
-}
-
-class NullableUtf8Vector extends Utf8Vector {
- private validityView: BitArray;
-
- public get(i) {
- if (this.validityView.get(i)) {
- return super.get(i);
- } else {
- return null;
- }
- }
-
- public getValidityVector() {
- return this.validityView;
- }
-
- protected loadBuffers(bb, node, buffers) {
- this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
- this.offsetView = Vector.loadOffsetBuffer(bb, buffers[1]);
- this.loadDataBuffer(bb, buffers[2]);
- }
-}
-
-// Nested Types
-class ListVector extends Uint32Vector {
- private dataVector: Vector;
-
- constructor(field, dataVector: Vector) {
- super(field);
- this.dataVector = dataVector;
- }
-
- public getChildVectors() {
- return [this.dataVector];
- }
-
- public get(i) {
- const offset = super.get(i);
- if (offset === null) {
- return null;
- }
- const nextOffset = super.get(i + 1);
- return this.dataVector.slice(offset, nextOffset);
- }
-
- public toString() {
- return "length: " + (this.length);
- }
-
- public slice(start: number, end: number) {
- const result = [];
- for (let i = start; i < end; i++) {
- result.push(this.get(i));
- }
- return result;
- }
-
- protected loadBuffers(bb, node, buffers) {
- super.loadBuffers(bb, node, buffers);
- this.length -= 1;
- }
-}
-
-class NullableListVector extends ListVector {
- private validityView: BitArray;
-
- public get(i) {
- if (this.validityView.get(i)) {
- return super.get(i);
- } else {
- return null;
- }
- }
-
- public getValidityVector() {
- return this.validityView;
- }
-
- protected loadBuffers(bb, node, buffers) {
- this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
- this.loadDataBuffer(bb, buffers[1]);
- this.length -= 1;
- }
-}
-
-class FixedSizeListVector extends Vector {
- public size: number;
- private dataVector: Vector;
-
- constructor(field, size: number, dataVector: Vector) {
- super(field);
- this.size = size;
- this.dataVector = dataVector;
- }
-
- public getChildVectors() {
- return [this.dataVector];
- }
-
- public get(i: number) {
- return this.dataVector.slice(i * this.size, (i + 1) * this.size);
- }
-
- public slice(start: number, end: number) {
- const result = [];
- for (let i = start; i < end; i++) {
- result.push(this.get(i));
- }
- return result;
- }
-
- public getListSize() {
- return this.size;
- }
-
- protected loadBuffers(bb, node, buffers) {
- // no buffers to load
- }
-}
-
-class NullableFixedSizeListVector extends FixedSizeListVector {
- private validityView: BitArray;
-
- public get(i: number) {
- if (this.validityView.get(i)) {
- return super.get(i);
- } else {
- return null;
- }
- }
-
- public getValidityVector() {
- return this.validityView;
- }
-
- protected loadBuffers(bb, node, buffers) {
- this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
- }
-}
-
-class StructVector extends Vector {
- private validityView: BitArray;
- private vectors: Vector[];
-
- constructor(field, vectors: Vector[]) {
- super(field);
- this.vectors = vectors;
- }
-
- public getChildVectors() {
- return this.vectors;
- }
-
- public get(i: number) {
- if (this.validityView.get(i)) {
- return this.vectors.map((v: Vector) => v.get(i));
- } else {
- return null;
- }
- }
-
- public slice(start: number, end: number) {
- const result = [];
- for (let i = start; i < end; i++) {
- result.push(this.get(i));
- }
- return result;
- }
-
- public getValidityVector() {
- return this.validityView;
- }
-
- protected loadBuffers(bb, node, buffers) {
- this.validityView = Vector.loadValidityBuffer(bb, buffers[0]);
- }
-}
-
-class DictionaryVector extends Vector {
- private indices: Vector;
- private dictionary: Vector;
-
- constructor(field, indices: Vector, dictionary: Vector) {
- super(field);
- this.indices = indices;
- this.dictionary = dictionary;
- }
-
- public get(i) {
- const encoded = this.indices.get(i);
- if (encoded == null) {
- return null;
- } else {
- return this.dictionary.get(encoded);
- }
- }
-
- /** Get the dictionary encoded value */
- public getEncoded(i) {
- return this.indices.get(i);
- }
-
- public slice(start, end) {
- return this.indices.slice(start, end); // TODO decode
- }
-
- public getChildVectors() {
- return this.indices.getChildVectors();
- }
-
- /** Get the index (encoded) vector */
- public getIndexVector() {
- return this.indices;
- }
-
- /** Get the dictionary vector */
- public getDictionaryVector() {
- return this.dictionary;
- }
-
- public toString() {
- return this.indices.toString();
- }
-
- protected loadBuffers(bb, node, buffers) {
- this.indices.loadData(bb, node, buffers);
- }
-}
-
-export function vectorFromField(field, dictionaries): Vector {
- const dictionary = field.dictionary();
- const nullable = field.nullable();
- if (dictionary == null) {
- const typeType = field.typeType();
- if (typeType === Type.List) {
- const dataVector = vectorFromField(field.children(0), dictionaries);
- return nullable ? new NullableListVector(field, dataVector) : new ListVector(field, dataVector);
- } else if (typeType === Type.FixedSizeList) {
- const dataVector = vectorFromField(field.children(0), dictionaries);
- const size = field.type(new org.apache.arrow.flatbuf.FixedSizeList()).listSize();
- if (nullable) {
- return new NullableFixedSizeListVector(field, size, dataVector);
- } else {
- return new FixedSizeListVector(field, size, dataVector);
- }
- } else if (typeType === Type.Struct_) {
- const vectors: Vector[] = [];
- for (let i: number = 0; i < field.childrenLength(); i++) {
- vectors.push(vectorFromField(field.children(i), dictionaries));
- }
- return new StructVector(field, vectors);
- } else {
- if (typeType === Type.Int) {
- const type = field.type(new org.apache.arrow.flatbuf.Int());
- return _createIntVector(field, type.bitWidth(), type.isSigned(), nullable);
- } else if (typeType === Type.FloatingPoint) {
- const precision = field.type(new org.apache.arrow.flatbuf.FloatingPoint()).precision();
- if (precision === org.apache.arrow.flatbuf.Precision.SINGLE) {
- return nullable ? new NullableFloat32Vector(field) : new Float32Vector(field);
- } else if (precision === org.apache.arrow.flatbuf.Precision.DOUBLE) {
- return nullable ? new NullableFloat64Vector(field) : new Float64Vector(field);
- } else {
- throw new Error("Unimplemented FloatingPoint precision " + precision);
- }
- } else if (typeType === Type.Utf8) {
- return nullable ? new NullableUtf8Vector(field) : new Utf8Vector(field);
- } else if (typeType === Type.Date) {
- return nullable ? new NullableDateVector(field) : new DateVector(field);
- } else {
- throw new Error("Unimplemented type " + typeType);
- }
- }
- } else {
- // determine arrow type - default is signed 32 bit int
- const type = dictionary.indexType();
- let bitWidth = 32;
- let signed = true;
- if (type != null) {
- bitWidth = type.bitWidth();
- signed = type.isSigned();
- }
- const indices = _createIntVector(field, bitWidth, signed, nullable);
- return new DictionaryVector(field, indices, dictionaries[dictionary.id().toFloat64().toString()]);
- }
-}
-
-function _createIntVector(field, bitWidth, signed, nullable) {
- if (bitWidth === 64) {
- if (signed) {
- return nullable ? new NullableInt64Vector(field) : new Int64Vector(field);
- } else {
- return nullable ? new NullableUint64Vector(field) : new Uint64Vector(field);
- }
- } else if (bitWidth === 32) {
- if (signed) {
- return nullable ? new NullableInt32Vector(field) : new Int32Vector(field);
- } else {
- return nullable ? new NullableUint32Vector(field) : new Uint32Vector(field);
- }
- } else if (bitWidth === 16) {
- if (signed) {
- return nullable ? new NullableInt16Vector(field) : new Int16Vector(field);
- } else {
- return nullable ? new NullableUint16Vector(field) : new Uint16Vector(field);
- }
- } else if (bitWidth === 8) {
- if (signed) {
- return nullable ? new NullableInt8Vector(field) : new Int8Vector(field);
- } else {
- return nullable ? new NullableUint8Vector(field) : new Uint8Vector(field);
- }
- } else {
- throw new Error("Unimplemented Int bit width " + bitWidth);
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/src/vector/dictionary.ts
----------------------------------------------------------------------
diff --git a/js/src/vector/dictionary.ts b/js/src/vector/dictionary.ts
new file mode 100644
index 0000000..de811ea
--- /dev/null
+++ b/js/src/vector/dictionary.ts
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+
+export class DictionaryVector<T> extends Vector<T> {
+ protected data: Vector<T>;
+ protected keys: Vector<number>;
+ constructor(index: Vector<number>, dictionary: Vector<T>) {
+ super();
+ this.keys = index;
+ this.data = dictionary;
+ this.length = index && index.length || 0;
+ }
+ index(index: number) {
+ return this.keys.get(index);
+ }
+ value(index: number) {
+ return this.data.get(index);
+ }
+ get(index: number) {
+ return this.value(this.index(index));
+ }
+ concat(vector: DictionaryVector<T>) {
+ return DictionaryVector.from(this,
+ this.length + vector.length,
+ this.keys.concat(vector.keys),
+ this.data
+ );
+ }
+ *[Symbol.iterator]() {
+ let { data } = this;
+ for (const loc of this.keys) {
+ yield data.get(loc);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/src/vector/list.ts
----------------------------------------------------------------------
diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts
new file mode 100644
index 0000000..7360d96
--- /dev/null
+++ b/js/src/vector/list.ts
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+import { TextDecoder } from 'text-encoding';
+import { IndexVector, BitVector, ValidityArgs } from './typed';
+
+export class ListVectorBase<T> extends Vector<T> {
+ protected values: Vector<T>;
+ protected offsets: IndexVector;
+ constructor(validity: ValidityArgs, values: Vector<any>, offsets: IndexVector) {
+ super();
+ this.values = values;
+ this.offsets = offsets;
+ validity && (this.validity = BitVector.from(validity));
+ }
+ get(index: number) {
+ let batch, from, to, { offsets } = this;
+ if (!this.validity.get(index) ||
+ /* return null if `to` is null */
+ ((to = offsets.get(index + 1)) === null) || !(
+ /*
+ return null if `batch` is less than than 0. this check is placed
+ second to avoid creating the [from, batch] tuple if `to` is null
+ */
+ ([from, batch] = offsets.get(index, true) as number[]) && batch > -1)) {
+ return null;
+ }
+ return this.values.slice(from, to, batch) as any;
+ }
+ concat(vector: ListVectorBase<T>) {
+ return (this.constructor as typeof ListVectorBase).from(this,
+ this.length + vector.length,
+ this.validity.concat(vector.validity),
+ this.values.concat(vector.values),
+ this.offsets.concat(vector.offsets)
+ );
+ }
+ *[Symbol.iterator]() {
+ let v, r1, r2, { values } = this;
+ let it = this.offsets[Symbol.iterator]();
+ let iv = this.validity[Symbol.iterator]();
+ while (!(v = iv.next()).done && !(r1 = it.next()).done && !(r2 = it.next()).done) {
+ yield !v.value ? null : values.slice(r1.value[0], r2.value, r1.value[1]) as any;
+ }
+ }
+}
+
+export class ListVector<T> extends ListVectorBase<T[]> {}
+export class Utf8Vector extends ListVectorBase<string> {
+ protected static decoder = new TextDecoder(`utf-8`);
+ get(index: number) {
+ let chars = super.get(index) as any;
+ return chars ? Utf8Vector.decoder.decode(chars) : null;
+ }
+ *[Symbol.iterator]() {
+ let decoder = Utf8Vector.decoder;
+ for (const chars of super[Symbol.iterator]()) {
+ yield !chars ? null : decoder.decode(chars);
+ }
+ }
+}
+
+export class FixedSizeListVector<T> extends Vector<T[]> {
+ protected size: number;
+ protected values: Vector<T>;
+ constructor(size: number, validity: ValidityArgs, values: Vector<T>) {
+ super();
+ this.values = values;
+ this.size = Math.abs(size | 0) || 1;
+ validity && (this.validity = BitVector.from(validity));
+ }
+ get(index: number) {
+ return !this.validity.get(index) ? null : this.values.slice(
+ this.size * index, this.size * (index + 1)
+ ) as T[];
+ }
+ concat(vector: FixedSizeListVector<T>) {
+ return FixedSizeListVector.from(this,
+ this.length + vector.length,
+ this.size,
+ this.validity.concat(vector.validity),
+ this.values.concat(vector.values)
+ );
+ }
+ *[Symbol.iterator]() {
+ let v, i = -1;
+ let { size, length, values } = this;
+ let iv = this.validity[Symbol.iterator]();
+ while (!(v = iv.next()).done && ++i < length) {
+ yield !v.value ? null : values.slice(size * i, size * (i + 1)) as T[];
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/src/vector/struct.ts
----------------------------------------------------------------------
diff --git a/js/src/vector/struct.ts b/js/src/vector/struct.ts
new file mode 100644
index 0000000..e59ac91
--- /dev/null
+++ b/js/src/vector/struct.ts
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+import { BitVector, ValidityArgs } from './typed';
+
+export class StructVector extends Vector<any[]> {
+ protected vectors: Vector<any>[];
+ constructor(validity: ValidityArgs, ...vectors: Vector<any>[]) {
+ super();
+ this.vectors = vectors;
+ this.length = Math.max(0, ...vectors.map((v) => v.length));
+ validity && (this.validity = BitVector.from(validity));
+ }
+ get(index: number) {
+ return this.validity.get(index) ? this.vectors.map((v) => v.get(index)) : null;
+ }
+ concat(vector: StructVector) {
+ return StructVector.from(this,
+ this.length + vector.length,
+ this.validity.concat(vector.validity),
+ ...this.vectors.map((v, i) => v.concat(vector.vectors[i]))
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/src/vector/typed.ts
----------------------------------------------------------------------
diff --git a/js/src/vector/typed.ts b/js/src/vector/typed.ts
new file mode 100644
index 0000000..b38812e
--- /dev/null
+++ b/js/src/vector/typed.ts
@@ -0,0 +1,326 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { Vector } from './vector';
+import { flatbuffers } from 'flatbuffers';
+
+import Long = flatbuffers.Long;
+
+export type VArray<T = any> = {
+ [k: number]: T; length: number;
+ constructor: VArrayCtor<VArray<T>>;
+};
+
+export type VArrayCtor<VArray> = {
+ readonly prototype: VArray;
+ BYTES_PER_ELEMENT?: number;
+ new(...args: any[]): VArray;
+};
+
+export class VirtualVector<T, TArrayType = VArray<T>> extends Vector<T> {
+ protected lists: TArrayType[];
+ protected _arrayType: VArrayCtor<TArrayType>;
+ public get arrayType() { return this._arrayType; }
+ constructor(...lists: TArrayType[]) {
+ super();
+ this.lists = lists.filter(Boolean);
+ }
+ get(index: number): T {
+ /* inlined `findVirtual` impl */
+ let rows, length, lists = this.lists;
+ for (let batch = -1;
+ (rows = lists[++batch]) &&
+ (length = rows.length) <= index &&
+ 0 <= (index -= length);) {}
+ return rows && -1 < index ? rows[index] : null;
+ }
+ protected range(from: number, total: number, batch?: number) {
+ /* inlined `findVirtual` impl */
+ let rows, local = from, length;
+ let { lists, _arrayType } = this;
+ for (batch = (batch || 0) - 1;
+ (rows = lists[++batch]) &&
+ (length = rows.length) <= local &&
+ 0 <= (local -= length);) {}
+ if (rows && local > -1) {
+ let index = 0, listsLength = lists.length;
+ let set: any = Array.isArray(rows) ? arraySet : typedArraySet;
+ let slice = _arrayType['prototype']['subarray'] || _arrayType['prototype']['slice'];
+ let source = slice.call(rows, local, local + total), target = source;
+ // Perf optimization: if the first slice contains all the values we're looking for,
+ // we don't have to copy values to a target Array. If we're slicing a TypedArray,
+ // this is a significant improvement as we avoid the memcpy 🎉
+ if (source.length < total) {
+ target = new _arrayType(total);
+ while ((index = set(source, target, index)) < total) {
+ rows = lists[batch = ((batch + 1) % listsLength)];
+ source = slice.call(rows, 0, Math.min(rows.length, total - index));
+ }
+ }
+ return target as any;
+ }
+ return new _arrayType(0);
+ }
+ *[Symbol.iterator]() {
+ let index = -1, { lists, length } = this;
+ for (let outer = -1, n = lists.length; ++outer < n;) {
+ let list = lists[outer] as any;
+ for (let inner = -1, k = list.length; ++index < length && ++inner < k;) {
+ yield list[inner];
+ }
+ }
+ }
+}
+
+export type ValidityArgs = Vector<boolean> | Uint8Array;
+export class BitVector extends VirtualVector<boolean, Uint8Array> {
+ static constant: Vector<boolean> = new (class ValidVector extends Vector<boolean> {
+ get() { return true; }
+ *[Symbol.iterator]() {
+ do { yield true; } while (true);
+ }
+ })();
+ static from(src: any) {
+ return src instanceof BitVector ? src
+ : src === BitVector.constant ? src
+ : src instanceof Uint8Array ? new BitVector(src)
+ : src instanceof Array ? new BitVector(BitVector.pack(src))
+ : src instanceof Vector ? new BitVector(BitVector.pack(src))
+ : BitVector.constant as Vector<any>;
+ }
+ static pack(values: Iterable<any>) {
+ let xs = [], n, i = 0;
+ let bit = 0, byte = 0;
+ for (const value of values) {
+ value && (byte |= 1 << bit);
+ if (++bit === 8) {
+ xs[i++] = byte;
+ byte = bit = 0;
+ }
+ }
+ if (i === 0 || bit > 0) { xs[i++] = byte; }
+ if (i % 8 && (n = n = i + 8 - i % 8)) {
+ do { xs[i] = 0; } while (++i < n);
+ }
+ return new Uint8Array(xs);
+ }
+ constructor(...lists: Uint8Array[]) {
+ super(...lists);
+ this.length = this.lists.reduce((l, xs) => l + xs['length'], 0);
+ }
+ get(index: number) {
+ /* inlined `findVirtual` impl */
+ let rows, length, lists = this.lists;
+ for (let batch = -1;
+ (rows = lists[++batch]) &&
+ (length = rows.length) <= index &&
+ 0 <= (index -= length);) {}
+ return !(!rows || index < 0 || (rows[index >> 3 | 0] & 1 << index % 8) === 0);
+ }
+ set(index: number, value: boolean) {
+ /* inlined `findVirtual` impl */
+ let rows, length, lists = this.lists;
+ for (let batch = -1;
+ (rows = lists[++batch]) &&
+ (length = rows.length) <= index &&
+ 0 <= (index -= length);) {}
+ if (rows && index > -1) {
+ value
+ ? (rows[index >> 3 | 0] |= (1 << (index % 8)))
+ : (rows[index >> 3 | 0] &= ~(1 << (index % 8)));
+ }
+ }
+ concat(vector: BitVector) {
+ return new BitVector(...this.lists, ...vector.lists);
+ }
+ *[Symbol.iterator]() {
+ for (const byte of super[Symbol.iterator]()) {
+ for (let i = -1; ++i < 8;) {
+ yield (byte & 1 << i) !== 0;
+ }
+ }
+ }
+}
+
+export class TypedVector<T, TArrayType> extends VirtualVector<T, TArrayType> {
+ constructor(validity: ValidityArgs, ...lists: TArrayType[]) {
+ super(...lists);
+ validity && (this.validity = BitVector.from(validity));
+ }
+ concat(vector: TypedVector<T, TArrayType>) {
+ return (this.constructor as typeof TypedVector).from(this,
+ this.length + vector.length,
+ this.validity.concat(vector.validity),
+ ...this.lists, ...vector.lists
+ );
+ }
+}
+
+export class DateVector extends TypedVector<Date, Uint32Array> {
+ get(index: number) {
+ return !this.validity.get(index) ? null : new Date(
+ Math.pow(2, 32) *
+ <any> super.get(2 * index + 1) +
+ <any> super.get(2 * index)
+ );
+ }
+ *[Symbol.iterator]() {
+ let v, low, high;
+ let it = super[Symbol.iterator]();
+ let iv = this.validity[Symbol.iterator]();
+ while (!(v = iv.next()).done && !(low = it.next()).done && !(high = it.next()).done) {
+ yield !v.value ? null : new Date(Math.pow(2, 32) * high.value + low.value);
+ }
+ }
+}
+
+export class IndexVector extends TypedVector<number | number[], Int32Array> {
+ get(index: number, returnWithBatchIndex = false) {
+ /* inlined `findVirtual` impl */
+ let rows, length, batch = -1, lists = this.lists;
+ for (;
+ (rows = lists[++batch]) &&
+ (length = rows.length) <= index &&
+ 0 <= (index -= length);) {}
+ return !returnWithBatchIndex
+ ? (rows && -1 < index ? rows[index + batch] : null) as number
+ : (rows && -1 < index ? [rows[index + batch], batch] : [0, -1]) as number[];
+ }
+ *[Symbol.iterator]() {
+ // Alternate between iterating a tuple of [from, batch], and to. The from
+ // and to values are relative to the record batch they're defined in, so
+ // `ListVectorBase` needs to know the right batch to read.
+ let xs = new Int32Array(2), { lists } = this;
+ for (let i = -1, n = lists.length; ++i < n;) {
+ let list = lists[i] as any;
+ for (let j = -1, k = list.length - 1; ++j < k;) {
+ xs[1] = i;
+ xs[0] = list[j];
+ yield xs;
+ yield list[j + 1];
+ }
+ }
+ }
+}
+
+export class ByteVector<TList> extends TypedVector<number, TList> {
+ get(index: number) {
+ return this.validity.get(index) ? super.get(index) : null;
+ }
+ *[Symbol.iterator]() {
+ let v, r, { validity } = this;
+ let it = super[Symbol.iterator]();
+ // fast path the case of no nulls
+ if (validity === BitVector.constant) {
+ yield* it;
+ } else {
+ let iv = validity[Symbol.iterator]();
+ while (!(v = iv.next()).done && !(r = it.next()).done) {
+ yield !v.value ? null : r.value;
+ }
+ }
+ }
+}
+
+export class LongVector<TList> extends TypedVector<Long, TList> {
+ get(index: number) {
+ return !this.validity.get(index) ? null : new Long(
+ <any> super.get(index * 2), /* low */
+ <any> super.get(index * 2 + 1) /* high */
+ );
+ }
+ *[Symbol.iterator]() {
+ let v, low, high;
+ let it = super[Symbol.iterator]();
+ let iv = this.validity[Symbol.iterator]();
+ while (!(v = iv.next()).done && !(low = it.next()).done && !(high = it.next()).done) {
+ yield !v.value ? null : new Long(low.value, high.value);
+ }
+ }
+}
+
+export class Int8Vector extends ByteVector<Int8Array> {}
+export class Int16Vector extends ByteVector<Int16Array> {}
+export class Int32Vector extends ByteVector<Int32Array> {}
+export class Int64Vector extends LongVector<Int32Array> {}
+export class Uint8Vector extends ByteVector<Uint8Array> {}
+export class Uint16Vector extends ByteVector<Uint16Array> {}
+export class Uint32Vector extends ByteVector<Uint32Array> {}
+export class Uint64Vector extends LongVector<Uint32Array> {}
+export class Float32Vector extends ByteVector<Float32Array> {}
+export class Float64Vector extends ByteVector<Float64Array> {}
+
+LongVector.prototype.stride = 2;
+(Vector.prototype as any).lists = [];
+(Vector.prototype as any).validity = BitVector.constant;
+(VirtualVector.prototype as any)._arrayType = Array;
+(BitVector.prototype as any)._arrayType = Uint8Array;
+(Int8Vector.prototype as any)._arrayType = Int8Array;
+(Int16Vector.prototype as any)._arrayType = Int16Array;
+(Int32Vector.prototype as any)._arrayType = Int32Array;
+(Int64Vector.prototype as any)._arrayType = Int32Array;
+(Uint8Vector.prototype as any)._arrayType = Uint8Array;
+(Uint16Vector.prototype as any)._arrayType = Uint16Array;
+(Uint32Vector.prototype as any)._arrayType = Uint32Array;
+(Uint64Vector.prototype as any)._arrayType = Uint32Array;
+(DateVector.prototype as any)._arrayType = Uint32Array;
+(IndexVector.prototype as any)._arrayType = Int32Array;
+(Float32Vector.prototype as any)._arrayType = Float32Array;
+(Float64Vector.prototype as any)._arrayType = Float64Array;
+
+function arraySet<T>(source: Array<T>, target: Array<T>, index: number) {
+ for (let i = 0, n = source.length; i < n;) {
+ target[index++] = source[i++];
+ }
+ return index;
+}
+
+function typedArraySet(source: TypedArray, target: TypedArray, index: number) {
+ return target.set(source, index) || index + source.length;
+}
+
+// Rather than eat the iterator cost, we've inlined this function into the relevant functions
+// function* findVirtual<TList>(index: number, lists: TList[], batch?: number) {
+// let rows, length;
+// for (batch = (batch || 0) - 1;
+// (rows = lists[++batch]) &&
+// (length = rows.length) <= index &&
+// 0 <= (index -= length);) {}
+// return rows && -1 < index ? yield [rows, index, batch] : null;
+// }
+
+export type TypedArrayCtor<T extends TypedArray> = {
+ readonly prototype: T;
+ readonly BYTES_PER_ELEMENT: number;
+ new(length: number): T;
+ new(array: ArrayLike<number>): T;
+ new(buffer: ArrayBufferLike, byteOffset?: number, length?: number): T;
+};
+
+export type FloatArray = Float32Array | Float64Array;
+export type IntArray = Int8Array | Int16Array | Int32Array | Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array;
+
+export type TypedArray = (
+ Int8Array |
+ Uint8Array |
+ Int16Array |
+ Int32Array |
+ Uint16Array |
+ Uint32Array |
+ Float32Array |
+ Float64Array |
+ Uint8ClampedArray);
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/src/vector/vector.ts
----------------------------------------------------------------------
diff --git a/js/src/vector/vector.ts b/js/src/vector/vector.ts
new file mode 100644
index 0000000..1f39f87
--- /dev/null
+++ b/js/src/vector/vector.ts
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import * as Schema_ from '../format/Schema_generated';
+export import Type = Schema_.org.apache.arrow.flatbuf.Type;
+export import Field = Schema_.org.apache.arrow.flatbuf.Field;
+
+export function sliceToRangeArgs(length: number, start: number, end?: number) {
+ let total = length, from = start || 0;
+ let to = end === end && typeof end == 'number' ? end : total;
+ if (to < 0) { to = total + to; }
+ if (from < 0) { from = total - (from * -1) % total; }
+ if (to < from) { from = to; to = start; }
+ total = !isFinite(total = (to - from)) || total < 0 ? 0 : total;
+ return [from, total];
+}
+
+export class Vector<T> implements Iterable<T> {
+ static defaultName = '';
+ static defaultProps = new Map();
+ static defaultType = Type[Type.NONE];
+ static create<T = any>(field: Field, length: number, ...args: any[]) {
+ let vector = new this<T>(...args), m;
+ vector.length = length;
+ vector.name = field.name();
+ vector.type = Type[field.typeType()];
+ if ((m = field.customMetadataLength()) > 0) {
+ let entry, i = 0, data = vector.props = new Map();
+ do {
+ entry = field.customMetadata(i);
+ data[entry.key()] = entry.value();
+ } while (++i < m);
+ }
+ return vector;
+ }
+ static from<T = any>(source: Vector<T>, length: number, ...args: any[]) {
+ let vector = new this<T>(...args);
+ vector.length = length;
+ source.name !== Vector.defaultName && (vector.name = source.name);
+ source.type !== Vector.defaultType && (vector.type = source.type);
+ source.props !== Vector.defaultProps && (vector.props = source.props);
+ return vector;
+ }
+ public name: string;
+ public type: string;
+ public length: number;
+ public stride: number;
+ public props: Map<PropertyKey, any>;
+ protected validity: Vector<boolean>;
+ get(index: number): T { return null; }
+ concat(vector: Vector<T>) { return vector; }
+ slice<R = T>(start?: number, end?: number, batch?: number) {
+ const { stride } = this;
+ const [offset, length] = sliceToRangeArgs(
+ stride * this.length, stride * (start || 0), stride * end
+ );
+ return this.range<R>(offset, length, batch);
+ }
+ protected range<R = T>(index: number, length: number, batch?: number) {
+ const result = new Array<R>(length);
+ for (let i = -1, n = this.length; ++i < length;) {
+ result[i] = this.get((i + index) % n) as any;
+ }
+ return result as Iterable<R>;
+ }
+ *[Symbol.iterator]() {
+ for (let i = -1, n = this.length; ++i < n;) {
+ yield this.get(i);
+ }
+ }
+}
+
+Vector.prototype.length = 0;
+Vector.prototype.stride = 1;
+Vector.prototype.name = Vector.defaultName;
+Vector.prototype.type = Vector.defaultType;
+Vector.prototype.props = Vector.defaultProps;
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/test/Arrow.ts
----------------------------------------------------------------------
diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts
new file mode 100644
index 0000000..a9ab2b7
--- /dev/null
+++ b/js/test/Arrow.ts
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/* tslint:disable */
+// Dynamically load an Ix target build based on command line arguments
+
+const target = process.env.TEST_TARGET;
+const format = process.env.TEST_MODULE;
+const resolve = require('path').resolve;
+
+// these are duplicated in the gulpfile :<
+const targets = [`es5`, `es2015`, `esnext`];
+const formats = [`cjs`, `esm`, `cls`, `umd`];
+
+function throwInvalidImportError(name: string, value: string, values: string[]) {
+ throw new Error('Unrecognized ' + name + ' \'' + value + '\'. Please run tests with \'--' + name + ' <any of ' + values.join(', ') + '>\'');
+}
+
+if (!~targets.indexOf(target)) throwInvalidImportError('target', target, targets);
+if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats);
+
+let Arrow: any = require(resolve(`./targets/${target}/${format}/Arrow.js`));
+let ArrowInternal: any = require(resolve(`./targets/${target}/${format}/Arrow.internal.js`));
+
+import { vectors as vectors_ } from '../src/Arrow.internal';
+import { Table as Table_, readBuffers as readBuffers_ } from '../src/Arrow';
+
+export let Table = Arrow.Table as typeof Table_;
+export let readBuffers = Arrow.readBuffers as typeof readBuffers_;
+
+export let vectors: typeof vectors_ = ArrowInternal.vectors;
+export namespace vectors {
+ export type Vector<T> = vectors_.Vector<T>;
+ export type BitVector = vectors_.BitVector;
+ export type ListVector<T> = vectors_.ListVector<T>;
+ export type Utf8Vector = vectors_.Utf8Vector;
+ export type DateVector = vectors_.DateVector;
+ export type IndexVector = vectors_.IndexVector;
+ export type Int8Vector = vectors_.Int8Vector;
+ export type Int16Vector = vectors_.Int16Vector;
+ export type Int32Vector = vectors_.Int32Vector;
+ export type Int64Vector = vectors_.Int64Vector;
+ export type Uint8Vector = vectors_.Uint8Vector;
+ export type Uint16Vector = vectors_.Uint16Vector;
+ export type Uint32Vector = vectors_.Uint32Vector;
+ export type Uint64Vector = vectors_.Uint64Vector;
+ export type Float32Vector = vectors_.Float32Vector;
+ export type Float64Vector = vectors_.Float64Vector;
+ export type StructVector = vectors_.StructVector;
+ export type DictionaryVector<T> = vectors_.DictionaryVector<T>;
+ export type FixedSizeListVector<T> = vectors_.FixedSizeListVector<T>;
+};
+
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/test/__snapshots__/reader-tests.ts.snap
----------------------------------------------------------------------
diff --git a/js/test/__snapshots__/reader-tests.ts.snap b/js/test/__snapshots__/reader-tests.ts.snap
new file mode 100644
index 0000000..961ce87
--- /dev/null
+++ b/js/test/__snapshots__/reader-tests.ts.snap
@@ -0,0 +1,497 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"example-csv"`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
+Array [
+ "Hermione",
+ 25,
+ Float32Array [
+ -53.235599517822266,
+ 40.231998443603516,
+ ],
+]
+`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+ "Severus",
+ 30,
+ Float32Array [
+ -62.22999954223633,
+ 3,
+ ],
+]
+`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"example-csv"`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"Struct_"`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `1`;
+
+exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
+Array [
+ "Harry",
+ 20,
+ Float32Array [
+ 23,
+ -100.23652648925781,
+ ],
+]
+`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"example-csv"`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
+Array [
+ "Hermione",
+ 25,
+ Float32Array [
+ -53.235599517822266,
+ 40.231998443603516,
+ ],
+]
+`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+ "Severus",
+ 30,
+ Float32Array [
+ -62.22999954223633,
+ 3,
+ ],
+]
+`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"example-csv"`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"Struct_"`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `1`;
+
+exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
+Array [
+ "Harry",
+ 20,
+ Float32Array [
+ 23,
+ -100.23652648925781,
+ ],
+]
+`;
+
+exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct"`;
+
+exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
+
+exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
+Array [
+ "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
+ "Airbus",
+ 1502880750,
+ Float32Array [
+ 32.45663833618164,
+ 1.8712350130081177,
+ ],
+]
+`;
+
+exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+ "50fb46f4-fefa-42c1-919c-0121974cdd00",
+ "Boeing",
+ 1502880750,
+ Float32Array [
+ 38.766666412353516,
+ -4.181231498718262,
+ ],
+]
+`;
+
+exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct"`;
+
+exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`;
+
+exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `
+Array [
+ "a0fb47f9-f8fb-4403-a64a-786d7611f8ef",
+ "12345",
+ "Airbus",
+ 1502880750,
+ Float32Array [
+ 32.45663833618164,
+ 1.8712350130081177,
+ ],
+]
+`;
+
+exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+ "50fb46f4-fefa-42c1-919c-0121974cdd00",
+ "67890",
+ "Boeing",
+ 1502880750,
+ Float32Array [
+ 38.766666412353516,
+ -4.181231498718262,
+ ],
+]
+`;
+
+exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"row_count"`;
+
+exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`;
+
+exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `1`;
+
+exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `10000`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"origin_lat"`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"FloatingPoint"`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `35.393089294433594`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `35.393089294433594`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `35.393089294433594`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `29.533695220947266`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `29.533695220947266`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"origin_lon"`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `-97.6007308959961`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `-97.6007308959961`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `-97.6007308959961`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `-98.46977996826172`;
+
+exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `-98.46977996826172`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"origin_city"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Utf8"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `"Oklahoma City"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `"Oklahoma City"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"Oklahoma City"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"San Antonio"`;
+
+exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `"San Antonio"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"foo"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `1`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `null`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `3`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `4`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `5`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"bar"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `1`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `null`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `4`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `5`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `"baz"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `"Utf8"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `5`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `"aa"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `null`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `null`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `"bbb"`;
+
+exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 24`] = `"cccc"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"foo"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `1`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `null`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `3`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `4`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `5`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"bar"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `1`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `null`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `4`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `5`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `"baz"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `"Utf8"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `5`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `"aa"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `null`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `null`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `"bbb"`;
+
+exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 24`] = `"cccc"`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct_nullable"`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `7`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+ null,
+ "MhRNxD4",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `
+Array [
+ 137773603,
+ "3F9HBxK",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `
+Array [
+ 410361374,
+ "aVd88fp",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
+Array [
+ null,
+ "3loZrRf",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `"struct_nullable"`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `"Struct_"`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `10`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `
+Array [
+ null,
+ null,
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `
+Array [
+ null,
+ null,
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `
+Array [
+ null,
+ "78SLiRw",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `null`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `
+Array [
+ null,
+ "0ilsf82",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `
+Array [
+ null,
+ "LjS9MbU",
+]
+`;
+
+exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `
+Array [
+ null,
+ null,
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct_nullable"`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `7`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `
+Array [
+ null,
+ "MhRNxD4",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `
+Array [
+ 137773603,
+ "3F9HBxK",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `
+Array [
+ 410361374,
+ "aVd88fp",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `
+Array [
+ null,
+ "3loZrRf",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `"struct_nullable"`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `"Struct_"`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `10`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `
+Array [
+ null,
+ null,
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `
+Array [
+ null,
+ null,
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `
+Array [
+ null,
+ "78SLiRw",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `null`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `
+Array [
+ null,
+ "0ilsf82",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `
+Array [
+ null,
+ "LjS9MbU",
+]
+`;
+
+exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `
+Array [
+ null,
+ null,
+]
+`;