You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by bh...@apache.org on 2019/06/22 00:26:57 UTC
[arrow] branch master updated: ARROW-5537: [JS] Support delta
dictionaries in RecordBatchWriter and DictionaryBuilder
This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 40fcebe ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder
40fcebe is described below
commit 40fcebea19140b4275757bd4b94f49918a0ce365
Author: ptaylor <pa...@me.com>
AuthorDate: Fri Jun 21 17:26:41 2019 -0700
ARROW-5537: [JS] Support delta dictionaries in RecordBatchWriter and DictionaryBuilder
Adds support for building and writing delta dictionaries. Moves the `dictionary` Vector pointer to the Data class, similar to https://github.com/apache/arrow/pull/4316.
Forked from https://github.com/apache/arrow/pull/4476 since this adds support for delta dictionaries to the DictionaryBuilder. Will rebase this PR after that's merged. All the work is in the last commit, here: https://github.com/apache/arrow/commit/b12d842ebe1b341b2b0c67525bcf3f85ea88fecd
Author: ptaylor <pa...@me.com>
Closes #4502 from trxcllnt/js/delta-dictionaries and squashes the following commits:
6a70a25bd <ptaylor> make dictionarybuilder and recordbatchwriter support delta dictionaries
---
js/bin/print-buffer-alignment.js | 4 +-
js/src/builder.ts | 16 +---
js/src/builder/dictionary.ts | 36 ++++++---
js/src/column.ts | 2 +-
js/src/data.ts | 19 +++--
js/src/interfaces.ts | 4 +-
js/src/io/node/builder.ts | 15 ----
js/src/io/whatwg/builder.ts | 16 ----
js/src/ipc/metadata/json.ts | 29 ++++---
js/src/ipc/metadata/message.ts | 29 ++++---
js/src/ipc/reader.ts | 16 ++--
js/src/ipc/writer.ts | 102 +++++++++++++++----------
js/src/recordbatch.ts | 33 +++++++-
js/src/schema.ts | 37 +++------
js/src/type.ts | 6 +-
js/src/util/args.ts | 6 +-
js/src/vector/chunked.ts | 6 +-
js/src/vector/date.ts | 43 +++++------
js/src/vector/dictionary.ts | 12 +--
js/src/visitor/vectorloader.ts | 14 +++-
js/test/data/tables.ts | 1 -
js/test/generate-test-data.ts | 25 +++---
js/test/unit/ipc/message-reader-tests.ts | 19 ++---
js/test/unit/ipc/writer/stream-writer-tests.ts | 45 ++++++++++-
js/test/unit/table-tests.ts | 24 +++---
js/test/unit/vector/vector-tests.ts | 4 +-
26 files changed, 300 insertions(+), 263 deletions(-)
diff --git a/js/bin/print-buffer-alignment.js b/js/bin/print-buffer-alignment.js
index fe7f493..4c32603 100755
--- a/js/bin/print-buffer-alignment.js
+++ b/js/bin/print-buffer-alignment.js
@@ -73,9 +73,9 @@ const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`)
})().catch((e) => { console.error(e); process.exit(1); });
function loadRecordBatch(schema, header, body) {
- return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers).visitMany(schema.fields));
+ return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields));
}
function loadDictionaryBatch(header, body, dictionaryType) {
- return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers).visitMany([dictionaryType]));
+ return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]));
}
diff --git a/js/src/builder.ts b/js/src/builder.ts
index 6f12afc..4251ce4 100644
--- a/js/src/builder.ts
+++ b/js/src/builder.ts
@@ -157,13 +157,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
* @nocollapse
*/
public static throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
- const build = throughIterable(options);
- if (!DataType.isDictionary(options.type)) {
- return build;
- }
- return function*(source: Iterable<T['TValue'] | TNull>) {
- const chunks = []; for (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
- };
+ return throughIterable(options);
}
/**
@@ -192,13 +186,7 @@ export abstract class Builder<T extends DataType = any, TNull = any> {
* @nocollapse
*/
public static throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) {
- const build = throughAsyncIterable(options);
- if (!DataType.isDictionary(options.type)) {
- return build;
- }
- return async function* (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) {
- const chunks = []; for await (const chunk of build(source)) { chunks.push(chunk); } yield* chunks;
- };
+ return throughAsyncIterable(options);
}
/**
diff --git a/js/src/builder/dictionary.ts b/js/src/builder/dictionary.ts
index dd46d61..dda2df2 100644
--- a/js/src/builder/dictionary.ts
+++ b/js/src/builder/dictionary.ts
@@ -29,13 +29,17 @@ export interface DictionaryBuilderOptions<T extends DataType = any, TNull = any>
/** @ignore */
export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builder<T, TNull> {
- protected _codes = Object.create(null);
+ protected _dictionaryOffset: number;
+ protected _dictionary?: Vector<T['dictionary']>;
+ protected _keysToIndices: { [key: string]: number };
public readonly indices: IntBuilder<T['indices']>;
public readonly dictionary: Builder<T['dictionary']>;
constructor({ 'type': type, 'nullValues': nulls, 'dictionaryHashFunction': hashFn }: DictionaryBuilderOptions<T, TNull>) {
- super({ type });
+ super({ type: new Dictionary(type.dictionary, type.indices, type.id, type.isOrdered) as T });
this._nulls = <any> null;
+ this._dictionaryOffset = 0;
+ this._keysToIndices = Object.create(null);
this.indices = Builder.new({ 'type': this.type.indices, 'nullValues': nulls }) as IntBuilder<T['indices']>;
this.dictionary = Builder.new({ 'type': this.type.dictionary, 'nullValues': null }) as Builder<T['dictionary']>;
if (typeof hashFn === 'function') {
@@ -46,9 +50,9 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
public get values() { return this.indices.values; }
public get nullCount() { return this.indices.nullCount; }
public get nullBitmap() { return this.indices.nullBitmap; }
- public get byteLength() { return this.indices.byteLength; }
- public get reservedLength() { return this.indices.reservedLength; }
- public get reservedByteLength() { return this.indices.reservedByteLength; }
+ public get byteLength() { return this.indices.byteLength + this.dictionary.byteLength; }
+ public get reservedLength() { return this.indices.reservedLength + this.dictionary.reservedLength; }
+ public get reservedByteLength() { return this.indices.reservedByteLength + this.dictionary.reservedByteLength; }
public isValid(value: T['TValue'] | TNull) { return this.indices.isValid(value); }
public setValid(index: number, valid: boolean) {
const indices = this.indices;
@@ -57,25 +61,35 @@ export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builde
return valid;
}
public setValue(index: number, value: T['TValue']) {
- let keysToCodesMap = this._codes;
+ let keysToIndices = this._keysToIndices;
let key = this.valueToKey(value);
- let idx = keysToCodesMap[key];
+ let idx = keysToIndices[key];
if (idx === undefined) {
- keysToCodesMap[key] = idx = this.dictionary.append(value).length - 1;
+ keysToIndices[key] = idx = this._dictionaryOffset + this.dictionary.append(value).length - 1;
}
return this.indices.setValue(index, idx);
}
public flush() {
- const chunk = this.indices.flush().clone(this.type);
+ const type = this.type;
+ const prev = this._dictionary;
+ const curr = this.dictionary.toVector();
+ const data = this.indices.flush().clone(type);
+ data.dictionary = prev ? prev.concat(curr) : curr;
+ this.finished || (this._dictionaryOffset += curr.length);
+ this._dictionary = data.dictionary as Vector<T['dictionary']>;
this.clear();
- return chunk;
+ return data;
}
public finish() {
- this.type.dictionaryVector = Vector.new(this.dictionary.finish().flush());
+ this.indices.finish();
+ this.dictionary.finish();
+ this._dictionaryOffset = 0;
+ this._keysToIndices = Object.create(null);
return super.finish();
}
public clear() {
this.indices.clear();
+ this.dictionary.clear();
return super.clear();
}
public valueToKey(val: any): string | number {
diff --git a/js/src/column.ts b/js/src/column.ts
index ceb078f..3a2cb4a 100644
--- a/js/src/column.ts
+++ b/js/src/column.ts
@@ -49,7 +49,7 @@ export class Column<T extends DataType = any>
if (typeof field === 'string') {
const type = chunks[0].data.type;
- field = new Field(field, type, chunks.some(({ nullCount }) => nullCount > 0));
+ field = new Field(field, type, true);
} else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) {
field = field.clone({ nullable: true });
}
diff --git a/js/src/data.ts b/js/src/data.ts
index 01a2cf0..a17be8c 100644
--- a/js/src/data.ts
+++ b/js/src/data.ts
@@ -64,6 +64,12 @@ export class Data<T extends DataType = DataType> {
public readonly offset: number;
public readonly stride: number;
public readonly childData: Data[];
+
+ /**
+ * The dictionary for this Vector, if any. Only used for Dictionary type.
+ */
+ public dictionary?: Vector;
+
public readonly values: Buffers<T>[BufferType.DATA];
// @ts-ignore
public readonly typeIds: Buffers<T>[BufferType.TYPE];
@@ -98,8 +104,9 @@ export class Data<T extends DataType = DataType> {
return nullCount;
}
- constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]) {
+ constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector) {
this.type = type;
+ this.dictionary = dictionary;
this.offset = Math.floor(Math.max(offset || 0, 0));
this.length = Math.floor(Math.max(length || 0, 0));
this._nullCount = Math.floor(Math.max(nullCount || 0, -1));
@@ -123,7 +130,7 @@ export class Data<T extends DataType = DataType> {
}
public clone<R extends DataType>(type: R, offset = this.offset, length = this.length, nullCount = this._nullCount, buffers: Buffers<R> = <any> this, childData: (Data | Vector)[] = this.childData) {
- return new Data(type, offset, length, nullCount, buffers, childData);
+ return new Data(type, offset, length, nullCount, buffers, childData, this.dictionary);
}
public slice(offset: number, length: number): Data<T> {
@@ -173,12 +180,12 @@ export class Data<T extends DataType = DataType> {
// Convenience methods for creating Data instances for each of the Arrow Vector types
//
/** @nocollapse */
- public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]): Data<T> {
+ public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector): Data<T> {
if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; }
switch (type.typeId) {
case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[BufferType.VALIDITY]) as Data<T>;
case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
- case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
+ case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>;
case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>;
@@ -207,8 +214,8 @@ export class Data<T extends DataType = DataType> {
return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
}
/** @nocollapse */
- public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
- return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)]);
+ public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>, dictionary: Vector<T['dictionary']>) {
+ return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)], [], dictionary);
}
/** @nocollapse */
public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
diff --git a/js/src/interfaces.ts b/js/src/interfaces.ts
index 84a20b9..bebcf0c 100644
--- a/js/src/interfaces.ts
+++ b/js/src/interfaces.ts
@@ -141,7 +141,7 @@ export type BuilderType<T extends Type | DataType = any, TNull = any> =
/** @ignore */
export type VectorCtor<T extends Type | DataType | VectorType> =
- T extends VectorType ? VectorCtorType<T> :
+ T extends VectorType ? VectorCtorType<T> :
T extends Type ? VectorCtorType<VectorType<T>> :
T extends DataType ? VectorCtorType<VectorType<T['TType']>> :
VectorCtorType<vecs.BaseVector>
@@ -157,7 +157,7 @@ export type BuilderCtor<T extends Type | DataType = any> =
/** @ignore */
export type DataTypeCtor<T extends Type | DataType | VectorType = any> =
T extends DataType ? ConstructorType<T> :
- T extends VectorType ? ConstructorType<T['type']> :
+ T extends VectorType ? ConstructorType<T['type']> :
T extends Type ? ConstructorType<TypeToDataType<T>> :
never
;
diff --git a/js/src/io/node/builder.ts b/js/src/io/node/builder.ts
index 1197efb..eb95795 100644
--- a/js/src/io/node/builder.ts
+++ b/js/src/io/node/builder.ts
@@ -47,7 +47,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {
constructor(builder: Builder<T, TNull>, options: BuilderDuplexOptions<T, TNull>) {
- const isDictionary = DataType.isDictionary(builder.type);
const { queueingStrategy = 'count', autoDestroy = true } = options;
const { highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options;
@@ -58,20 +57,6 @@ class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex {
this._builder = builder;
this._desiredSize = highWaterMark;
this._getSize = queueingStrategy !== 'bytes' ? builderLength : builderByteLength;
-
- if (isDictionary) {
- let chunks: any[] = [];
- this.push = (chunk: any, _?: string) => {
- if (chunk !== null) {
- chunks.push(chunk);
- return true;
- }
- const chunks_ = chunks;
- chunks = [];
- chunks_.forEach((x) => super.push(x));
- return super.push(null) && false;
- };
- }
}
_read(size: number) {
this._maybeFlush(this._builder, this._desiredSize = size);
diff --git a/js/src/io/whatwg/builder.ts b/js/src/io/whatwg/builder.ts
index aae7cc6..15a4333 100644
--- a/js/src/io/whatwg/builder.ts
+++ b/js/src/io/whatwg/builder.ts
@@ -82,22 +82,6 @@ export class BuilderTransform<T extends DataType = any, TNull = any> {
'highWaterMark': writableHighWaterMark,
'size': (value: T['TValue'] | TNull) => this._writeValueAndReturnChunkSize(value),
});
-
- if (DataType.isDictionary(builderOptions.type)) {
- let chunks: any[] = [];
- this._enqueue = (controller: ReadableStreamDefaultController<V<T>>, chunk: V<T> | null) => {
- this._bufferedSize = 0;
- if (chunk !== null) {
- chunks.push(chunk);
- } else {
- const chunks_ = chunks;
- chunks = [];
- chunks_.forEach((x) => controller.enqueue(x));
- controller.close();
- this._controller = null;
- }
- };
- }
}
private _writeValueAndReturnChunkSize(value: T['TValue'] | TNull) {
diff --git a/js/src/ipc/metadata/json.ts b/js/src/ipc/metadata/json.ts
index a74494b..640a8dd 100644
--- a/js/src/ipc/metadata/json.ts
+++ b/js/src/ipc/metadata/json.ts
@@ -27,11 +27,11 @@ import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message
import { TimeUnit, Precision, IntervalUnit, UnionMode, DateUnit } from '../../enum';
/** @ignore */
-export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map(), dictionaryFields: Map<number, Field<Dictionary>[]> = new Map()) {
+export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map()) {
return new Schema(
- schemaFieldsFromJSON(_schema, dictionaries, dictionaryFields),
+ schemaFieldsFromJSON(_schema, dictionaries),
customMetadataFromJSON(_schema['customMetadata']),
- dictionaries, dictionaryFields
+ dictionaries
);
}
@@ -53,13 +53,13 @@ export function dictionaryBatchFromJSON(b: any) {
}
/** @ignore */
-function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
- return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
+function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>) {
+ return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}
/** @ignore */
-function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>): Field[] {
- return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries, dictionaryFields));
+function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>): Field[] {
+ return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries));
}
/** @ignore */
@@ -93,7 +93,7 @@ function nullCountFromJSON(validity: number[]) {
}
/** @ignore */
-export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
+export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>) {
let id: number;
let keys: TKeys | null;
@@ -101,11 +101,10 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
let dictMeta: any;
let type: DataType<any>;
let dictType: Dictionary;
- let dictField: Field<Dictionary>;
// If no dictionary encoding
- if (!dictionaries || !dictionaryFields || !(dictMeta = _field['dictionary'])) {
- type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields));
+ if (!dictionaries || !(dictMeta = _field['dictionary'])) {
+ type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries));
field = new Field(_field['name'], type, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
}
// tslint:disable
@@ -115,10 +114,9 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
else if (!dictionaries.has(id = dictMeta['id'])) {
// a dictionary index defaults to signed 32 bit int if unspecified
keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
- dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries, dictionaryFields)));
+ dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries)));
dictType = new Dictionary(type, keys, id, dictMeta['isOrdered']);
- dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
- dictionaryFields.set(id, [field = dictField]);
+ field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
}
// If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the
// data type and wrap in a new Dictionary type and field.
@@ -126,8 +124,7 @@ export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>,
// a dictionary index defaults to signed 32 bit int if unspecified
keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32();
dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta['isOrdered']);
- dictField = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
- dictionaryFields.get(id)!.push(field = dictField);
+ field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata']));
}
return field || null;
}
diff --git a/js/src/ipc/metadata/message.ts b/js/src/ipc/metadata/message.ts
index 6687b65..7f40e8d 100644
--- a/js/src/ipc/metadata/message.ts
+++ b/js/src/ipc/metadata/message.ts
@@ -280,9 +280,9 @@ declare module './message' {
}
/** @ignore */
-function decodeSchema(_schema: _Schema, dictionaries: Map<number, DataType> = new Map(), dictionaryFields: Map<number, Field<Dictionary>[]> = new Map()) {
- const fields = decodeSchemaFields(_schema, dictionaries, dictionaryFields);
- return new Schema(fields, decodeCustomMetadata(_schema), dictionaries, dictionaryFields);
+function decodeSchema(_schema: _Schema, dictionaries: Map<number, DataType> = new Map()) {
+ const fields = decodeSchemaFields(_schema, dictionaries);
+ return new Schema(fields, decodeCustomMetadata(_schema), dictionaries);
}
/** @ignore */
@@ -334,29 +334,29 @@ function decodeBuffers(batch: _RecordBatch, version: MetadataVersion) {
}
/** @ignore */
-function decodeSchemaFields(schema: _Schema, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
+function decodeSchemaFields(schema: _Schema, dictionaries?: Map<number, DataType>) {
const fields = [] as Field[];
for (let f, i = -1, j = -1, n = schema.fieldsLength(); ++i < n;) {
if (f = schema.fields(i)) {
- fields[++j] = Field.decode(f, dictionaries, dictionaryFields);
+ fields[++j] = Field.decode(f, dictionaries);
}
}
return fields;
}
/** @ignore */
-function decodeFieldChildren(field: _Field, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>): Field[] {
+function decodeFieldChildren(field: _Field, dictionaries?: Map<number, DataType>): Field[] {
const children = [] as Field[];
for (let f, i = -1, j = -1, n = field.childrenLength(); ++i < n;) {
if (f = field.children(i)) {
- children[++j] = Field.decode(f, dictionaries, dictionaryFields);
+ children[++j] = Field.decode(f, dictionaries);
}
}
return children;
}
/** @ignore */
-function decodeField(f: _Field, dictionaries?: Map<number, DataType>, dictionaryFields?: Map<number, Field<Dictionary>[]>) {
+function decodeField(f: _Field, dictionaries?: Map<number, DataType>) {
let id: number;
let field: Field | void;
@@ -364,11 +364,10 @@ function decodeField(f: _Field, dictionaries?: Map<number, DataType>, dictionary
let keys: _Int | TKeys | null;
let dictType: Dictionary;
let dictMeta: _DictionaryEncoding | null;
- let dictField: Field<Dictionary>;
// If no dictionary encoding
- if (!dictionaries || !dictionaryFields || !(dictMeta = f.dictionary())) {
- type = decodeFieldType(f, decodeFieldChildren(f, dictionaries, dictionaryFields));
+ if (!dictionaries || !(dictMeta = f.dictionary())) {
+ type = decodeFieldType(f, decodeFieldChildren(f, dictionaries));
field = new Field(f.name()!, type, f.nullable(), decodeCustomMetadata(f));
}
// tslint:disable
@@ -378,10 +377,9 @@ function decodeField(f: _Field, dictionaries?: Map<number, DataType>, dictionary
else if (!dictionaries.has(id = dictMeta.id().low)) {
// a dictionary index defaults to signed 32 bit int if unspecified
keys = (keys = dictMeta.indexType()) ? decodeIndexType(keys) as TKeys : new Int32();
- dictionaries.set(id, type = decodeFieldType(f, decodeFieldChildren(f, dictionaries, dictionaryFields)));
+ dictionaries.set(id, type = decodeFieldType(f, decodeFieldChildren(f, dictionaries)));
dictType = new Dictionary(type, keys, id, dictMeta.isOrdered());
- dictField = new Field(f.name()!, dictType, f.nullable(), decodeCustomMetadata(f));
- dictionaryFields.set(id, [field = dictField]);
+ field = new Field(f.name()!, dictType, f.nullable(), decodeCustomMetadata(f));
}
// If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the
// data type and wrap in a new Dictionary type and field.
@@ -389,8 +387,7 @@ function decodeField(f: _Field, dictionaries?: Map<number, DataType>, dictionary
// a dictionary index defaults to signed 32 bit int if unspecified
keys = (keys = dictMeta.indexType()) ? decodeIndexType(keys) as TKeys : new Int32();
dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta.isOrdered());
- dictField = new Field(f.name()!, dictType, f.nullable(), decodeCustomMetadata(f));
- dictionaryFields.get(id)!.push(field = dictField);
+ field = new Field(f.name()!, dictType, f.nullable(), decodeCustomMetadata(f));
}
return field || null;
}
diff --git a/js/src/ipc/reader.ts b/js/src/ipc/reader.ts
index 6778c50..d655d8e 100644
--- a/js/src/ipc/reader.ts
+++ b/js/src/ipc/reader.ts
@@ -354,21 +354,17 @@ abstract class RecordBatchReaderImpl<T extends { [key: string]: DataType } = any
protected _loadDictionaryBatch(header: metadata.DictionaryBatch, body: any) {
const { id, isDelta, data } = header;
const { dictionaries, schema } = this;
- if (isDelta || !dictionaries.get(id)) {
-
+ const dictionary = dictionaries.get(id);
+ if (isDelta || !dictionary) {
const type = schema.dictionaries.get(id)!;
- const vector = (isDelta ? dictionaries.get(id)!.concat(
+ return (dictionary && isDelta ? dictionary.concat(
Vector.new(this._loadVectors(data, body, [type])[0])) :
Vector.new(this._loadVectors(data, body, [type])[0])) as Vector;
-
- (schema.dictionaryFields.get(id) || []).forEach(({ type }) => type.dictionaryVector = vector);
-
- return vector;
}
- return dictionaries.get(id)!;
+ return dictionary;
}
protected _loadVectors(header: metadata.RecordBatch, body: any, types: (Field | DataType)[]) {
- return new VectorLoader(body, header.nodes, header.buffers).visitMany(types);
+ return new VectorLoader(body, header.nodes, header.buffers, this.dictionaries).visitMany(types);
}
}
@@ -676,7 +672,7 @@ class RecordBatchJSONReaderImpl<T extends { [key: string]: DataType } = any> ext
super(source, dictionaries);
}
protected _loadVectors(header: metadata.RecordBatch, body: any, types: (Field | DataType)[]) {
- return new JSONVectorLoader(body, header.nodes, header.buffers).visitMany(types);
+ return new JSONVectorLoader(body, header.nodes, header.buffers, this.dictionaries).visitMany(types);
}
}
diff --git a/js/src/ipc/writer.ts b/js/src/ipc/writer.ts
index 14a8d0f..2e60c5c 100644
--- a/js/src/ipc/writer.ts
+++ b/js/src/ipc/writer.ts
@@ -19,11 +19,10 @@ import { Table } from '../table';
import { MAGIC } from './message';
import { Vector } from '../vector';
import { Column } from '../column';
+import { DataType } from '../type';
import { Schema, Field } from '../schema';
-import { Chunked } from '../vector/chunked';
import { Message } from './metadata/message';
import * as metadata from './metadata/message';
-import { DataType, Dictionary } from '../type';
import { FileBlock, Footer } from './metadata/file';
import { MessageHeader, MetadataVersion } from '../enum';
import { WritableSink, AsyncByteQueue } from '../io/stream';
@@ -65,6 +64,7 @@ export class RecordBatchWriter<T extends { [key: string]: DataType } = any> exte
protected _schema: Schema | null = null;
protected _dictionaryBlocks: FileBlock[] = [];
protected _recordBatchBlocks: FileBlock[] = [];
+ protected _dictionaryDeltaOffsets = new Map<number, number>();
public toString(sync: true): string;
public toString(sync?: false): Promise<string>;
@@ -119,12 +119,13 @@ export class RecordBatchWriter<T extends { [key: string]: DataType } = any> exte
}
if (this._started && this._schema) {
- this._writeFooter();
+ this._writeFooter(this._schema);
}
this._started = false;
this._dictionaryBlocks = [];
this._recordBatchBlocks = [];
+ this._dictionaryDeltaOffsets = new Map();
if (!schema || !(schema.compareTo(this._schema))) {
if (schema === null) {
@@ -206,12 +207,11 @@ export class RecordBatchWriter<T extends { [key: string]: DataType } = any> exte
}
protected _writeSchema(schema: Schema<T>) {
- return this
- ._writeMessage(Message.from(schema))
- ._writeDictionaries(schema.dictionaryFields);
+ return this._writeMessage(Message.from(schema));
}
- protected _writeFooter() {
+ // @ts-ignore
+ protected _writeFooter(schema: Schema<T>) {
return this._writePadding(4); // eos bytes
}
@@ -223,16 +223,18 @@ export class RecordBatchWriter<T extends { [key: string]: DataType } = any> exte
return nBytes > 0 ? this._write(new Uint8Array(nBytes)) : this;
}
- protected _writeRecordBatch(records: RecordBatch<T>) {
- const { byteLength, nodes, bufferRegions, buffers } = VectorAssembler.assemble(records);
- const recordBatch = new metadata.RecordBatch(records.length, nodes, bufferRegions);
+ protected _writeRecordBatch(batch: RecordBatch<T>) {
+ const { byteLength, nodes, bufferRegions, buffers } = VectorAssembler.assemble(batch);
+ const recordBatch = new metadata.RecordBatch(batch.length, nodes, bufferRegions);
const message = Message.from(recordBatch, byteLength);
return this
+ ._writeDictionaries(batch)
._writeMessage(message)
._writeBodyBuffers(buffers);
}
protected _writeDictionaryBatch(dictionary: Vector, id: number, isDelta = false) {
+ this._dictionaryDeltaOffsets.set(id, dictionary.length + (this._dictionaryDeltaOffsets.get(id) || 0));
const { byteLength, nodes, bufferRegions, buffers } = VectorAssembler.assemble(dictionary);
const recordBatch = new metadata.RecordBatch(dictionary.length, nodes, bufferRegions);
const dictionaryBatch = new metadata.DictionaryBatch(recordBatch, id, isDelta);
@@ -256,15 +258,14 @@ export class RecordBatchWriter<T extends { [key: string]: DataType } = any> exte
return this;
}
- protected _writeDictionaries(dictionaryFields: Map<number, Field<Dictionary<any, any>>[]>) {
- for (const [id, fields] of dictionaryFields) {
- const vector = fields[0].type.dictionaryVector;
- if (!(vector instanceof Chunked)) {
- this._writeDictionaryBatch(vector, id, false);
- } else {
- const chunks = vector.chunks;
- for (let i = -1, n = chunks.length; ++i < n;) {
- this._writeDictionaryBatch(chunks[i], id, i > 0);
+ protected _writeDictionaries(batch: RecordBatch<T>) {
+ for (let [id, dictionary] of batch.dictionaries) {
+ let offset = this._dictionaryDeltaOffsets.get(id) || 0;
+ if (offset === 0 || (dictionary = dictionary.slice(offset)).length > 0) {
+ const chunks = 'chunks' in dictionary ? (dictionary as any).chunks : [dictionary];
+ for (const chunk of chunks) {
+ this._writeDictionaryBatch(chunk, id, offset > 0);
+ offset += chunk.length;
}
}
}
@@ -312,15 +313,14 @@ export class RecordBatchFileWriter<T extends { [key: string]: DataType } = any>
this._autoDestroy = true;
}
+ // @ts-ignore
protected _writeSchema(schema: Schema<T>) {
- return this
- ._writeMagic()._writePadding(2)
- ._writeDictionaries(schema.dictionaryFields);
+ return this._writeMagic()._writePadding(2);
}
- protected _writeFooter() {
+ protected _writeFooter(schema: Schema<T>) {
const buffer = Footer.encode(new Footer(
- this._schema!, MetadataVersion.V4,
+ schema, MetadataVersion.V4,
this._recordBatchBlocks, this._dictionaryBlocks
));
return this
@@ -344,43 +344,66 @@ export class RecordBatchJSONWriter<T extends { [key: string]: DataType } = any>
return new RecordBatchJSONWriter<T>().writeAll(input as any);
}
+ private _recordBatches: RecordBatch[];
+ private _dictionaries: RecordBatch[];
+
constructor() {
super();
this._autoDestroy = true;
+ this._recordBatches = [];
+ this._dictionaries = [];
}
protected _writeMessage() { return this; }
protected _writeSchema(schema: Schema<T>) {
return this._write(`{\n "schema": ${
JSON.stringify({ fields: schema.fields.map(fieldToJSON) }, null, 2)
- }`)._writeDictionaries(schema.dictionaryFields);
+ }`);
}
- protected _writeDictionaries(dictionaryFields: Map<number, Field<Dictionary<any, any>>[]>) {
- this._write(`,\n "dictionaries": [\n`);
- super._writeDictionaries(dictionaryFields);
- return this._write(`\n ]`);
+ protected _writeDictionaries(batch: RecordBatch<T>) {
+ if (batch.dictionaries.size > 0) {
+ this._dictionaries.push(batch);
+ }
+ return this;
}
protected _writeDictionaryBatch(dictionary: Vector, id: number, isDelta = false) {
+ this._dictionaryDeltaOffsets.set(id, dictionary.length + (this._dictionaryDeltaOffsets.get(id) || 0));
this._write(this._dictionaryBlocks.length === 0 ? ` ` : `,\n `);
- this._write(`${dictionaryBatchToJSON(this._schema!, dictionary, id, isDelta)}`);
+ this._write(`${dictionaryBatchToJSON(dictionary, id, isDelta)}`);
this._dictionaryBlocks.push(new FileBlock(0, 0, 0));
return this;
}
- protected _writeRecordBatch(records: RecordBatch<T>) {
- this._write(this._recordBatchBlocks.length === 0
- ? `,\n "batches": [\n `
- : `,\n `);
- this._write(`${recordBatchToJSON(records)}`);
- this._recordBatchBlocks.push(new FileBlock(0, 0, 0));
+ protected _writeRecordBatch(batch: RecordBatch<T>) {
+ this._writeDictionaries(batch);
+ this._recordBatches.push(batch);
return this;
}
public close() {
- if (this._recordBatchBlocks.length > 0) {
+
+ if (this._dictionaries.length > 0) {
+ this._write(`,\n "dictionaries": [\n`);
+ for (const batch of this._dictionaries) {
+ super._writeDictionaries(batch);
+ }
+ this._write(`\n ]`);
+ }
+
+ if (this._recordBatches.length > 0) {
+ for (let i = -1, n = this._recordBatches.length; ++i < n;) {
+ this._write(i === 0 ? `,\n "batches": [\n ` : `,\n `);
+ this._write(`${recordBatchToJSON(this._recordBatches[i])}`);
+ this._recordBatchBlocks.push(new FileBlock(0, 0, 0));
+ }
this._write(`\n ]`);
}
+
if (this._schema) {
this._write(`\n}`);
}
+
+ this._dictionaries = [];
+ this._recordBatches = [];
+
return super.close();
}
}
@@ -422,9 +445,8 @@ function fieldToJSON({ name, type, nullable }: Field): object {
}
/** @ignore */
-function dictionaryBatchToJSON(schema: Schema, dictionary: Vector, id: number, isDelta = false) {
- const f = schema.dictionaryFields.get(id)![0];
- const field = new Field(f.name, f.type.dictionary, f.nullable, f.metadata);
+function dictionaryBatchToJSON(dictionary: Vector, id: number, isDelta = false) {
+ const field = new Field(`${id}`, dictionary.type, dictionary.nullCount > 0);
const columns = JSONVectorAssembler.assemble(new Column(field, [dictionary]));
return JSON.stringify({
'id': id,
diff --git a/js/src/recordbatch.ts b/js/src/recordbatch.ts
index 2e1af41..af4991e 100644
--- a/js/src/recordbatch.ts
+++ b/js/src/recordbatch.ts
@@ -18,12 +18,13 @@
import { Data } from './data';
import { Table } from './table';
import { Vector } from './vector';
+import { Visitor } from './visitor';
import { Schema, Field } from './schema';
import { isIterable } from './util/compat';
import { Chunked } from './vector/chunked';
import { MapVector } from './vector/index';
import { selectFieldArgs } from './util/args';
-import { DataType, Struct, Map_ } from './type';
+import { DataType, Struct, Map_, Dictionary } from './type';
import { ensureSameLengthData } from './util/recordbatch';
import { Clonable, Sliceable, Applicative } from './vector';
import { VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index';
@@ -64,6 +65,7 @@ export class RecordBatch<T extends { [key: string]: DataType } = any>
}
protected _schema: Schema;
+ protected _dictionaries?: Map<number, Vector>;
constructor(schema: Schema<T>, length: number, children: (Data | Vector)[]);
constructor(schema: Schema<T>, data: Data<Map_<T>>, children?: Vector[]);
@@ -93,6 +95,9 @@ export class RecordBatch<T extends { [key: string]: DataType } = any>
public get schema() { return this._schema; }
public get numCols() { return this._schema.fields.length; }
+ public get dictionaries() {
+ return this._dictionaries || (this._dictionaries = DictionaryCollector.collect(this));
+ }
public select<K extends keyof T = any>(...columnNames: K[]) {
const nameToIndex = this._schema.fields.reduce((m, f, i) => m.set(f.name as K, i), new Map<K, number>());
@@ -119,3 +124,29 @@ export class _InternalEmptyPlaceholderRecordBatch<T extends { [key: string]: Dat
super(schema, 0, schema.fields.map((f) => Data.new(f.type, 0, 0, 0)));
}
}
+
+/** @ignore */
+class DictionaryCollector extends Visitor {
+ public dictionaries = new Map<number, Vector>();
+ public static collect<T extends RecordBatch>(batch: T) {
+ return new DictionaryCollector().visit(
+ batch.data, new Map_(batch.schema.fields)
+ ).dictionaries;
+ }
+ public visit(data: Data, type: DataType) {
+ if (DataType.isDictionary(type)) {
+ return this.visitDictionary(data, type);
+ } else {
+ data.childData.forEach((child, i) =>
+ this.visit(child, type.children[i].type));
+ }
+ return this;
+ }
+ public visitDictionary(data: Data, type: Dictionary) {
+ const dictionary = data.dictionary;
+ if (dictionary && dictionary.length > 0) {
+ this.dictionaries.set(type.id, dictionary);
+ }
+ return this;
+ }
+}
diff --git a/js/src/schema.ts b/js/src/schema.ts
index d4cbd05..c67ff0f 100644
--- a/js/src/schema.ts
+++ b/js/src/schema.ts
@@ -17,8 +17,8 @@
import { Data } from './data';
import { Vector } from './vector';
+import { DataType } from './type';
import { selectArgs } from './util/args';
-import { DataType, Dictionary } from './type';
import { selectFieldArgs } from './util/args';
import { instance as comparer } from './visitor/typecomparator';
@@ -47,21 +47,16 @@ export class Schema<T extends { [key: string]: DataType } = any> {
public readonly fields: Field<T[keyof T]>[];
public readonly metadata: Map<string, string>;
public readonly dictionaries: Map<number, DataType>;
- public readonly dictionaryFields: Map<number, Field<Dictionary>[]>;
constructor(fields: Field[] = [],
metadata?: Map<string, string> | null,
- dictionaries?: Map<number, DataType> | null,
- dictionaryFields?: Map<number, Field<Dictionary>[]> | null) {
+ dictionaries?: Map<number, DataType> | null) {
this.fields = (fields || []) as Field<T[keyof T]>[];
this.metadata = metadata || new Map();
- if (!dictionaries || !dictionaryFields) {
- ({ dictionaries, dictionaryFields } = generateDictionaryMap(
- fields, dictionaries || new Map(), dictionaryFields || new Map()
- ));
+ if (!dictionaries) {
+ dictionaries = generateDictionaryMap(fields);
}
this.dictionaries = dictionaries;
- this.dictionaryFields = dictionaryFields;
}
public get [Symbol.toStringTag]() { return 'Schema'; }
public toString() {
@@ -88,7 +83,6 @@ export class Schema<T extends { [key: string]: DataType } = any> {
: new Schema<R>(selectArgs<Field<R[keyof R]>>(Field, args));
const curFields = [...this.fields] as Field[];
- const curDictionaryFields = this.dictionaryFields;
const metadata = mergeMaps(mergeMaps(new Map(), this.metadata), other.metadata);
const newFields = other.fields.filter((f2) => {
const i = curFields.findIndex((f) => f.name === f2.name);
@@ -97,17 +91,11 @@ export class Schema<T extends { [key: string]: DataType } = any> {
})) && false : true;
}) as Field[];
- const { dictionaries: newDictionaries, dictionaryFields } = generateDictionaryMap(newFields, new Map(), new Map());
- const newDictionaryFields = [...dictionaryFields].map(([id, newDictFields]) => {
- return [id, [...(curDictionaryFields.get(id) || []), ...newDictFields.map((f) => {
- return newFields[newFields.findIndex((f2) => f.name === f2.name)] = f.clone();
- })]] as [number, Field<Dictionary>[]];
- });
+ const newDictionaries = generateDictionaryMap(newFields, new Map());
return new Schema<T & R>(
[...curFields, ...newFields], metadata,
- new Map([...this.dictionaries, ...newDictionaries]),
- new Map([...curDictionaryFields, ...newDictionaryFields])
+ new Map([...this.dictionaries, ...newDictionaries])
);
}
}
@@ -163,28 +151,24 @@ function mergeMaps<TKey, TVal>(m1?: Map<TKey, TVal> | null, m2?: Map<TKey, TVal>
}
/** @ignore */
-function generateDictionaryMap(fields: Field[], dictionaries: Map<number, DataType>, dictionaryFields: Map<number, Field<Dictionary>[]>) {
+function generateDictionaryMap(fields: Field[], dictionaries = new Map<number, DataType>()): Map<number, DataType> {
for (let i = -1, n = fields.length; ++i < n;) {
const field = fields[i];
const type = field.type;
if (DataType.isDictionary(type)) {
- if (!dictionaryFields.get(type.id)) {
- dictionaryFields.set(type.id, []);
- }
if (!dictionaries.has(type.id)) {
dictionaries.set(type.id, type.dictionary);
- dictionaryFields.get(type.id)!.push(field as any);
} else if (dictionaries.get(type.id) !== type.dictionary) {
throw new Error(`Cannot create Schema containing two different dictionaries with the same Id`);
}
}
- if (type.children) {
- generateDictionaryMap(type.children, dictionaries, dictionaryFields);
+ if (type.children && type.children.length > 0) {
+ generateDictionaryMap(type.children, dictionaries);
}
}
- return { dictionaries, dictionaryFields };
+ return dictionaries;
}
// Add these here so they're picked up by the externs creator
@@ -192,7 +176,6 @@ function generateDictionaryMap(fields: Field[], dictionaries: Map<number, DataTy
(Schema.prototype as any).fields = null;
(Schema.prototype as any).metadata = null;
(Schema.prototype as any).dictionaries = null;
-(Schema.prototype as any).dictionaryFields = null;
(Field.prototype as any).type = null;
(Field.prototype as any).name = null;
diff --git a/js/src/type.ts b/js/src/type.ts
index b43a353..d1aee98 100644
--- a/js/src/type.ts
+++ b/js/src/type.ts
@@ -18,7 +18,6 @@
/* tslint:disable:class-name */
import { Field } from './schema';
-import { Vector } from './vector';
import { flatbuffers } from 'flatbuffers';
import { TypedArrayConstructor } from './interfaces';
import { VectorType as V, TypeToDataType } from './interfaces';
@@ -553,13 +552,11 @@ export class Dictionary<T extends DataType = any, TKey extends TKeys = TKeys> ex
public readonly indices: TKey;
public readonly dictionary: T;
public readonly isOrdered: boolean;
- public dictionaryVector: Vector<T>;
- constructor(dictionary: T, indices: TKey, id?: Long | number | null, isOrdered?: boolean | null, dictionaryVector?: Vector<T>) {
+ constructor(dictionary: T, indices: TKey, id?: Long | number | null, isOrdered?: boolean | null) {
super();
this.indices = indices;
this.dictionary = dictionary;
this.isOrdered = isOrdered || false;
- this.dictionaryVector = dictionaryVector!;
this.id = id == null ? getId() : typeof id === 'number' ? id : id.low;
}
public get typeId() { return Type.Dictionary as Type.Dictionary; }
@@ -572,7 +569,6 @@ export class Dictionary<T extends DataType = any, TKey extends TKeys = TKeys> ex
(<any> proto).indices = null;
(<any> proto).isOrdered = null;
(<any> proto).dictionary = null;
- (<any> proto).dictionaryVector = null;
return proto[Symbol.toStringTag] = 'Dictionary';
})(Dictionary.prototype);
}
diff --git a/js/src/util/args.ts b/js/src/util/args.ts
index 048f662..ca6f638 100644
--- a/js/src/util/args.ts
+++ b/js/src/util/args.ts
@@ -132,14 +132,14 @@ function _selectFieldArgs<T extends { [key: string]: DataType }>(vals: any[], re
while (++idx < len) {
val = vals[idx];
if (val instanceof Column && (values[++valueIndex] = val)) {
- fields[++fieldIndex] = val.field.clone(keys[idx], val.type, val.nullCount > 0);
+ fields[++fieldIndex] = val.field.clone(keys[idx], val.type, true);
} else {
({ [idx]: field = idx } = keys);
if (val instanceof DataType && (values[++valueIndex] = val)) {
- fields[++fieldIndex] = Field.new(field, val as DataType) as Field<T[keyof T]>;
+ fields[++fieldIndex] = Field.new(field, val as DataType, true) as Field<T[keyof T]>;
} else if (val && val.type && (values[++valueIndex] = val)) {
val instanceof Data && (values[valueIndex] = val = Vector.new(val) as Vector);
- fields[++fieldIndex] = Field.new(field, val.type, val.nullCount > 0) as Field<T[keyof T]>;
+ fields[++fieldIndex] = Field.new(field, val.type, true) as Field<T[keyof T]>;
}
}
}
diff --git a/js/src/vector/chunked.ts b/js/src/vector/chunked.ts
index 882f934..5a0a483 100644
--- a/js/src/vector/chunked.ts
+++ b/js/src/vector/chunked.ts
@@ -19,13 +19,13 @@ import { Data } from '../data';
import { Field } from '../schema';
import { clampRange } from '../util/vector';
import { DataType, Dictionary } from '../type';
+import { selectChunkArgs } from '../util/args';
import { DictionaryVector } from './dictionary';
import { AbstractVector, Vector } from '../vector';
-import { selectChunkArgs } from '../util/args';
import { Clonable, Sliceable, Applicative } from '../vector';
/** @ignore */
-type ChunkedDict<T extends DataType> = T extends Dictionary ? T['dictionaryVector'] : null | never;
+type ChunkedDict<T extends DataType> = T extends Dictionary ? Vector<T['dictionary']> : null | never;
/** @ignore */
type ChunkedKeys<T extends DataType> = T extends Dictionary ? Vector<T['indices']> | Chunked<T['indices']> : null | never;
@@ -105,7 +105,7 @@ export class Chunked<T extends DataType = any>
}
public get dictionary(): ChunkedDict<T> | null {
if (DataType.isDictionary(this._type)) {
- return (<any> this._type.dictionaryVector) as ChunkedDict<T>;
+ return this._chunks[this._chunks.length - 1].data.dictionary as ChunkedDict<T>;
}
return null;
}
diff --git a/js/src/vector/date.ts b/js/src/vector/date.ts
index 4e55562..4b28dee 100644
--- a/js/src/vector/date.ts
+++ b/js/src/vector/date.ts
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+import { DateUnit } from '../enum';
import { Chunked } from './chunked';
import { BaseVector } from './base';
import { VectorType as V } from '../interfaces';
@@ -24,37 +25,27 @@ import { VectorBuilderOptionsAsync } from './index';
import { Date_, DateDay, DateMillisecond } from '../type';
/** @ignore */
+type FromArgs<T extends Date_> = [Iterable<Date>, T['unit']];
+
+/** @ignore */
export class DateVector<T extends Date_ = Date_> extends BaseVector<T> {
- public static from<T extends Date_ = DateMillisecond, TNull = any>(input: Iterable<Date | TNull>): V<T>;
- public static from<T extends Date_ = DateMillisecond, TNull = any>(input: AsyncIterable<Date | TNull>): Promise<V<T>>;
- public static from<T extends Date_ = DateMillisecond, TNull = any>(input: VectorBuilderOptions<T, TNull>): Chunked<T>;
- public static from<T extends Date_ = DateMillisecond, TNull = any>(input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>;
+ public static from<T extends DateUnit.DAY>(...args: FromArgs<DateDay>): V<DateDay>;
+ public static from<T extends DateUnit.MILLISECOND>(...args: FromArgs<DateMillisecond>): V<DateMillisecond>;
+ public static from<T extends Date_, TNull = any>(input: Iterable<Date | TNull>): V<T>;
+ public static from<T extends Date_, TNull = any>(input: AsyncIterable<Date | TNull>): Promise<V<T>>;
+ public static from<T extends Date_, TNull = any>(input: VectorBuilderOptions<T, TNull>): Chunked<T>;
+ public static from<T extends Date_, TNull = any>(input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>;
/** @nocollapse */
- public static from<T extends Date_ = DateMillisecond, TNull = any>(input: Iterable<Date | TNull> | AsyncIterable<Date | TNull> | VectorBuilderOptions<T, TNull> | VectorBuilderOptionsAsync<T, TNull>) {
- return vectorFromValuesWithType(() => new DateMillisecond() as T, input);
+ public static from<T extends Date_, TNull = any>(...args: FromArgs<T> | [Iterable<Date | TNull> | AsyncIterable<Date | TNull> | VectorBuilderOptions<T, TNull> | VectorBuilderOptionsAsync<T, TNull>]) {
+ if (args.length === 2) {
+ return vectorFromValuesWithType(() => args[1] === DateUnit.DAY ? new DateDay() : new DateMillisecond() as T, args[0]);
+ }
+ return vectorFromValuesWithType(() => new DateMillisecond() as T, args[0]);
}
}
/** @ignore */
-export class DateDayVector extends DateVector<DateDay> {
- public static from<TNull = any>(input: Iterable<Date | TNull>): DateDayVector;
- public static from<TNull = any>(input: AsyncIterable<Date | TNull>): Promise<DateDayVector>;
- public static from<TNull = any>(input: VectorBuilderOptions<DateDay, TNull>): Chunked<DateDay>;
- public static from<TNull = any>(input: VectorBuilderOptionsAsync<DateDay, TNull>): Promise<Chunked<DateDay>>;
- /** @nocollapse */
- public static from<TNull = any>(input: Iterable<Date | TNull> | AsyncIterable<Date | TNull> | VectorBuilderOptions<DateDay, TNull> | VectorBuilderOptionsAsync<DateDay, TNull>) {
- return vectorFromValuesWithType(() => new DateDay(), input);
- }
-}
+export class DateDayVector extends DateVector<DateDay> {}
/** @ignore */
-export class DateMillisecondVector extends DateVector<DateMillisecond> {
- public static from<TNull = any>(input: Iterable<Date | TNull>): DateMillisecondVector;
- public static from<TNull = any>(input: AsyncIterable<Date | TNull>): Promise<DateMillisecondVector>;
- public static from<TNull = any>(input: VectorBuilderOptions<DateMillisecond, TNull>): Chunked<DateMillisecond>;
- public static from<TNull = any>(input: VectorBuilderOptionsAsync<DateMillisecond, TNull>): Promise<Chunked<DateMillisecond>>;
- /** @nocollapse */
- public static from<TNull = any>(input: Iterable<Date | TNull> | AsyncIterable<Date | TNull> | VectorBuilderOptions<DateMillisecond, TNull> | VectorBuilderOptionsAsync<DateMillisecond, TNull>) {
- return vectorFromValuesWithType(() => new DateMillisecond(), input);
- }
-}
+export class DateMillisecondVector extends DateVector<DateMillisecond> {}
diff --git a/js/src/vector/dictionary.ts b/js/src/vector/dictionary.ts
index 5556fda..4b39dbe 100644
--- a/js/src/vector/dictionary.ts
+++ b/js/src/vector/dictionary.ts
@@ -25,19 +25,19 @@ import { VectorBuilderOptionsAsync } from './index';
import { DataType, Dictionary, TKeys } from '../type';
/** @ignore */
-type DictionaryFromArgs<T extends DataType = any, TKey extends TKeys = TKeys> = [Vector<T>, TKey, ArrayLike<number> | TKey['TArray']];
+type FromArgs<T extends DataType = any, TKey extends TKeys = TKeys> = [Vector<T>, TKey, ArrayLike<number> | TKey['TArray']];
/** @ignore */
export class DictionaryVector<T extends DataType = any, TKey extends TKeys = TKeys> extends BaseVector<Dictionary<T, TKey>> {
- public static from<T extends DataType = any, TKey extends TKeys = TKeys>(...args: DictionaryFromArgs<T, TKey>): V<Dictionary<T, TKey>>;
+ public static from<T extends DataType = any, TKey extends TKeys = TKeys>(...args: FromArgs<T, TKey>): V<Dictionary<T, TKey>>;
public static from<T extends DataType = any, TKey extends TKeys = TKeys>(input: VectorBuilderOptions<Dictionary<T, TKey>>): Vector<Dictionary<T, TKey>>;
public static from<T extends DataType = any, TKey extends TKeys = TKeys>(input: VectorBuilderOptionsAsync<Dictionary<T, TKey>>): Promise<Vector<Dictionary<T, TKey>>>;
/** @nocollapse */
public static from<T extends DataType = any, TKey extends TKeys = TKeys>(...args: any[]) {
if (args.length === 3) {
- const [values, indices, keys] = args as DictionaryFromArgs<T, TKey>;
- const type = new Dictionary(values.type, indices, null, null, values);
- return Vector.new(Data.Dictionary(type, 0, keys.length, 0, null, keys));
+ const [values, indices, keys] = args as FromArgs<T, TKey>;
+ const type = new Dictionary(values.type, indices, null, null);
+ return Vector.new(Data.Dictionary(type, 0, keys.length, 0, null, keys, values));
}
return vectorFromValuesWithType(() => args[0].type, args[0]);
}
@@ -49,7 +49,7 @@ export class DictionaryVector<T extends DataType = any, TKey extends TKeys = TKe
public readonly indices: V<TKey>;
- public get dictionary() { return this.data.type.dictionaryVector; }
+ public get dictionary() { return <Vector<T>> this.data.dictionary; }
public reverseLookup(value: T) { return this.dictionary.indexOf(value); }
public getKey(idx: number): TKey['TValue'] | null { return this.indices.get(idx); }
public getValue(key: number): T['TValue'] | null { return this.dictionary.get(key); }
diff --git a/js/src/visitor/vectorloader.ts b/js/src/visitor/vectorloader.ts
index e11e59e..3d0ccd6 100644
--- a/js/src/visitor/vectorloader.ts
+++ b/js/src/visitor/vectorloader.ts
@@ -18,6 +18,7 @@
import { Data } from '../data';
import * as type from '../type';
import { Field } from '../schema';
+import { Vector } from '../vector';
import { DataType } from '../type';
import { Visitor } from '../visitor';
import { packBools } from '../util/bit';
@@ -40,11 +41,13 @@ export class VectorLoader extends Visitor {
private nodesIndex: number = -1;
private buffers: BufferRegion[];
private buffersIndex: number = -1;
- constructor(bytes: Uint8Array, nodes: FieldNode[], buffers: BufferRegion[]) {
+ private dictionaries: Map<number, Vector<any>>;
+ constructor(bytes: Uint8Array, nodes: FieldNode[], buffers: BufferRegion[], dictionaries: Map<number, Vector<any>>) {
super();
this.bytes = bytes;
this.nodes = nodes;
this.buffers = buffers;
+ this.dictionaries = dictionaries;
}
public visit<T extends DataType>(node: Field<T> | T): Data<T> {
@@ -67,7 +70,7 @@ export class VectorLoader extends Visitor {
public visitUnion <T extends type.Union> (type: T ) { return type.mode === UnionMode.Sparse ? this.visitSparseUnion(type as type.SparseUnion) : this.visitDenseUnion(type as type.DenseUnion); }
public visitDenseUnion <T extends type.DenseUnion> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Union(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readTypeIds(type), this.readOffsets(type), this.visitMany(type.children)); }
public visitSparseUnion <T extends type.SparseUnion> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Union(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readTypeIds(type), this.visitMany(type.children)); }
- public visitDictionary <T extends type.Dictionary> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Dictionary(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type.indices)); }
+ public visitDictionary <T extends type.Dictionary> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Dictionary(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type.indices), this.readDictionary(type)); }
public visitInterval <T extends type.Interval> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Interval(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); }
public visitFixedSizeList <T extends type.FixedSizeList> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.FixedSizeList(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.visit(type.children[0])); }
public visitMap <T extends type.Map_> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Map(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.visitMany(type.children)); }
@@ -82,13 +85,16 @@ export class VectorLoader extends Visitor {
protected readData<T extends DataType>(_type: T, { length, offset } = this.nextBufferRange()) {
return this.bytes.subarray(offset, offset + length);
}
+ protected readDictionary<T extends type.Dictionary>(type: T): Vector<T['dictionary']> {
+ return this.dictionaries.get(type.id)!;
+ }
}
/** @ignore */
export class JSONVectorLoader extends VectorLoader {
private sources: any[][];
- constructor(sources: any[][], nodes: FieldNode[], buffers: BufferRegion[]) {
- super(new Uint8Array(0), nodes, buffers);
+ constructor(sources: any[][], nodes: FieldNode[], buffers: BufferRegion[], dictionaries: Map<number, Vector<any>>) {
+ super(new Uint8Array(0), nodes, buffers, dictionaries);
this.sources = sources;
}
protected readNullBitmap<T extends DataType>(_type: T, nullCount: number, { offset } = this.nextBufferRange()) {
diff --git a/js/test/data/tables.ts b/js/test/data/tables.ts
index beb90b9..6ce2c86 100644
--- a/js/test/data/tables.ts
+++ b/js/test/data/tables.ts
@@ -43,7 +43,6 @@ export function* generateRandomTables(batchLengths = [1000, 2000, 3000], minCols
let names = allNames.slice(0, numCols);
let types = names.map((fn) => vecs[fn](0).vector.type);
- types.forEach((t) => t.dictionaryVector && (t.dictionaryVector = null));
let schema = new Schema(names.map((name, i) => new Field(name, types[i])));
yield generate.table(batchLengths, schema).table;
diff --git a/js/test/generate-test-data.ts b/js/test/generate-test-data.ts
index dede6a0..ff6056d 100644
--- a/js/test/generate-test-data.ts
+++ b/js/test/generate-test-data.ts
@@ -44,6 +44,8 @@ import {
DateUnit, TimeUnit, UnionMode
} from './Arrow';
+type TKeys = Int8 | Int16 | Int32 | Uint8 | Uint16 | Uint32;
+
interface TestDataVectorGenerator extends Visitor {
visit<T extends Null> (type: T, length?: number): GeneratedVector<V<T>>;
@@ -167,6 +169,7 @@ export const table = (lengths = [100], schema: Schema = new Schema(defaultRecord
return { rows, cols, keys, rowBatches, colBatches, keyBatches, table: new Table(schema, generated.map(({ recordBatch }) => recordBatch)) };
};
+
export const recordBatch = (length = 100, schema: Schema = new Schema(defaultRecordBatchChildren())): GeneratedRecordBatch => {
const generated = schema.fields.map((f) => vectorGenerator.visit(f.type, length));
@@ -181,6 +184,7 @@ export const recordBatch = (length = 100, schema: Schema = new Schema(defaultRec
return { rows, cols, keys, recordBatch: new RecordBatch(schema, length, vecs) };
};
+
export const null_ = (length = 100) => vectorGenerator.visit(new Null(), length);
export const bool = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Bool(), length, nullCount);
export const int8 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Int8(), length, nullCount);
@@ -209,14 +213,14 @@ export const timeMicrosecond = (length = 100, nullCount = length * 0.2 | 0) => v
export const timeNanosecond = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new TimeNanosecond(), length, nullCount);
export const decimal = (length = 100, nullCount = length * 0.2 | 0, scale = 2, precision = 9) => vectorGenerator.visit(new Decimal(scale, precision), length, nullCount);
export const list = (length = 100, nullCount = length * 0.2 | 0, child = defaultListChild) => vectorGenerator.visit(new List(child), length, nullCount);
-export const struct = (length = 100, nullCount = length * 0.2 | 0, children: Field[] = defaultStructChildren()) => vectorGenerator.visit(new Struct(children), length, nullCount);
+export const struct = <T extends { [key: string]: DataType } = any>(length = 100, nullCount = length * 0.2 | 0, children: Field<T[keyof T]>[] = <any> defaultStructChildren()) => vectorGenerator.visit(new Struct<T>(children), length, nullCount);
export const denseUnion = (length = 100, nullCount = length * 0.2 | 0, children: Field[] = defaultUnionChildren()) => vectorGenerator.visit(new DenseUnion(children.map((f) => f.typeId), children), length, nullCount);
export const sparseUnion = (length = 100, nullCount = length * 0.2 | 0, children: Field[] = defaultUnionChildren()) => vectorGenerator.visit(new SparseUnion(children.map((f) => f.typeId), children), length, nullCount);
-export const dictionary = (length = 100, nullCount = length * 0.2 | 0, dict: DataType = new Utf8(), keys: Int = new Int32()) => vectorGenerator.visit(new Dictionary(dict, <any> keys), length, nullCount);
+export const dictionary = <T extends DataType = Utf8, TKey extends TKeys = Int32> (length = 100, nullCount = length * 0.2 | 0, dict: T = <any> new Utf8(), keys: TKey = <any> new Int32()) => vectorGenerator.visit(new Dictionary(dict, keys), length, nullCount);
export const intervalDayTime = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new IntervalDayTime(), length, nullCount);
export const intervalYearMonth = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new IntervalYearMonth(), length, nullCount);
export const fixedSizeList = (length = 100, nullCount = length * 0.2 | 0, listSize = 2, child = defaultListChild) => vectorGenerator.visit(new FixedSizeList(listSize, child), length, nullCount);
-export const map = (length = 100, nullCount = length * 0.2 | 0, children: Field[] = defaultStructChildren()) => vectorGenerator.visit(new Map_(children), length, nullCount);
+export const map = <T extends { [key: string]: DataType } = any>(length = 100, nullCount = length * 0.2 | 0, children: Field<T[keyof T]>[] = <any> defaultStructChildren()) => vectorGenerator.visit(new Map_<T>(children), length, nullCount);
export const vecs = {
null_, bool, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, dateDay, dateMillisecond, timestampSecond, timestampMillisecond, timestampMicrosecond, timestampNanosecond, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, decimal, list, struct, denseUnion, sparseUnion, dictionary, intervalDayTime, intervalYearMonth, fixedSizeList, map
@@ -410,11 +414,14 @@ function generateFixedSizeList<T extends FixedSizeList>(this: TestDataVectorGene
function generateDictionary<T extends Dictionary>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0, dictionary = this.visit(type.dictionary, length, 0)): GeneratedVector<V<T>> {
- const dict = type.dictionaryVector ? type.dictionaryVector : dictionary.vector;
- const vals = type.dictionaryVector ? (<any> type).dictVals : dictionary.values;
+ const t = <any> type;
+ const currValues = t.dictionaryValues;
+ const hasDict = t.dictionaryVector && t.dictionaryVector.length > 0;
+ const dict = hasDict ? t.dictionaryVector.concat(dictionary.vector) : dictionary.vector;
+ const vals = hasDict ? (() => [...currValues(), ...dictionary.values()]) : dictionary.values;
const maxIdx = dict.length - 1;
- const keys = new type.indices.ArrayType(length) as any;
+ const keys = new t.indices.ArrayType(length);
const nullBitmap = createBitmap(length, nullCount);
const values = memoize(() => {
@@ -430,10 +437,10 @@ function generateDictionary<T extends Dictionary>(this: TestDataVectorGenerator,
keys[i] = !valid ? 0 : rand() * maxIdx | 0;
});
- type.dictionaryVector = dict;
- (<any> type).dictVals = vals;
+ t.dictionaryVector = dict;
+ t.dictionaryValues = vals;
- return { values, keys, vector: Vector.new(Data.Dictionary(type, 0, length, nullCount, nullBitmap, keys)) };
+ return { values, keys, vector: Vector.new(Data.Dictionary(type, 0, length, nullCount, nullBitmap, keys, dict)) };
}
function generateUnion<T extends Union>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0, children?: GeneratedVector<any>[]): GeneratedVector<V<T>> {
diff --git a/js/test/unit/ipc/message-reader-tests.ts b/js/test/unit/ipc/message-reader-tests.ts
index 2c49a06..7320d6e 100644
--- a/js/test/unit/ipc/message-reader-tests.ts
+++ b/js/test/unit/ipc/message-reader-tests.ts
@@ -22,24 +22,17 @@ import {
} from '../../data/tables';
import { ArrowIOTestHelper } from './helpers';
-
-import {
- Chunked,
- MessageReader,
- AsyncMessageReader
-} from '../../Arrow';
+import { MessageReader, AsyncMessageReader } from '../../Arrow';
for (const table of generateRandomTables([10, 20, 30])) {
const io = ArrowIOTestHelper.stream(table);
const name = `[\n ${table.schema.fields.join(',\n ')}\n]`;
- let numMessages = /* schema message */ 1 + table.chunks.length;
-
- // count dictionary chunks
- table.schema.dictionaryFields.forEach((fields) => {
- const vector = fields[0].type.dictionaryVector as Chunked;
- numMessages += (vector.chunks ? vector.chunks.length : 1);
- });
+ const numMessages = table.chunks.reduce((numMessages, batch) => {
+ return numMessages +
+ /* recordBatch message */ 1 +
+ /* dictionary messages */ batch.dictionaries.size;
+ }, /* schema message */ 1);
const validate = validateMessageReader.bind(0, numMessages);
const validateAsync = validateAsyncMessageReader.bind(0, numMessages);
diff --git a/js/test/unit/ipc/writer/stream-writer-tests.ts b/js/test/unit/ipc/writer/stream-writer-tests.ts
index 9bafb56..ec6f581 100644
--- a/js/test/unit/ipc/writer/stream-writer-tests.ts
+++ b/js/test/unit/ipc/writer/stream-writer-tests.ts
@@ -20,11 +20,20 @@ import {
generateDictionaryTables
} from '../../../data/tables';
+import * as generate from '../../../generate-test-data';
import { validateRecordBatchIterator } from '../validate';
-import { Table, RecordBatchReader, RecordBatchStreamWriter } from '../../../Arrow';
+import { DictionaryVector, Dictionary, Uint32, Int32 } from '../../../Arrow';
+import { Table, Schema, Chunked, Builder, RecordBatch, RecordBatchReader, RecordBatchStreamWriter } from '../../../Arrow';
describe('RecordBatchStreamWriter', () => {
+ (() => {
+ const type = generate.sparseUnion(0, 0).vector.type;
+ const schema = Schema.new({ 'dictSparseUnion': type });
+ const table = generate.table([10, 20, 30], schema).table;
+ testStreamWriter(table, `[${table.schema.fields.join(', ')}]`);
+ })();
+
for (const table of generateRandomTables([10, 20, 30])) {
testStreamWriter(table, `[${table.schema.fields.join(', ')}]`);
}
@@ -33,7 +42,7 @@ describe('RecordBatchStreamWriter', () => {
testStreamWriter(table, `${table.schema.fields[0]}`);
}
- test(`should write multiple tables to the same output stream`, async () => {
+ it(`should write multiple tables to the same output stream`, async () => {
const tables = [] as Table[];
const writer = new RecordBatchStreamWriter({ autoDestroy: false });
const validate = (async () => {
@@ -55,6 +64,38 @@ describe('RecordBatchStreamWriter', () => {
writer.close();
await validate;
});
+
+ it('should write delta dictionary batches', async () => {
+
+ const name = 'dictionary_encoded_uint32';
+ const chunks: DictionaryVector<Uint32, Int32>[] = [];
+ const {
+ vector: sourceVector, values: sourceValues,
+ } = generate.dictionary(1000, 20, new Uint32(), new Int32());
+
+ const writer = RecordBatchStreamWriter.writeAll((function* () {
+ const transform = Builder.throughIterable({
+ type: sourceVector.type, nullValues: [null],
+ queueingStrategy: 'count', highWaterMark: 50,
+ });
+ for (const chunk of transform(sourceValues())) {
+ chunks.push(chunk);
+ yield RecordBatch.new({ [name]: chunk });
+ }
+ })());
+
+ expect(Chunked.concat(chunks)).toEqualVector(sourceVector);
+
+ type T = { [name]: Dictionary<Uint32, Int32> };
+ const sourceTable = Table.new({ [name]: sourceVector });
+ const resultTable = await Table.from<T>(writer.toUint8Array());
+
+ const { dictionary } = resultTable.getColumn(name);
+
+ expect(resultTable).toEqualTable(sourceTable);
+ expect((dictionary as Chunked)).toBeInstanceOf(Chunked);
+ expect((dictionary as Chunked).chunks.length).toBe(20);
+ });
});
function testStreamWriter(table: Table, name: string) {
diff --git a/js/test/unit/table-tests.ts b/js/test/unit/table-tests.ts
index b8cf690..22c50a7 100644
--- a/js/test/unit/table-tests.ts
+++ b/js/test/unit/table-tests.ts
@@ -123,7 +123,7 @@ describe(`Table`, () => {
let i32 = Column.new('i32', Data.Int(new Int32(), 0, i32s.length, 0, null, i32s));
expect(i32.name).toBe('i32');
expect(i32.length).toBe(i32s.length);
- expect(i32.nullable).toBe(false);
+ expect(i32.nullable).toBe(true);
expect(i32.nullCount).toBe(0);
const table = Table.new(i32);
@@ -131,7 +131,7 @@ describe(`Table`, () => {
expect(i32.name).toBe('i32');
expect(i32.length).toBe(i32s.length);
- expect(i32.nullable).toBe(false);
+ expect(i32.nullable).toBe(true);
expect(i32.nullCount).toBe(0);
expect(i32).toEqualVector(Int32Vector.from(i32s));
@@ -148,8 +148,8 @@ describe(`Table`, () => {
expect(f32.name).toBe('f32');
expect(i32.length).toBe(i32s.length);
expect(f32.length).toBe(f32s.length);
- expect(i32.nullable).toBe(false);
- expect(f32.nullable).toBe(false);
+ expect(i32.nullable).toBe(true);
+ expect(f32.nullable).toBe(true);
expect(i32.nullCount).toBe(0);
expect(f32.nullCount).toBe(0);
@@ -161,8 +161,8 @@ describe(`Table`, () => {
expect(f32.name).toBe('f32');
expect(i32.length).toBe(i32s.length);
expect(f32.length).toBe(f32s.length);
- expect(i32.nullable).toBe(false);
- expect(f32.nullable).toBe(false);
+ expect(i32.nullable).toBe(true);
+ expect(f32.nullable).toBe(true);
expect(i32.nullCount).toBe(0);
expect(f32.nullCount).toBe(0);
@@ -182,8 +182,8 @@ describe(`Table`, () => {
expect(f32.name).toBe('f32');
expect(i32.length).toBe(i32s.length);
expect(f32.length).toBe(f32s.length);
- expect(i32.nullable).toBe(false);
- expect(f32.nullable).toBe(false);
+ expect(i32.nullable).toBe(true);
+ expect(f32.nullable).toBe(true);
expect(i32.nullCount).toBe(0);
expect(f32.nullCount).toBe(0);
@@ -195,7 +195,7 @@ describe(`Table`, () => {
expect(f32.name).toBe('f32');
expect(i32.length).toBe(i32s.length);
expect(f32.length).toBe(i32s.length); // new length should be the same as the longest sibling
- expect(i32.nullable).toBe(false);
+ expect(i32.nullable).toBe(true);
expect(f32.nullable).toBe(true); // true, with 12 additional nulls
expect(i32.nullCount).toBe(0);
expect(f32.nullCount).toBe(i32s.length - f32s.length);
@@ -221,8 +221,8 @@ describe(`Table`, () => {
expect(f32.name).toBe('f32');
expect(i32.length).toBe(i32s.length);
expect(f32.length).toBe(f32s.length);
- expect(i32.nullable).toBe(false);
- expect(f32.nullable).toBe(false);
+ expect(i32.nullable).toBe(true);
+ expect(f32.nullable).toBe(true);
expect(i32.nullCount).toBe(0);
expect(f32.nullCount).toBe(0);
@@ -234,7 +234,7 @@ describe(`Table`, () => {
expect(f32.name).toBe('f32Renamed');
expect(i32.length).toBe(i32s.length);
expect(f32.length).toBe(i32s.length); // new length should be the same as the longest sibling
- expect(i32.nullable).toBe(false);
+ expect(i32.nullable).toBe(true);
expect(f32.nullable).toBe(true); // true, with 4 additional nulls
expect(i32.nullCount).toBe(0);
expect(f32.nullCount).toBe(i32s.length - f32s.length);
diff --git a/js/test/unit/vector/vector-tests.ts b/js/test/unit/vector/vector-tests.ts
index 3291c72..91c4026 100644
--- a/js/test/unit/vector/vector-tests.ts
+++ b/js/test/unit/vector/vector-tests.ts
@@ -74,8 +74,8 @@ describe(`DictionaryVector`, () => {
const nullBitmap = util.packBools(validity);
const nullCount = validity.reduce((acc, d) => acc + (d ? 0 : 1), 0);
const values = Array.from(indices).map((d, i) => validity[i] ? dictionary[d] : null);
- const type = new Dictionary(dictionary_vec.type, new Int32(), null, null, dictionary_vec);
- const vector = Vector.new(Data.Dictionary(type, 0, indices.length, nullCount, nullBitmap, indices));
+ const type = new Dictionary(dictionary_vec.type, new Int32(), null, null);
+ const vector = Vector.new(Data.Dictionary(type, 0, indices.length, nullCount, nullBitmap, indices, dictionary_vec));
basicVectorTests(vector, values, ['abc', '123']);
describe(`sliced`, () => {