You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2019/03/06 11:31:38 UTC
[arrow] branch master updated: ARROW-4781: [JS] Ensure empty table
column data have 0-length buffers
This is an automated email from the ASF dual-hosted git repository.
kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new a7ec79e ARROW-4781: [JS] Ensure empty table column data have 0-length buffers
a7ec79e is described below
commit a7ec79ea4bab4693a79acc288d8e4a4ab88f2c7a
Author: ptaylor <pa...@me.com>
AuthorDate: Wed Mar 6 12:31:20 2019 +0100
ARROW-4781: [JS] Ensure empty table column data have 0-length buffers
Closes https://issues.apache.org/jira/browse/ARROW-4781
Author: ptaylor <pa...@me.com>
Closes #3819 from trxcllnt/js/init-empty-table-data and squashes the following commits:
ece6cb2d <ptaylor> protect Data.new from closure
1cfc811b <ptaylor> ensure empty table column data have 0-length buffers
---
js/src/data.ts | 29 ++++++++++++++++++++++++++++-
js/src/table.ts | 2 +-
js/src/util/recordbatch.ts | 4 ++--
js/test/unit/table-tests.ts | 13 +++++++++++++
4 files changed, 44 insertions(+), 4 deletions(-)
diff --git a/js/src/data.ts b/js/src/data.ts
index de7cd7e..3f64567 100644
--- a/js/src/data.ts
+++ b/js/src/data.ts
@@ -174,6 +174,32 @@ export class Data<T extends DataType = DataType> {
// Convenience methods for creating Data instances for each of the Arrow Vector types
//
/** @nocollapse */
+ public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[]): Data<T> {
+ if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; }
+ switch (type.typeId) {
+ case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length, nullCount || 0, buffers[2]) as Data<T>;
+ case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[2], buffers[1] || []) as Data<T>;
+ case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[2], buffers[1] || []) as Data<T>;
+ case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[2], buffers[1] || []) as Data<T>;
+ case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[2], buffers[1] || []) as Data<T>;
+ case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[2], buffers[1] || []) as Data<T>;
+ case Type.Date: return <unknown> Data.Date( <unknown> type as Date_, offset, length, nullCount || 0, buffers[2], buffers[1] || []) as Data<T>;
+ case Type.Time: return <unknown> Data.Time( <unknown> type as Time, offset, length, nullCount || 0, buffers[2], buffers[1] || []) as Data<T>;
+ case Type.Timestamp: return <unknown> Data.Timestamp( <unknown> type as Timestamp, offset, length, nullCount || 0, buffers[2], buffers[1] || []) as Data<T>;
+ case Type.Interval: return <unknown> Data.Interval( <unknown> type as Interval, offset, length, nullCount || 0, buffers[2], buffers[1] || []) as Data<T>;
+ case Type.FixedSizeBinary: return <unknown> Data.FixedSizeBinary( <unknown> type as FixedSizeBinary, offset, length, nullCount || 0, buffers[2], buffers[1] || []) as Data<T>;
+ case Type.Binary: return <unknown> Data.Binary( <unknown> type as Binary, offset, length, nullCount || 0, buffers[2], buffers[0] || [], buffers[1] || []) as Data<T>;
+ case Type.Utf8: return <unknown> Data.Utf8( <unknown> type as Utf8, offset, length, nullCount || 0, buffers[2], buffers[0] || [], buffers[1] || []) as Data<T>;
+ case Type.List: return <unknown> Data.List( <unknown> type as List, offset, length, nullCount || 0, buffers[2], buffers[0] || [], (childData || [])[0]) as Data<T>;
+ case Type.FixedSizeList: return <unknown> Data.FixedSizeList( <unknown> type as FixedSizeList, offset, length, nullCount || 0, buffers[2], (childData || [])[0]) as Data<T>;
+ case Type.Struct: return <unknown> Data.Struct( <unknown> type as Struct, offset, length, nullCount || 0, buffers[2], childData || []) as Data<T>;
+ case Type.Map: return <unknown> Data.Map( <unknown> type as Map_, offset, length, nullCount || 0, buffers[2], childData || []) as Data<T>;
+ case Type.Union: return <unknown> Data.Union( <unknown> type as Union, offset, length, nullCount || 0, buffers[2], buffers[3] || [], buffers[1] || childData, childData) as Data<T>;
+ }
+ throw new Error(`Unrecognized typeId ${type.typeId}`);
+ }
+
+ /** @nocollapse */
public static Null<T extends Null>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, _data?: NullBuffer) {
return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)]);
}
@@ -241,8 +267,9 @@ export class Data<T extends DataType = DataType> {
public static Map<T extends Map_>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, children: (Data | Vector)[]) {
return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], children);
}
- public static Union<T extends SparseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, children: (Data | Vector)[]): Data<T>;
+ public static Union<T extends SparseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, children: (Data | Vector)[], _?: any): Data<T>;
public static Union<T extends DenseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, valueOffsets: ValueOffsetsBuffer, children: (Data | Vector)[]): Data<T>;
+ public static Union<T extends Union>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, valueOffsetsOrChildren: ValueOffsetsBuffer | (Data | Vector)[], children?: (Data | Vector)[]): Data<T>;
/** @nocollapse */
public static Union<T extends Union>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, valueOffsetsOrChildren: ValueOffsetsBuffer | (Data | Vector)[], children?: (Data | Vector)[]) {
const buffers = <unknown> [
diff --git a/js/src/table.ts b/js/src/table.ts
index 89f6840..738eb56 100644
--- a/js/src/table.ts
+++ b/js/src/table.ts
@@ -180,7 +180,7 @@ export class Table<T extends { [key: string]: DataType } = any>
}
if (!chunks[0]) {
- chunks[0] = new RecordBatch(schema, 0, schema.fields.map((f) => new Data(f.type, 0, 0)));
+ chunks[0] = new RecordBatch(schema, 0, schema.fields.map((f) => Data.new(f.type, 0, 0)));
}
super(new Struct<T>(schema.fields), chunks);
diff --git a/js/src/util/recordbatch.ts b/js/src/util/recordbatch.ts
index 4551d0c..2828a6e 100644
--- a/js/src/util/recordbatch.ts
+++ b/js/src/util/recordbatch.ts
@@ -46,7 +46,7 @@ export function ensureSameLengthData<T extends { [key: string]: DataType } = any
} else {
(field = fields[i]).nullable || (fields[i] = fields[i].clone({ nullable: true }) as Field<T[keyof T]>);
batchData[i] = data ? data._changeLengthAndBackfillNullBitmap(batchLength)
- : new Data(field.type, 0, batchLength, batchLength, nullBufs(bitmapLength)) as Data<T[keyof T]>;
+ : Data.new(field.type, 0, batchLength, batchLength, nullBufs(bitmapLength)) as Data<T[keyof T]>;
}
}
return [new Schema<T>(fields), batchLength, batchData] as [Schema<T>, number, Data<T[keyof T]>[]];
@@ -111,7 +111,7 @@ function distributeChildData<T extends { [key: string]: DataType } = any>(fields
} else {
(field = fields[i]).nullable || (fields[i] = field.clone({ nullable: true }) as Field<T[keyof T]>);
childData[i] = data ? data._changeLengthAndBackfillNullBitmap(batchLength)
- : new Data(field.type, 0, batchLength, batchLength, nullBufs(bitmapLength)) as Data<T[keyof T]>;
+ : Data.new(field.type, 0, batchLength, batchLength, nullBufs(bitmapLength)) as Data<T[keyof T]>;
}
}
return childData;
diff --git a/js/test/unit/table-tests.ts b/js/test/unit/table-tests.ts
index e5aade3..b8cf690 100644
--- a/js/test/unit/table-tests.ts
+++ b/js/test/unit/table-tests.ts
@@ -103,6 +103,19 @@ describe(`Table`, () => {
return arr;
};
+ test(`creates an empty Table with Columns`, () => {
+ let i32 = Column.new('i32', Data.new(new Int32(), 0, 0));
+ let f32 = Column.new('f32', Data.new(new Float32(), 0, 0));
+ const table = Table.new(i32, f32);
+ i32 = table.getColumn('i32')!;
+ f32 = table.getColumn('f32')!;
+ expect(table.length).toBe(0);
+ expect(i32.length).toBe(0);
+ expect(f32.length).toBe(0);
+ expect(i32.toArray()).toBeInstanceOf(Int32Array);
+ expect(f32.toArray()).toBeInstanceOf(Float32Array);
+ });
+
test(`creates a new Table from a Column`, () => {
const i32s = new Int32Array(arange(new Array<number>(10)));