You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/04/21 21:43:05 UTC
arrow git commit: ARROW-869 [JS] Rename directory to js/
Repository: arrow
Updated Branches:
refs/heads/master 6c352e205 -> 6867e93cc
ARROW-869 [JS] Rename directory to js/
Author: Brian Hulette <br...@ccri.com>
Closes #578 from TheNeuralBit/js-rename and squashes the following commits:
62244d0 [Brian Hulette] moved javascript/ to js/
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/6867e93c
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/6867e93c
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/6867e93c
Branch: refs/heads/master
Commit: 6867e93cc78bffdb42b38ad9581999b567de28d6
Parents: 6c352e2
Author: Brian Hulette <br...@ccri.com>
Authored: Fri Apr 21 17:42:54 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Fri Apr 21 17:42:54 2017 -0400
----------------------------------------------------------------------
javascript/.gitignore | 4 -
javascript/README.md | 50 -----
javascript/bin/arrow2csv.js | 47 -----
javascript/bin/arrow_schema.js | 25 ---
javascript/examples/read_file.html | 79 --------
javascript/lib/Arrow_generated.d.ts | 5 -
javascript/lib/arrow.ts | 201 -------------------
javascript/lib/bitarray.ts | 55 ------
javascript/lib/types.ts | 328 -------------------------------
javascript/package.json | 19 --
javascript/postinstall.sh | 18 --
javascript/tsconfig.json | 14 --
javascript/webpack.config.js | 21 --
js/.gitignore | 4 +
js/README.md | 50 +++++
js/bin/arrow2csv.js | 47 +++++
js/bin/arrow_schema.js | 25 +++
js/examples/read_file.html | 79 ++++++++
js/lib/Arrow_generated.d.ts | 5 +
js/lib/arrow.ts | 201 +++++++++++++++++++
js/lib/bitarray.ts | 55 ++++++
js/lib/types.ts | 328 +++++++++++++++++++++++++++++++
js/package.json | 19 ++
js/postinstall.sh | 18 ++
js/tsconfig.json | 14 ++
js/webpack.config.js | 21 ++
26 files changed, 866 insertions(+), 866 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/.gitignore
----------------------------------------------------------------------
diff --git a/javascript/.gitignore b/javascript/.gitignore
deleted file mode 100644
index 3b97e3a..0000000
--- a/javascript/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-lib/*_generated.js
-dist
-node_modules
-typings
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/README.md
----------------------------------------------------------------------
diff --git a/javascript/README.md b/javascript/README.md
deleted file mode 100644
index 98ef756..0000000
--- a/javascript/README.md
+++ /dev/null
@@ -1,50 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-### Installation
-
-From this directory, run:
-
-``` bash
-$ npm install # pull dependencies
-$ tsc # build typescript
-$ webpack # bundle for the browser
-```
-
-### Usage
-The library is designed to be used with node.js or in the browser, this repository contains examples of both.
-
-#### Node
-Import the arrow module:
-
-``` js
-var arrow = require("arrow.js");
-```
-
-See [bin/arrow_schema.js](bin/arrow_schema.js) and [bin/arrow2csv.js](bin/arrow2csv.js) for usage examples.
-
-#### Browser
-Include `dist/arrow-bundle.js` in a `<script />` tag:
-``` html
-<script src="arrow-bundle.js"/>
-```
-See [examples/read_file.html](examples/read_file.html) for a usage example - or try it out now at [theneuralbit.github.io/arrow](http://theneuralbit.github.io/arrow)
-
-### API
-##### `arrow.loadSchema(buffer)`
-Returns a JSON representation of the file's Arrow schema.
-
-##### `arrow.loadVectors(buffer)`
-Returns a dictionary of `Vector` objects, one for each column, indexed by the column's name.
-Vector objects have, at minimum, a `get(i)` method and a `length` attribute.
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/bin/arrow2csv.js
----------------------------------------------------------------------
diff --git a/javascript/bin/arrow2csv.js b/javascript/bin/arrow2csv.js
deleted file mode 100755
index 48df2f9..0000000
--- a/javascript/bin/arrow2csv.js
+++ /dev/null
@@ -1,47 +0,0 @@
-#! /usr/bin/env node
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-var fs = require('fs')
-var process = require('process');
-var loadVectors = require('../dist/arrow.js').loadVectors;
-var program = require('commander');
-
-function list (val) {
- return val.split(',');
-}
-
-program
- .version('0.1.0')
- .usage('[options] <file>')
- .option('-s --schema <list>', 'A comma-separated list of column names', list)
- .parse(process.argv);
-
-if (!program.schema) {
- program.outputHelp();
- process.exit(1);
-}
-
-var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
-var vectors = loadVectors(buf);
-
-for (var i = 0; i < vectors[program.schema[0]].length; i += 1|0) {
- console.log(program.schema.map(function (field) {
- return '' + vectors[field].get(i);
- }).join(','));
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/bin/arrow_schema.js
----------------------------------------------------------------------
diff --git a/javascript/bin/arrow_schema.js b/javascript/bin/arrow_schema.js
deleted file mode 100755
index 7044778..0000000
--- a/javascript/bin/arrow_schema.js
+++ /dev/null
@@ -1,25 +0,0 @@
-#! /usr/bin/env node
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-var fs = require('fs');
-var process = require('process');
-var loadSchema = require('../dist/arrow.js').loadSchema;
-
-var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
-console.log(JSON.stringify(loadSchema(buf), null, '\t'));
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/examples/read_file.html
----------------------------------------------------------------------
diff --git a/javascript/examples/read_file.html b/javascript/examples/read_file.html
deleted file mode 100644
index 02b6f08..0000000
--- a/javascript/examples/read_file.html
+++ /dev/null
@@ -1,79 +0,0 @@
-<!DOCTYPE html>
-
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-<html>
- <head>
- <title>arrow.js browser test</title>
- <meta charset="utf-8">
- <style>
-table {
- border-collapse: collapse;
-}
-table, th, td {
- border: 1px solid black;
-}
- </style>
- <script type="text/javascript">
-var reader = new FileReader();
-function addCell (tr, type, name) {
- var td = document.createElement(type)
- td.textContent = name;
- tr.appendChild(td);
-}
-reader.onload = function (evt) {
- var buf = new Uint8Array(evt.target.result);
- var schema = arrow.loadSchema(buf);
- var vectors = arrow.loadVectors(buf);
- var length = vectors[schema[0].name].length;
-
- var thead = document.getElementById("thead");
- var tbody = document.getElementById("tbody");
- var header_row = document.createElement("tr");
-
- schema.forEach(function (d) {
- addCell(header_row, "th", d.name);
- });
-
- thead.appendChild(header_row);
-
- for (var i = 0; i < length; i += 1|0) {
- var tr = document.createElement("tr");
- schema.forEach(function (d) { addCell(tr, "td", vectors[d.name].get(i)); });
- tbody.appendChild(tr);
- }
-}
-
-function handleFiles(files) {
- reader.readAsArrayBuffer(files[0]);
-}
- </script>
- </head>
- <body>
- <input id="arrow-in" type="file" onchange="handleFiles(this.files)" />
- <table>
- <thead id="thead">
- </thead>
- <tbody id="tbody">
- </tbody>
- </table>
- <script type="text/javascript" src="../dist/arrow-bundle.js"></script>
- </body>
-</html>
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/lib/Arrow_generated.d.ts
----------------------------------------------------------------------
diff --git a/javascript/lib/Arrow_generated.d.ts b/javascript/lib/Arrow_generated.d.ts
deleted file mode 100644
index 1f5b454..0000000
--- a/javascript/lib/Arrow_generated.d.ts
+++ /dev/null
@@ -1,5 +0,0 @@
-export var org: {
- apache: {
- arrow: any
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/lib/arrow.ts
----------------------------------------------------------------------
diff --git a/javascript/lib/arrow.ts b/javascript/lib/arrow.ts
deleted file mode 100644
index 0762885..0000000
--- a/javascript/lib/arrow.ts
+++ /dev/null
@@ -1,201 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { flatbuffers } from 'flatbuffers';
-import { org } from './Arrow_generated';
-var arrow = org.apache.arrow;
-import { vectorFromField, Vector } from './types';
-
-export function loadVectors(buf) {
- var fileLength = buf.length, bb, footerLengthOffset, footerLength,
- footerOffset, footer, schema, field, type, type_str, i,
- len, rb_metas, rb_meta, rtrn, recordBatchBlock, recordBatchBlocks = [];
- var vectors : Vector[] = [];
-
- bb = new flatbuffers.ByteBuffer(buf);
-
- footer = _loadFooter(bb);
-
- schema = footer.schema();
-
- for (i = 0, len = schema.fieldsLength(); i < len; i += 1|0) {
- field = schema.fields(i);
- vectors.push(vectorFromField(field));
- }
-
- for (i = 0; i < footer.recordBatchesLength(); i += 1|0) {
- recordBatchBlock = footer.recordBatches(i);
- recordBatchBlocks.push({
- offset: recordBatchBlock.offset(),
- metaDataLength: recordBatchBlock.metaDataLength(),
- bodyLength: recordBatchBlock.bodyLength(),
- })
- }
-
- loadBuffersIntoVectors(recordBatchBlocks, bb, vectors);
- var rtrn : any = {};
- for (var i : any = 0; i < vectors.length; i += 1|0) {
- rtrn[vectors[i].name] = vectors[i]
- }
- return rtrn;
-}
-
-export function loadSchema(buf) {
- var footer = _loadFooter(new flatbuffers.ByteBuffer(buf));
- var schema = footer.schema();
-
- return parseSchema(schema);
-}
-
-function _loadFooter(bb) {
- var fileLength: number = bb.bytes_.length;
-
- if (fileLength < MAGIC.length*2 + 4) {
- console.error("file too small " + fileLength);
- return;
- }
-
- if (!_checkMagic(bb.bytes_, 0)) {
- console.error("missing magic bytes at beginning of file")
- return;
- }
-
- if (!_checkMagic(bb.bytes_, fileLength - MAGIC.length)) {
- console.error("missing magic bytes at end of file")
- return;
- }
-
- var footerLengthOffset: number = fileLength - MAGIC.length - 4;
- bb.setPosition(footerLengthOffset);
- var footerLength: number = Int64FromByteBuffer(bb, footerLengthOffset)
-
- if (footerLength <= 0 || footerLength + MAGIC.length*2 + 4 > fileLength) {
- console.log("Invalid footer length: " + footerLength)
- }
-
- var footerOffset: number = footerLengthOffset - footerLength;
- bb.setPosition(footerOffset);
- var footer = arrow.flatbuf.Footer.getRootAsFooter(bb);
-
- return footer;
-}
-
-function Int64FromByteBuffer(bb, offset) {
- return ((bb.bytes_[offset + 3] & 255) << 24) |
- ((bb.bytes_[offset + 2] & 255) << 16) |
- ((bb.bytes_[offset + 1] & 255) << 8) |
- ((bb.bytes_[offset] & 255));
-}
-
-
-var MAGIC_STR = "ARROW1";
-var MAGIC = new Uint8Array(MAGIC_STR.length);
-for (var i = 0; i < MAGIC_STR.length; i += 1|0) {
- MAGIC[i] = MAGIC_STR.charCodeAt(i);
-}
-
-function _checkMagic(buf, index) {
- for (var i = 0; i < MAGIC.length; i += 1|0) {
- if (MAGIC[i] != buf[index + i]) {
- return false;
- }
- }
- return true;
-}
-
-var TYPEMAP = {}
-TYPEMAP[arrow.flatbuf.Type.NONE] = "NONE";
-TYPEMAP[arrow.flatbuf.Type.Null] = "Null";
-TYPEMAP[arrow.flatbuf.Type.Int] = "Int";
-TYPEMAP[arrow.flatbuf.Type.FloatingPoint] = "FloatingPoint";
-TYPEMAP[arrow.flatbuf.Type.Binary] = "Binary";
-TYPEMAP[arrow.flatbuf.Type.Utf8] = "Utf8";
-TYPEMAP[arrow.flatbuf.Type.Bool] = "Bool";
-TYPEMAP[arrow.flatbuf.Type.Decimal] = "Decimal";
-TYPEMAP[arrow.flatbuf.Type.Date] = "Date";
-TYPEMAP[arrow.flatbuf.Type.Time] = "Time";
-TYPEMAP[arrow.flatbuf.Type.Timestamp] = "Timestamp";
-TYPEMAP[arrow.flatbuf.Type.Interval] = "Interval";
-TYPEMAP[arrow.flatbuf.Type.List] = "List";
-TYPEMAP[arrow.flatbuf.Type.Struct_] = "Struct";
-TYPEMAP[arrow.flatbuf.Type.Union] = "Union";
-
-var VECTORTYPEMAP = {};
-VECTORTYPEMAP[arrow.flatbuf.VectorType.OFFSET] = 'OFFSET';
-VECTORTYPEMAP[arrow.flatbuf.VectorType.DATA] = 'DATA';
-VECTORTYPEMAP[arrow.flatbuf.VectorType.VALIDITY] = 'VALIDITY';
-VECTORTYPEMAP[arrow.flatbuf.VectorType.TYPE] = 'TYPE';
-
-function parseField(field) {
- var children = [];
- for (var i = 0; i < field.childrenLength(); i += 1|0) {
- children.push(parseField(field.children(i)));
- }
-
- var layouts = [];
- for (var i = 0; i < field.layoutLength(); i += 1|0) {
- layouts.push(VECTORTYPEMAP[field.layout(i).type()]);
-
- }
-
- return {
- name: field.name(),
- nullable: field.nullable(),
- type: TYPEMAP[field.typeType()],
- children: children,
- layout: layouts
- };
-}
-
-function parseSchema(schema) {
- var result = [];
- var this_result, type;
- for (var i = 0, len = schema.fieldsLength(); i < len; i += 1|0) {
- result.push(parseField(schema.fields(i)));
- }
- return result;
-}
-
-function parseBuffer(buffer) {
- return {
- offset: buffer.offset(),
- length: buffer.length()
- };
-}
-
-function loadBuffersIntoVectors(recordBatchBlocks, bb, vectors : Vector[]) {
- var fieldNode, recordBatchBlock, recordBatch, numBuffers, bufReader = {index: 0, node_index: 1}, field_ctr = 0;
- var buffer = bb.bytes_.buffer;
- var baseOffset = bb.bytes_.byteOffset;
- for (var i = recordBatchBlocks.length - 1; i >= 0; i -= 1|0) {
- recordBatchBlock = recordBatchBlocks[i];
- bb.setPosition(recordBatchBlock.offset.low);
- recordBatch = arrow.flatbuf.RecordBatch.getRootAsRecordBatch(bb);
- bufReader.index = 0;
- bufReader.node_index = 0;
- numBuffers = recordBatch.buffersLength();
-
- //console.log('num buffers: ' + recordBatch.buffersLength());
- //console.log('num nodes: ' + recordBatch.nodesLength());
-
- while (bufReader.index < numBuffers) {
- //console.log('Allocating buffers starting at ' + bufReader.index + '/' + numBuffers + ' to field ' + field_ctr);
- vectors[field_ctr].loadData(recordBatch, buffer, bufReader, baseOffset + recordBatchBlock.offset.low + recordBatchBlock.metaDataLength)
- field_ctr += 1;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/lib/bitarray.ts
----------------------------------------------------------------------
diff --git a/javascript/lib/bitarray.ts b/javascript/lib/bitarray.ts
deleted file mode 100644
index 82fff32..0000000
--- a/javascript/lib/bitarray.ts
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-export class BitArray {
- private view: Uint8Array;
- constructor(buffer: ArrayBuffer, offset: number, length: number) {
- //if (ArrayBuffer.isView(buffer)) {
- // var og_view = buffer;
- // buffer = buffer.buffer;
- // offset = og_view.offset;
- // length = og_view.length/og_view.BYTES_PER_ELEMENT*8;
- //} else if (buffer instanceof ArrayBuffer) {
- var offset = offset || 0;
- var length = length;// || buffer.length*8;
- //} else if (buffer instanceof Number) {
- // length = buffer;
- // buffer = new ArrayBuffer(Math.ceil(length/8));
- // offset = 0;
- //}
-
- this.view = new Uint8Array(buffer, offset, Math.ceil(length/8));
- }
-
- get(i) {
- var index = (i >> 3) | 0; // | 0 converts to an int. Math.floor works too.
- var bit = i % 8; // i % 8 is just as fast as i & 7
- return (this.view[index] & (1 << bit)) !== 0;
- }
-
- set(i) {
- var index = (i >> 3) | 0;
- var bit = i % 8;
- this.view[index] |= 1 << bit;
- }
-
- unset(i) {
- var index = (i >> 3) | 0;
- var bit = i % 8;
- this.view[index] &= ~(1 << bit);
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/lib/types.ts
----------------------------------------------------------------------
diff --git a/javascript/lib/types.ts b/javascript/lib/types.ts
deleted file mode 100644
index bbc7558..0000000
--- a/javascript/lib/types.ts
+++ /dev/null
@@ -1,328 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import { BitArray } from './bitarray';
-import { TextDecoder } from 'text-encoding';
-import { org } from './Arrow_generated';
-var arrow = org.apache.arrow;
-
-interface ArrayView {
- slice(start: number, end: number) : ArrayView
- toString() : string
-}
-
-export abstract class Vector {
- name: string;
- length: number;
- null_count: number;
- constructor(name: string) {
- this.name = name;
- }
- /* Access datum at index i */
- abstract get(i);
- /* Return array representing data in the range [start, end) */
- abstract slice(start: number, end: number);
-
- /* Use recordBatch fieldNodes and Buffers to construct this Vector */
- public loadData(recordBatch: any, buffer: any, bufReader: any, baseOffset: any) {
- var fieldNode = recordBatch.nodes(bufReader.node_index);
- this.length = fieldNode.length();
- this.null_count = fieldNode.length();
- bufReader.node_index += 1|0;
-
- this.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
- }
-
- protected abstract loadBuffers(recordBatch: any, buffer: any, bufReader: any, baseOffset: any);
-
- /* Helper function for loading a VALIDITY buffer (for Nullable types) */
- static loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset) : BitArray {
- var buf_meta = recordBatch.buffers(bufReader.index);
- var offset = baseOffset + buf_meta.offset().low;
- var length = buf_meta.length().low;
- bufReader.index += 1|0;
- return new BitArray(buffer, offset, length*8);
- }
-
- /* Helper function for loading an OFFSET buffer */
- static loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset) : Int32Array {
- var buf_meta = recordBatch.buffers(bufReader.index);
- var offset = baseOffset + buf_meta.offset().low;
- var length = buf_meta.length().low/Int32Array.BYTES_PER_ELEMENT;
- bufReader.index += 1|0;
- return new Int32Array(buffer, offset, length);
- }
-
-}
-
-class SimpleVector<T extends ArrayView> extends Vector {
- protected dataView: T;
- private TypedArray: {new(buffer: any, offset: number, length: number) : T, BYTES_PER_ELEMENT: number};
-
- constructor (TypedArray: {new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number}, name: string) {
- super(name);
- this.TypedArray = TypedArray;
- }
-
- get(i) {
- return this.dataView[i];
- }
-
- loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
- this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
- }
-
- loadDataBuffer(recordBatch, buffer, bufReader, baseOffset) : T {
- var buf_meta = recordBatch.buffers(bufReader.index);
- var offset = baseOffset + buf_meta.offset().low;
- var length = buf_meta.length().low/this.TypedArray.BYTES_PER_ELEMENT;
- bufReader.index += 1|0;
- return new this.TypedArray(buffer, offset, length);
- }
-
- getDataView() {
- return this.dataView;
- }
-
- toString() {
- return this.dataView.toString();
- }
-
- slice(start, end) {
- return this.dataView.slice(start, end);
- }
-}
-
-class NullableSimpleVector<T extends ArrayView> extends SimpleVector<T> {
- private validityView: BitArray;
-
- get(i: number) {
- if (this.validityView.get(i)) return this.dataView[i];
- else return null
- }
-
- loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
- this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
- this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
- }
-
-}
-
-class Uint8Vector extends SimpleVector<Uint8Array> { constructor(name: string) { super(Uint8Array, name); }; }
-class Uint16Vector extends SimpleVector<Uint16Array> { constructor(name: string) { super(Uint16Array, name); }; }
-class Uint32Vector extends SimpleVector<Uint32Array> { constructor(name: string) { super(Uint32Array, name); }; }
-class Int8Vector extends SimpleVector<Uint8Array> { constructor(name: string) { super(Uint8Array, name); }; }
-class Int16Vector extends SimpleVector<Uint16Array> { constructor(name: string) { super(Uint16Array, name); }; }
-class Int32Vector extends SimpleVector<Uint32Array> { constructor(name: string) { super(Uint32Array, name); }; }
-class Float32Vector extends SimpleVector<Float32Array> { constructor(name: string) { super(Float32Array, name); }; }
-class Float64Vector extends SimpleVector<Float64Array> { constructor(name: string) { super(Float64Array, name); }; }
-
-class NullableUint8Vector extends NullableSimpleVector<Uint8Array> { constructor(name: string) { super(Uint8Array, name); }; }
-class NullableUint16Vector extends NullableSimpleVector<Uint16Array> { constructor(name: string) { super(Uint16Array, name); }; }
-class NullableUint32Vector extends NullableSimpleVector<Uint32Array> { constructor(name: string) { super(Uint32Array, name); }; }
-class NullableInt8Vector extends NullableSimpleVector<Uint8Array> { constructor(name: string) { super(Uint8Array, name); }; }
-class NullableInt16Vector extends NullableSimpleVector<Uint16Array> { constructor(name: string) { super(Uint16Array, name); }; }
-class NullableInt32Vector extends NullableSimpleVector<Uint32Array> { constructor(name: string) { super(Uint32Array, name); }; }
-class NullableFloat32Vector extends NullableSimpleVector<Float32Array> { constructor(name: string) { super(Float32Array, name); }; }
-class NullableFloat64Vector extends NullableSimpleVector<Float64Array> { constructor(name: string) { super(Float64Array, name); }; }
-
-class Utf8Vector extends SimpleVector<Uint8Array> {
- protected offsetView: Int32Array;
- static decoder: TextDecoder = new TextDecoder('utf8');
-
- constructor(name: string) {
- super(Uint8Array, name);
- }
-
- loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
- this.offsetView = Vector.loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset);
- this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
- }
-
- get(i) {
- return Utf8Vector.decoder.decode
- (this.dataView.slice(this.offsetView[i], this.offsetView[i + 1]));
- }
-
- slice(start: number, end: number) {
- var rtrn: string[] = [];
- for (var i: number = start; i < end; i += 1|0) {
- rtrn.push(this.get(i));
- }
- return rtrn;
- }
-}
-
-class NullableUtf8Vector extends Utf8Vector {
- private validityView: BitArray;
-
- loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
- this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
- this.offsetView = Vector.loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset);
- this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
- }
-
- get(i) {
- if (!this.validityView.get(i)) return null;
- return super.get(i);
- }
-}
-
-// Nested Types
-class ListVector extends Uint32Vector {
- private dataVector: Vector;
-
- constructor(name, dataVector : Vector) {
- super(name);
- this.dataVector = dataVector;
- }
-
- loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
- super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
- this.dataVector.loadData(recordBatch, buffer, bufReader, baseOffset);
- this.length -= 1;
- }
-
- get(i) {
- var offset = super.get(i)
- if (offset === null) {
- return null;
- }
- var next_offset = super.get(i + 1)
- return this.dataVector.slice(offset, next_offset)
- }
-
- toString() {
- return "length: " + (this.length);
- }
-
- slice(start : number, end : number) { return []; };
-}
-
-class NullableListVector extends ListVector {
- private validityView: BitArray;
-
- loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
- this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
- super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
- }
-
- get(i) {
- if (!this.validityView.get(i)) return null;
- return super.get(i);
- }
-}
-
-class StructVector extends Vector {
- private validityView: BitArray;
- private vectors : Vector[];
- constructor(name: string, vectors: Vector[]) {
- super(name);
- this.vectors = vectors;
- }
-
- loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
- this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
- this.vectors.forEach((v: Vector) => v.loadData(recordBatch, buffer, bufReader, baseOffset));
- }
-
- get(i : number) {
- if (!this.validityView.get(i)) return null;
- return this.vectors.map((v: Vector) => v.get(i));
- }
-
- slice(start : number, end : number) {
- var rtrn = [];
- for (var i: number = start; i < end; i += 1|0) {
- rtrn.push(this.get(i));
- }
- return rtrn;
- }
-}
-
-class DateVector extends SimpleVector<Uint32Array> {
- constructor (name: string) {
- super(Uint32Array, name);
- }
-
- get (i) {
- return new Date(super.get(2*i+1)*Math.pow(2,32) + super.get(2*i));
- }
-}
-
-class NullableDateVector extends DateVector {
- private validityView: BitArray;
-
- loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
- this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
- super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
- }
-
- get (i) {
- if (!this.validityView.get(i)) return null;
- return super.get(i);
- }
-}
-
-var BASIC_TYPES = [arrow.flatbuf.Type.Int, arrow.flatbuf.Type.FloatingPoint, arrow.flatbuf.Type.Utf8, arrow.flatbuf.Type.Date];
-
-export function vectorFromField(field) : Vector {
- var typeType = field.typeType();
- if (BASIC_TYPES.indexOf(typeType) >= 0) {
- var type = field.typeType();
- if (type === arrow.flatbuf.Type.Int) {
- type = field.type(new arrow.flatbuf.Int());
- var VectorConstructor : {new(string) : Vector};
- if (type.isSigned()) {
- if (type.bitWidth() == 32)
- VectorConstructor = field.nullable() ? NullableInt32Vector : Int32Vector;
- else if (type.bitWidth() == 16)
- VectorConstructor = field.nullable() ? NullableInt16Vector : Int16Vector;
- else if (type.bitWidth() == 8)
- VectorConstructor = field.nullable() ? NullableInt8Vector : Int8Vector;
- } else {
- if (type.bitWidth() == 32)
- VectorConstructor = field.nullable() ? NullableUint32Vector : Uint32Vector;
- else if (type.bitWidth() == 16)
- VectorConstructor = field.nullable() ? NullableUint16Vector : Uint16Vector;
- else if (type.bitWidth() == 8)
- VectorConstructor = field.nullable() ? NullableUint8Vector : Uint8Vector;
- }
- } else if (type === arrow.flatbuf.Type.FloatingPoint) {
- type = field.type(new arrow.flatbuf.FloatingPoint());
- if (type.precision() == arrow.flatbuf.Precision.SINGLE)
- VectorConstructor = field.nullable() ? NullableFloat32Vector : Float32Vector;
- else if (type.precision() == arrow.flatbuf.Precision.DOUBLE)
- VectorConstructor = field.nullable() ? NullableFloat64Vector : Float64Vector;
- } else if (type === arrow.flatbuf.Type.Utf8) {
- VectorConstructor = field.nullable() ? NullableUtf8Vector : Utf8Vector;
- } else if (type === arrow.flatbuf.Type.Date) {
- VectorConstructor = field.nullable() ? NullableDateVector : DateVector;
- }
-
- return new VectorConstructor(field.name());
- } else if (typeType === arrow.flatbuf.Type.List) {
- var dataVector = vectorFromField(field.children(0));
- return field.nullable() ? new NullableListVector(field.name(), dataVector) : new ListVector(field.name(), dataVector);
- } else if (typeType === arrow.flatbuf.Type.Struct_) {
- var vectors : Vector[] = [];
- for (var i : number = 0; i < field.childrenLength(); i += 1|0) {
- vectors.push(vectorFromField(field.children(i)));
- }
- return new StructVector(field.name(), vectors);
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/package.json
----------------------------------------------------------------------
diff --git a/javascript/package.json b/javascript/package.json
deleted file mode 100644
index b1e583b..0000000
--- a/javascript/package.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "name": "arrow",
- "version": "0.0.0",
- "description": "",
- "main": "dist/arrow.js",
- "scripts": {
- "postinstall": "./postinstall.sh",
- "test": "echo \"Error: no test specified\" && exit 1"
- },
- "author": "",
- "license": "Apache-2.0",
- "devDependencies": {
- "flatbuffers": "^1.5.0",
- "text-encoding": "^0.6.4"
- },
- "dependencies": {
- "commander": "^2.9.0"
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/postinstall.sh
----------------------------------------------------------------------
diff --git a/javascript/postinstall.sh b/javascript/postinstall.sh
deleted file mode 100755
index 1e6622f..0000000
--- a/javascript/postinstall.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
-
-echo "Compiling flatbuffer schemas..."
-#flatc -o lib --js ../format/Message.fbs ../format/File.fbs
-flatc -o lib --js ../format/*.fbs
-cat lib/*_generated.js > lib/Arrow_generated.js
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/tsconfig.json
----------------------------------------------------------------------
diff --git a/javascript/tsconfig.json b/javascript/tsconfig.json
deleted file mode 100644
index 89c31ef..0000000
--- a/javascript/tsconfig.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "compilerOptions": {
- "outDir": "./dist/",
- "allowJs": true,
- "target": "es5",
- "module": "commonjs",
- "moduleResolution": "node"
- },
- "include": [
- "typings/index.d.ts",
- "lib/*.js",
- "lib/*.ts"
- ]
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/javascript/webpack.config.js
----------------------------------------------------------------------
diff --git a/javascript/webpack.config.js b/javascript/webpack.config.js
deleted file mode 100644
index a0ed563..0000000
--- a/javascript/webpack.config.js
+++ /dev/null
@@ -1,21 +0,0 @@
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License. See accompanying LICENSE file.
-
-module.exports = {
- entry: './dist/arrow.js',
- output: {
- path: __dirname + '/dist',
- filename: 'arrow-bundle.js',
- libraryTarget: 'var',
- library: 'arrow'
- }
-};
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/.gitignore
----------------------------------------------------------------------
diff --git a/js/.gitignore b/js/.gitignore
new file mode 100644
index 0000000..3b97e3a
--- /dev/null
+++ b/js/.gitignore
@@ -0,0 +1,4 @@
+lib/*_generated.js
+dist
+node_modules
+typings
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/README.md
----------------------------------------------------------------------
diff --git a/js/README.md b/js/README.md
new file mode 100644
index 0000000..98ef756
--- /dev/null
+++ b/js/README.md
@@ -0,0 +1,50 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+### Installation
+
+From this directory, run:
+
+``` bash
+$ npm install # pull dependencies
+$ tsc # build typescript
+$ webpack # bundle for the browser
+```
+
+### Usage
+The library is designed to be used with node.js or in the browser, this repository contains examples of both.
+
+#### Node
+Import the arrow module:
+
+``` js
+var arrow = require("arrow.js");
+```
+
+See [bin/arrow_schema.js](bin/arrow_schema.js) and [bin/arrow2csv.js](bin/arrow2csv.js) for usage examples.
+
+#### Browser
+Include `dist/arrow-bundle.js` in a `<script />` tag:
+``` html
+<script src="arrow-bundle.js"/>
+```
+See [examples/read_file.html](examples/read_file.html) for a usage example - or try it out now at [theneuralbit.github.io/arrow](http://theneuralbit.github.io/arrow)
+
+### API
+##### `arrow.loadSchema(buffer)`
+Returns a JSON representation of the file's Arrow schema.
+
+##### `arrow.loadVectors(buffer)`
+Returns a dictionary of `Vector` objects, one for each column, indexed by the column's name.
+Vector objects have, at minimum, a `get(i)` method and a `length` attribute.
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/bin/arrow2csv.js
----------------------------------------------------------------------
diff --git a/js/bin/arrow2csv.js b/js/bin/arrow2csv.js
new file mode 100755
index 0000000..48df2f9
--- /dev/null
+++ b/js/bin/arrow2csv.js
@@ -0,0 +1,47 @@
+#! /usr/bin/env node
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+var fs = require('fs')
+var process = require('process');
+var loadVectors = require('../dist/arrow.js').loadVectors;
+var program = require('commander');
+
+function list (val) {
+ return val.split(',');
+}
+
+program
+ .version('0.1.0')
+ .usage('[options] <file>')
+ .option('-s --schema <list>', 'A comma-separated list of column names', list)
+ .parse(process.argv);
+
+if (!program.schema) {
+ program.outputHelp();
+ process.exit(1);
+}
+
+var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
+var vectors = loadVectors(buf);
+
+for (var i = 0; i < vectors[program.schema[0]].length; i += 1|0) {
+ console.log(program.schema.map(function (field) {
+ return '' + vectors[field].get(i);
+ }).join(','));
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/bin/arrow_schema.js
----------------------------------------------------------------------
diff --git a/js/bin/arrow_schema.js b/js/bin/arrow_schema.js
new file mode 100755
index 0000000..7044778
--- /dev/null
+++ b/js/bin/arrow_schema.js
@@ -0,0 +1,25 @@
+#! /usr/bin/env node
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+var fs = require('fs');
+var process = require('process');
+var loadSchema = require('../dist/arrow.js').loadSchema;
+
+var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
+console.log(JSON.stringify(loadSchema(buf), null, '\t'));
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/examples/read_file.html
----------------------------------------------------------------------
diff --git a/js/examples/read_file.html b/js/examples/read_file.html
new file mode 100644
index 0000000..02b6f08
--- /dev/null
+++ b/js/examples/read_file.html
@@ -0,0 +1,79 @@
+<!DOCTYPE html>
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<html>
+ <head>
+ <title>arrow.js browser test</title>
+ <meta charset="utf-8">
+ <style>
+table {
+ border-collapse: collapse;
+}
+table, th, td {
+ border: 1px solid black;
+}
+ </style>
+ <script type="text/javascript">
+var reader = new FileReader();
+function addCell (tr, type, name) {
+ var td = document.createElement(type)
+ td.textContent = name;
+ tr.appendChild(td);
+}
+reader.onload = function (evt) {
+ var buf = new Uint8Array(evt.target.result);
+ var schema = arrow.loadSchema(buf);
+ var vectors = arrow.loadVectors(buf);
+ var length = vectors[schema[0].name].length;
+
+ var thead = document.getElementById("thead");
+ var tbody = document.getElementById("tbody");
+ var header_row = document.createElement("tr");
+
+ schema.forEach(function (d) {
+ addCell(header_row, "th", d.name);
+ });
+
+ thead.appendChild(header_row);
+
+ for (var i = 0; i < length; i += 1|0) {
+ var tr = document.createElement("tr");
+ schema.forEach(function (d) { addCell(tr, "td", vectors[d.name].get(i)); });
+ tbody.appendChild(tr);
+ }
+}
+
+function handleFiles(files) {
+ reader.readAsArrayBuffer(files[0]);
+}
+ </script>
+ </head>
+ <body>
+ <input id="arrow-in" type="file" onchange="handleFiles(this.files)" />
+ <table>
+ <thead id="thead">
+ </thead>
+ <tbody id="tbody">
+ </tbody>
+ </table>
+ <script type="text/javascript" src="../dist/arrow-bundle.js"></script>
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/lib/Arrow_generated.d.ts
----------------------------------------------------------------------
diff --git a/js/lib/Arrow_generated.d.ts b/js/lib/Arrow_generated.d.ts
new file mode 100644
index 0000000..1f5b454
--- /dev/null
+++ b/js/lib/Arrow_generated.d.ts
@@ -0,0 +1,5 @@
+export var org: {
+ apache: {
+ arrow: any
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/lib/arrow.ts
----------------------------------------------------------------------
diff --git a/js/lib/arrow.ts b/js/lib/arrow.ts
new file mode 100644
index 0000000..0762885
--- /dev/null
+++ b/js/lib/arrow.ts
@@ -0,0 +1,201 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { flatbuffers } from 'flatbuffers';
+import { org } from './Arrow_generated';
+var arrow = org.apache.arrow;
+import { vectorFromField, Vector } from './types';
+
+export function loadVectors(buf) {
+ var fileLength = buf.length, bb, footerLengthOffset, footerLength,
+ footerOffset, footer, schema, field, type, type_str, i,
+ len, rb_metas, rb_meta, rtrn, recordBatchBlock, recordBatchBlocks = [];
+ var vectors : Vector[] = [];
+
+ bb = new flatbuffers.ByteBuffer(buf);
+
+ footer = _loadFooter(bb);
+
+ schema = footer.schema();
+
+ for (i = 0, len = schema.fieldsLength(); i < len; i += 1|0) {
+ field = schema.fields(i);
+ vectors.push(vectorFromField(field));
+ }
+
+ for (i = 0; i < footer.recordBatchesLength(); i += 1|0) {
+ recordBatchBlock = footer.recordBatches(i);
+ recordBatchBlocks.push({
+ offset: recordBatchBlock.offset(),
+ metaDataLength: recordBatchBlock.metaDataLength(),
+ bodyLength: recordBatchBlock.bodyLength(),
+ })
+ }
+
+ loadBuffersIntoVectors(recordBatchBlocks, bb, vectors);
+ var rtrn : any = {};
+ for (var i : any = 0; i < vectors.length; i += 1|0) {
+ rtrn[vectors[i].name] = vectors[i]
+ }
+ return rtrn;
+}
+
+export function loadSchema(buf) {
+ var footer = _loadFooter(new flatbuffers.ByteBuffer(buf));
+ var schema = footer.schema();
+
+ return parseSchema(schema);
+}
+
+function _loadFooter(bb) {
+ var fileLength: number = bb.bytes_.length;
+
+ if (fileLength < MAGIC.length*2 + 4) {
+ console.error("file too small " + fileLength);
+ return;
+ }
+
+ if (!_checkMagic(bb.bytes_, 0)) {
+ console.error("missing magic bytes at beginning of file")
+ return;
+ }
+
+ if (!_checkMagic(bb.bytes_, fileLength - MAGIC.length)) {
+ console.error("missing magic bytes at end of file")
+ return;
+ }
+
+ var footerLengthOffset: number = fileLength - MAGIC.length - 4;
+ bb.setPosition(footerLengthOffset);
+ var footerLength: number = Int64FromByteBuffer(bb, footerLengthOffset)
+
+ if (footerLength <= 0 || footerLength + MAGIC.length*2 + 4 > fileLength) {
+ console.log("Invalid footer length: " + footerLength)
+ }
+
+ var footerOffset: number = footerLengthOffset - footerLength;
+ bb.setPosition(footerOffset);
+ var footer = arrow.flatbuf.Footer.getRootAsFooter(bb);
+
+ return footer;
+}
+
+function Int64FromByteBuffer(bb, offset) {
+ return ((bb.bytes_[offset + 3] & 255) << 24) |
+ ((bb.bytes_[offset + 2] & 255) << 16) |
+ ((bb.bytes_[offset + 1] & 255) << 8) |
+ ((bb.bytes_[offset] & 255));
+}
+
+
+var MAGIC_STR = "ARROW1";
+var MAGIC = new Uint8Array(MAGIC_STR.length);
+for (var i = 0; i < MAGIC_STR.length; i += 1|0) {
+ MAGIC[i] = MAGIC_STR.charCodeAt(i);
+}
+
+function _checkMagic(buf, index) {
+ for (var i = 0; i < MAGIC.length; i += 1|0) {
+ if (MAGIC[i] != buf[index + i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+var TYPEMAP = {}
+TYPEMAP[arrow.flatbuf.Type.NONE] = "NONE";
+TYPEMAP[arrow.flatbuf.Type.Null] = "Null";
+TYPEMAP[arrow.flatbuf.Type.Int] = "Int";
+TYPEMAP[arrow.flatbuf.Type.FloatingPoint] = "FloatingPoint";
+TYPEMAP[arrow.flatbuf.Type.Binary] = "Binary";
+TYPEMAP[arrow.flatbuf.Type.Utf8] = "Utf8";
+TYPEMAP[arrow.flatbuf.Type.Bool] = "Bool";
+TYPEMAP[arrow.flatbuf.Type.Decimal] = "Decimal";
+TYPEMAP[arrow.flatbuf.Type.Date] = "Date";
+TYPEMAP[arrow.flatbuf.Type.Time] = "Time";
+TYPEMAP[arrow.flatbuf.Type.Timestamp] = "Timestamp";
+TYPEMAP[arrow.flatbuf.Type.Interval] = "Interval";
+TYPEMAP[arrow.flatbuf.Type.List] = "List";
+TYPEMAP[arrow.flatbuf.Type.Struct_] = "Struct";
+TYPEMAP[arrow.flatbuf.Type.Union] = "Union";
+
+var VECTORTYPEMAP = {};
+VECTORTYPEMAP[arrow.flatbuf.VectorType.OFFSET] = 'OFFSET';
+VECTORTYPEMAP[arrow.flatbuf.VectorType.DATA] = 'DATA';
+VECTORTYPEMAP[arrow.flatbuf.VectorType.VALIDITY] = 'VALIDITY';
+VECTORTYPEMAP[arrow.flatbuf.VectorType.TYPE] = 'TYPE';
+
+function parseField(field) {
+ var children = [];
+ for (var i = 0; i < field.childrenLength(); i += 1|0) {
+ children.push(parseField(field.children(i)));
+ }
+
+ var layouts = [];
+ for (var i = 0; i < field.layoutLength(); i += 1|0) {
+ layouts.push(VECTORTYPEMAP[field.layout(i).type()]);
+
+ }
+
+ return {
+ name: field.name(),
+ nullable: field.nullable(),
+ type: TYPEMAP[field.typeType()],
+ children: children,
+ layout: layouts
+ };
+}
+
+function parseSchema(schema) {
+ var result = [];
+ var this_result, type;
+ for (var i = 0, len = schema.fieldsLength(); i < len; i += 1|0) {
+ result.push(parseField(schema.fields(i)));
+ }
+ return result;
+}
+
+function parseBuffer(buffer) {
+ return {
+ offset: buffer.offset(),
+ length: buffer.length()
+ };
+}
+
+function loadBuffersIntoVectors(recordBatchBlocks, bb, vectors : Vector[]) {
+ var fieldNode, recordBatchBlock, recordBatch, numBuffers, bufReader = {index: 0, node_index: 1}, field_ctr = 0;
+ var buffer = bb.bytes_.buffer;
+ var baseOffset = bb.bytes_.byteOffset;
+ for (var i = recordBatchBlocks.length - 1; i >= 0; i -= 1|0) {
+ recordBatchBlock = recordBatchBlocks[i];
+ bb.setPosition(recordBatchBlock.offset.low);
+ recordBatch = arrow.flatbuf.RecordBatch.getRootAsRecordBatch(bb);
+ bufReader.index = 0;
+ bufReader.node_index = 0;
+ numBuffers = recordBatch.buffersLength();
+
+ //console.log('num buffers: ' + recordBatch.buffersLength());
+ //console.log('num nodes: ' + recordBatch.nodesLength());
+
+ while (bufReader.index < numBuffers) {
+ //console.log('Allocating buffers starting at ' + bufReader.index + '/' + numBuffers + ' to field ' + field_ctr);
+ vectors[field_ctr].loadData(recordBatch, buffer, bufReader, baseOffset + recordBatchBlock.offset.low + recordBatchBlock.metaDataLength)
+ field_ctr += 1;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/lib/bitarray.ts
----------------------------------------------------------------------
diff --git a/js/lib/bitarray.ts b/js/lib/bitarray.ts
new file mode 100644
index 0000000..82fff32
--- /dev/null
+++ b/js/lib/bitarray.ts
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+export class BitArray {
+ private view: Uint8Array;
+ constructor(buffer: ArrayBuffer, offset: number, length: number) {
+ //if (ArrayBuffer.isView(buffer)) {
+ // var og_view = buffer;
+ // buffer = buffer.buffer;
+ // offset = og_view.offset;
+ // length = og_view.length/og_view.BYTES_PER_ELEMENT*8;
+ //} else if (buffer instanceof ArrayBuffer) {
+ var offset = offset || 0;
+ var length = length;// || buffer.length*8;
+ //} else if (buffer instanceof Number) {
+ // length = buffer;
+ // buffer = new ArrayBuffer(Math.ceil(length/8));
+ // offset = 0;
+ //}
+
+ this.view = new Uint8Array(buffer, offset, Math.ceil(length/8));
+ }
+
+ get(i) {
+ var index = (i >> 3) | 0; // | 0 converts to an int. Math.floor works too.
+ var bit = i % 8; // i % 8 is just as fast as i & 7
+ return (this.view[index] & (1 << bit)) !== 0;
+ }
+
+ set(i) {
+ var index = (i >> 3) | 0;
+ var bit = i % 8;
+ this.view[index] |= 1 << bit;
+ }
+
+ unset(i) {
+ var index = (i >> 3) | 0;
+ var bit = i % 8;
+ this.view[index] &= ~(1 << bit);
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/lib/types.ts
----------------------------------------------------------------------
diff --git a/js/lib/types.ts b/js/lib/types.ts
new file mode 100644
index 0000000..bbc7558
--- /dev/null
+++ b/js/lib/types.ts
@@ -0,0 +1,328 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import { BitArray } from './bitarray';
+import { TextDecoder } from 'text-encoding';
+import { org } from './Arrow_generated';
+var arrow = org.apache.arrow;
+
+interface ArrayView {
+ slice(start: number, end: number) : ArrayView
+ toString() : string
+}
+
+export abstract class Vector {
+ name: string;
+ length: number;
+ null_count: number;
+ constructor(name: string) {
+ this.name = name;
+ }
+ /* Access datum at index i */
+ abstract get(i);
+ /* Return array representing data in the range [start, end) */
+ abstract slice(start: number, end: number);
+
+ /* Use recordBatch fieldNodes and Buffers to construct this Vector */
+ public loadData(recordBatch: any, buffer: any, bufReader: any, baseOffset: any) {
+ var fieldNode = recordBatch.nodes(bufReader.node_index);
+ this.length = fieldNode.length();
+ this.null_count = fieldNode.length();
+ bufReader.node_index += 1|0;
+
+ this.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
+ }
+
+ protected abstract loadBuffers(recordBatch: any, buffer: any, bufReader: any, baseOffset: any);
+
+ /* Helper function for loading a VALIDITY buffer (for Nullable types) */
+ static loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset) : BitArray {
+ var buf_meta = recordBatch.buffers(bufReader.index);
+ var offset = baseOffset + buf_meta.offset().low;
+ var length = buf_meta.length().low;
+ bufReader.index += 1|0;
+ return new BitArray(buffer, offset, length*8);
+ }
+
+ /* Helper function for loading an OFFSET buffer */
+ static loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset) : Int32Array {
+ var buf_meta = recordBatch.buffers(bufReader.index);
+ var offset = baseOffset + buf_meta.offset().low;
+ var length = buf_meta.length().low/Int32Array.BYTES_PER_ELEMENT;
+ bufReader.index += 1|0;
+ return new Int32Array(buffer, offset, length);
+ }
+
+}
+
+class SimpleVector<T extends ArrayView> extends Vector {
+ protected dataView: T;
+ private TypedArray: {new(buffer: any, offset: number, length: number) : T, BYTES_PER_ELEMENT: number};
+
+ constructor (TypedArray: {new(buffer: any, offset: number, length: number): T, BYTES_PER_ELEMENT: number}, name: string) {
+ super(name);
+ this.TypedArray = TypedArray;
+ }
+
+ get(i) {
+ return this.dataView[i];
+ }
+
+ loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+ this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+ }
+
+ loadDataBuffer(recordBatch, buffer, bufReader, baseOffset) : T {
+ var buf_meta = recordBatch.buffers(bufReader.index);
+ var offset = baseOffset + buf_meta.offset().low;
+ var length = buf_meta.length().low/this.TypedArray.BYTES_PER_ELEMENT;
+ bufReader.index += 1|0;
+ return new this.TypedArray(buffer, offset, length);
+ }
+
+ getDataView() {
+ return this.dataView;
+ }
+
+ toString() {
+ return this.dataView.toString();
+ }
+
+ slice(start, end) {
+ return this.dataView.slice(start, end);
+ }
+}
+
+class NullableSimpleVector<T extends ArrayView> extends SimpleVector<T> {
+ private validityView: BitArray;
+
+ get(i: number) {
+ if (this.validityView.get(i)) return this.dataView[i];
+ else return null
+ }
+
+ loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+ this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
+ this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+ }
+
+}
+
+class Uint8Vector extends SimpleVector<Uint8Array> { constructor(name: string) { super(Uint8Array, name); }; }
+class Uint16Vector extends SimpleVector<Uint16Array> { constructor(name: string) { super(Uint16Array, name); }; }
+class Uint32Vector extends SimpleVector<Uint32Array> { constructor(name: string) { super(Uint32Array, name); }; }
+class Int8Vector extends SimpleVector<Uint8Array> { constructor(name: string) { super(Uint8Array, name); }; }
+class Int16Vector extends SimpleVector<Uint16Array> { constructor(name: string) { super(Uint16Array, name); }; }
+class Int32Vector extends SimpleVector<Uint32Array> { constructor(name: string) { super(Uint32Array, name); }; }
+class Float32Vector extends SimpleVector<Float32Array> { constructor(name: string) { super(Float32Array, name); }; }
+class Float64Vector extends SimpleVector<Float64Array> { constructor(name: string) { super(Float64Array, name); }; }
+
+class NullableUint8Vector extends NullableSimpleVector<Uint8Array> { constructor(name: string) { super(Uint8Array, name); }; }
+class NullableUint16Vector extends NullableSimpleVector<Uint16Array> { constructor(name: string) { super(Uint16Array, name); }; }
+class NullableUint32Vector extends NullableSimpleVector<Uint32Array> { constructor(name: string) { super(Uint32Array, name); }; }
+class NullableInt8Vector extends NullableSimpleVector<Uint8Array> { constructor(name: string) { super(Uint8Array, name); }; }
+class NullableInt16Vector extends NullableSimpleVector<Uint16Array> { constructor(name: string) { super(Uint16Array, name); }; }
+class NullableInt32Vector extends NullableSimpleVector<Uint32Array> { constructor(name: string) { super(Uint32Array, name); }; }
+class NullableFloat32Vector extends NullableSimpleVector<Float32Array> { constructor(name: string) { super(Float32Array, name); }; }
+class NullableFloat64Vector extends NullableSimpleVector<Float64Array> { constructor(name: string) { super(Float64Array, name); }; }
+
+class Utf8Vector extends SimpleVector<Uint8Array> {
+ protected offsetView: Int32Array;
+ static decoder: TextDecoder = new TextDecoder('utf8');
+
+ constructor(name: string) {
+ super(Uint8Array, name);
+ }
+
+ loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+ this.offsetView = Vector.loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset);
+ this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+ }
+
+ get(i) {
+ return Utf8Vector.decoder.decode
+ (this.dataView.slice(this.offsetView[i], this.offsetView[i + 1]));
+ }
+
+ slice(start: number, end: number) {
+ var rtrn: string[] = [];
+ for (var i: number = start; i < end; i += 1|0) {
+ rtrn.push(this.get(i));
+ }
+ return rtrn;
+ }
+}
+
+class NullableUtf8Vector extends Utf8Vector {
+ private validityView: BitArray;
+
+ loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+ this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
+ this.offsetView = Vector.loadOffsetBuffer(recordBatch, buffer, bufReader, baseOffset);
+ this.dataView = this.loadDataBuffer(recordBatch, buffer, bufReader, baseOffset);
+ }
+
+ get(i) {
+ if (!this.validityView.get(i)) return null;
+ return super.get(i);
+ }
+}
+
+// Nested Types
+class ListVector extends Uint32Vector {
+ private dataVector: Vector;
+
+ constructor(name, dataVector : Vector) {
+ super(name);
+ this.dataVector = dataVector;
+ }
+
+ loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+ super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
+ this.dataVector.loadData(recordBatch, buffer, bufReader, baseOffset);
+ this.length -= 1;
+ }
+
+ get(i) {
+ var offset = super.get(i)
+ if (offset === null) {
+ return null;
+ }
+ var next_offset = super.get(i + 1)
+ return this.dataVector.slice(offset, next_offset)
+ }
+
+ toString() {
+ return "length: " + (this.length);
+ }
+
+ slice(start : number, end : number) { return []; };
+}
+
+class NullableListVector extends ListVector {
+ private validityView: BitArray;
+
+ loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+ this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
+ super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
+ }
+
+ get(i) {
+ if (!this.validityView.get(i)) return null;
+ return super.get(i);
+ }
+}
+
+class StructVector extends Vector {
+ private validityView: BitArray;
+ private vectors : Vector[];
+ constructor(name: string, vectors: Vector[]) {
+ super(name);
+ this.vectors = vectors;
+ }
+
+ loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+ this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
+ this.vectors.forEach((v: Vector) => v.loadData(recordBatch, buffer, bufReader, baseOffset));
+ }
+
+ get(i : number) {
+ if (!this.validityView.get(i)) return null;
+ return this.vectors.map((v: Vector) => v.get(i));
+ }
+
+ slice(start : number, end : number) {
+ var rtrn = [];
+ for (var i: number = start; i < end; i += 1|0) {
+ rtrn.push(this.get(i));
+ }
+ return rtrn;
+ }
+}
+
+class DateVector extends SimpleVector<Uint32Array> {
+ constructor (name: string) {
+ super(Uint32Array, name);
+ }
+
+ get (i) {
+ return new Date(super.get(2*i+1)*Math.pow(2,32) + super.get(2*i));
+ }
+}
+
+class NullableDateVector extends DateVector {
+ private validityView: BitArray;
+
+ loadBuffers(recordBatch, buffer, bufReader, baseOffset) {
+ this.validityView = Vector.loadValidityBuffer(recordBatch, buffer, bufReader, baseOffset);
+ super.loadBuffers(recordBatch, buffer, bufReader, baseOffset);
+ }
+
+ get (i) {
+ if (!this.validityView.get(i)) return null;
+ return super.get(i);
+ }
+}
+
+var BASIC_TYPES = [arrow.flatbuf.Type.Int, arrow.flatbuf.Type.FloatingPoint, arrow.flatbuf.Type.Utf8, arrow.flatbuf.Type.Date];
+
+export function vectorFromField(field) : Vector {
+ var typeType = field.typeType();
+ if (BASIC_TYPES.indexOf(typeType) >= 0) {
+ var type = field.typeType();
+ if (type === arrow.flatbuf.Type.Int) {
+ type = field.type(new arrow.flatbuf.Int());
+ var VectorConstructor : {new(string) : Vector};
+ if (type.isSigned()) {
+ if (type.bitWidth() == 32)
+ VectorConstructor = field.nullable() ? NullableInt32Vector : Int32Vector;
+ else if (type.bitWidth() == 16)
+ VectorConstructor = field.nullable() ? NullableInt16Vector : Int16Vector;
+ else if (type.bitWidth() == 8)
+ VectorConstructor = field.nullable() ? NullableInt8Vector : Int8Vector;
+ } else {
+ if (type.bitWidth() == 32)
+ VectorConstructor = field.nullable() ? NullableUint32Vector : Uint32Vector;
+ else if (type.bitWidth() == 16)
+ VectorConstructor = field.nullable() ? NullableUint16Vector : Uint16Vector;
+ else if (type.bitWidth() == 8)
+ VectorConstructor = field.nullable() ? NullableUint8Vector : Uint8Vector;
+ }
+ } else if (type === arrow.flatbuf.Type.FloatingPoint) {
+ type = field.type(new arrow.flatbuf.FloatingPoint());
+ if (type.precision() == arrow.flatbuf.Precision.SINGLE)
+ VectorConstructor = field.nullable() ? NullableFloat32Vector : Float32Vector;
+ else if (type.precision() == arrow.flatbuf.Precision.DOUBLE)
+ VectorConstructor = field.nullable() ? NullableFloat64Vector : Float64Vector;
+ } else if (type === arrow.flatbuf.Type.Utf8) {
+ VectorConstructor = field.nullable() ? NullableUtf8Vector : Utf8Vector;
+ } else if (type === arrow.flatbuf.Type.Date) {
+ VectorConstructor = field.nullable() ? NullableDateVector : DateVector;
+ }
+
+ return new VectorConstructor(field.name());
+ } else if (typeType === arrow.flatbuf.Type.List) {
+ var dataVector = vectorFromField(field.children(0));
+ return field.nullable() ? new NullableListVector(field.name(), dataVector) : new ListVector(field.name(), dataVector);
+ } else if (typeType === arrow.flatbuf.Type.Struct_) {
+ var vectors : Vector[] = [];
+ for (var i : number = 0; i < field.childrenLength(); i += 1|0) {
+ vectors.push(vectorFromField(field.children(i)));
+ }
+ return new StructVector(field.name(), vectors);
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/package.json
----------------------------------------------------------------------
diff --git a/js/package.json b/js/package.json
new file mode 100644
index 0000000..b1e583b
--- /dev/null
+++ b/js/package.json
@@ -0,0 +1,19 @@
+{
+ "name": "arrow",
+ "version": "0.0.0",
+ "description": "",
+ "main": "dist/arrow.js",
+ "scripts": {
+ "postinstall": "./postinstall.sh",
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "",
+ "license": "Apache-2.0",
+ "devDependencies": {
+ "flatbuffers": "^1.5.0",
+ "text-encoding": "^0.6.4"
+ },
+ "dependencies": {
+ "commander": "^2.9.0"
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/postinstall.sh
----------------------------------------------------------------------
diff --git a/js/postinstall.sh b/js/postinstall.sh
new file mode 100755
index 0000000..1e6622f
--- /dev/null
+++ b/js/postinstall.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+
+echo "Compiling flatbuffer schemas..."
+#flatc -o lib --js ../format/Message.fbs ../format/File.fbs
+flatc -o lib --js ../format/*.fbs
+cat lib/*_generated.js > lib/Arrow_generated.js
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/tsconfig.json
----------------------------------------------------------------------
diff --git a/js/tsconfig.json b/js/tsconfig.json
new file mode 100644
index 0000000..89c31ef
--- /dev/null
+++ b/js/tsconfig.json
@@ -0,0 +1,14 @@
+{
+ "compilerOptions": {
+ "outDir": "./dist/",
+ "allowJs": true,
+ "target": "es5",
+ "module": "commonjs",
+ "moduleResolution": "node"
+ },
+ "include": [
+ "typings/index.d.ts",
+ "lib/*.js",
+ "lib/*.ts"
+ ]
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/6867e93c/js/webpack.config.js
----------------------------------------------------------------------
diff --git a/js/webpack.config.js b/js/webpack.config.js
new file mode 100644
index 0000000..a0ed563
--- /dev/null
+++ b/js/webpack.config.js
@@ -0,0 +1,21 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License. See accompanying LICENSE file.
+
+module.exports = {
+ entry: './dist/arrow.js',
+ output: {
+ path: __dirname + '/dist',
+ filename: 'arrow-bundle.js',
+ libraryTarget: 'var',
+ library: 'arrow'
+ }
+};