You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by bh...@apache.org on 2019/02/23 02:30:28 UTC
[arrow] branch master updated: ARROW-4652: [JS] Ensure RBReader
transform-stream stays open when autoDestroy=false
This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 01033a4 ARROW-4652: [JS] Ensure RBReader transform-stream stays open when autoDestroy=false
01033a4 is described below
commit 01033a432629342166f7088f0befa9553051ceab
Author: ptaylor <pa...@me.com>
AuthorDate: Fri Feb 22 18:30:14 2019 -0800
ARROW-4652: [JS] Ensure RBReader transform-stream stays open when autoDestroy=false
Closes https://issues.apache.org/jira/browse/ARROW-4652. Updates `arrow2csv` to print all tables from stdin, instead of stopping after the first.
Author: ptaylor <pa...@me.com>
Closes #3727 from trxcllnt/js/fix-reader-through-stream-autodestroy and squashes the following commits:
184888a8 <ptaylor> enable arrow2csv to print multiple tables
b6a81ea3 <ptaylor> ensure RecordBatchReader.throughNode resets when autoDestroy=false
---
js/src/bin/arrow2csv.ts | 35 +++++++++++++++++++----------------
js/src/ipc/node/reader.ts | 2 +-
2 files changed, 20 insertions(+), 17 deletions(-)
diff --git a/js/src/bin/arrow2csv.ts b/js/src/bin/arrow2csv.ts
index 4ae9c00..397abb5 100644
--- a/js/src/bin/arrow2csv.ts
+++ b/js/src/bin/arrow2csv.ts
@@ -43,12 +43,12 @@ const state = { ...argv, closed: false, hasRecords: false };
for (const source of sources) {
if (state.closed) { break; }
- if (reader = await createRecordBatchReader(source)) {
- await pipeline(
- reader.toNodeStream(),
- recordBatchRowsToString(state),
- process.stdout
- ).catch(() => state.closed = true);
+ for await (reader of recordBatchReaders(source)) {
+ const source = reader.toNodeStream();
+ const xform = batchesToString(state);
+ const sink = new stream.PassThrough();
+ sink.pipe(process.stdout, { end: false });
+ await pipeline(source, xform, sink).catch(() => state.closed = true);
}
if (state.closed) { break; }
}
@@ -62,35 +62,38 @@ const state = { ...argv, closed: false, hasRecords: false };
return process.exitCode || 1;
}).then((code) => process.exit(code));
-async function createRecordBatchReader(createSourceStream: () => NodeJS.ReadableStream) {
+async function *recordBatchReaders(createSourceStream: () => NodeJS.ReadableStream) {
let json = new AsyncByteQueue();
let stream = new AsyncByteQueue();
let source = createSourceStream();
let reader: RecordBatchReader | null = null;
+ let readers: AsyncIterable<RecordBatchReader> | null = null;
// tee the input source, just in case it's JSON
source.on('end', () => [stream, json].forEach((y) => y.close()))
.on('data', (x) => [stream, json].forEach((y) => y.write(x)))
.on('error', (e) => [stream, json].forEach((y) => y.abort(e)));
try {
- reader = await (await RecordBatchReader.from(stream)).open();
- } catch (e) { reader = null; }
+ for await (reader of RecordBatchReader.readAll(stream)) {
+ reader && (yield reader);
+ }
+ if (reader) return;
+ } catch (e) { readers = null; }
- if (!reader || reader.closed) {
- reader = null;
+ if (!readers) {
await json.closed;
if (source instanceof fs.ReadStream) { source.close(); }
// If the data in the `json` ByteQueue parses to JSON, then assume it's Arrow JSON from a file or stdin
try {
- reader = await (await RecordBatchReader.from(bignumJSONParse(await json.toString()))).open();
- } catch (e) { reader = null; }
+ for await (reader of RecordBatchReader.readAll(bignumJSONParse(await json.toString()))) {
+ reader && (yield reader);
+ }
+ } catch (e) { readers = null; }
}
-
- return (reader && !reader.closed) ? reader : null;
}
-function recordBatchRowsToString(state: { closed: boolean, schema: any, separator: string, hasRecords: boolean }) {
+function batchesToString(state: { closed: boolean, schema: any, separator: string, hasRecords: boolean }) {
let rowId = 0, maxColWidths = [15], separator = `${state.separator || ' |'} `;
diff --git a/js/src/ipc/node/reader.ts b/js/src/ipc/node/reader.ts
index 5cca84c..6284bea 100644
--- a/js/src/ipc/node/reader.ts
+++ b/js/src/ipc/node/reader.ts
@@ -77,7 +77,7 @@ class RecordBatchReaderDuplex<T extends { [key: string]: DataType } = any> exten
while (this.readable && !(r = await reader.next()).done) {
if (!this.push(r.value) || (size != null && --size <= 0)) { break; }
}
- if ((r && r.done || !this.readable)) {
+ if (!this.readable || (r && r.done && (reader.autoDestroy || (await reader.reset().open()).closed))) {
this.push(null);
await reader.cancel();
}