You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by bh...@apache.org on 2019/02/23 02:30:28 UTC

[arrow] branch master updated: ARROW-4652: [JS] Ensure RBReader transform-stream stays open when autoDestroy=false

This is an automated email from the ASF dual-hosted git repository.

bhulette pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new 01033a4  ARROW-4652: [JS] Ensure RBReader transform-stream stays open when autoDestroy=false
01033a4 is described below

commit 01033a432629342166f7088f0befa9553051ceab
Author: ptaylor <pa...@me.com>
AuthorDate: Fri Feb 22 18:30:14 2019 -0800

    ARROW-4652: [JS] Ensure RBReader transform-stream stays open when autoDestroy=false
    
    Closes https://issues.apache.org/jira/browse/ARROW-4652. Updates `arrow2csv` to print all tables from stdin, instead of stopping after the first.
    
    Author: ptaylor <pa...@me.com>
    
    Closes #3727 from trxcllnt/js/fix-reader-through-stream-autodestroy and squashes the following commits:
    
    184888a8 <ptaylor> enable arrow2csv to print multiple tables
    b6a81ea3 <ptaylor> ensure RecordBatchReader.throughNode resets when autoDestroy=false
---
 js/src/bin/arrow2csv.ts   | 35 +++++++++++++++++++----------------
 js/src/ipc/node/reader.ts |  2 +-
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/js/src/bin/arrow2csv.ts b/js/src/bin/arrow2csv.ts
index 4ae9c00..397abb5 100644
--- a/js/src/bin/arrow2csv.ts
+++ b/js/src/bin/arrow2csv.ts
@@ -43,12 +43,12 @@ const state = { ...argv, closed: false, hasRecords: false };
 
     for (const source of sources) {
         if (state.closed) { break; }
-        if (reader = await createRecordBatchReader(source)) {
-            await pipeline(
-                reader.toNodeStream(),
-                recordBatchRowsToString(state),
-                process.stdout
-            ).catch(() => state.closed = true);
+        for await (reader of recordBatchReaders(source)) {
+            const source = reader.toNodeStream();
+            const xform = batchesToString(state);
+            const sink = new stream.PassThrough();
+            sink.pipe(process.stdout, { end: false });
+            await pipeline(source, xform, sink).catch(() => state.closed = true);
         }
         if (state.closed) { break; }
     }
@@ -62,35 +62,38 @@ const state = { ...argv, closed: false, hasRecords: false };
     return process.exitCode || 1;
 }).then((code) => process.exit(code));
 
-async function createRecordBatchReader(createSourceStream: () => NodeJS.ReadableStream) {
+async function *recordBatchReaders(createSourceStream: () => NodeJS.ReadableStream) {
 
     let json = new AsyncByteQueue();
     let stream = new AsyncByteQueue();
     let source = createSourceStream();
     let reader: RecordBatchReader | null = null;
+    let readers: AsyncIterable<RecordBatchReader> | null = null;
     // tee the input source, just in case it's JSON
     source.on('end', () => [stream, json].forEach((y) => y.close()))
         .on('data', (x) => [stream, json].forEach((y) => y.write(x)))
        .on('error', (e) => [stream, json].forEach((y) => y.abort(e)));
 
     try {
-        reader = await (await RecordBatchReader.from(stream)).open();
-    } catch (e) { reader = null; }
+        for await (reader of RecordBatchReader.readAll(stream)) {
+            reader && (yield reader);
+        }
+        if (reader) return;
+    } catch (e) { readers = null; }
 
-    if (!reader || reader.closed) {
-        reader = null;
+    if (!readers) {
         await json.closed;
         if (source instanceof fs.ReadStream) { source.close(); }
         // If the data in the `json` ByteQueue parses to JSON, then assume it's Arrow JSON from a file or stdin
         try {
-            reader = await (await RecordBatchReader.from(bignumJSONParse(await json.toString()))).open();
-        } catch (e) { reader = null; }
+            for await (reader of RecordBatchReader.readAll(bignumJSONParse(await json.toString()))) {
+                reader && (yield reader);
+            }
+        } catch (e) { readers = null; }
     }
-
-    return (reader && !reader.closed) ? reader : null;
 }
 
-function recordBatchRowsToString(state: { closed: boolean, schema: any, separator: string, hasRecords: boolean }) {
+function batchesToString(state: { closed: boolean, schema: any, separator: string, hasRecords: boolean }) {
 
     let rowId = 0, maxColWidths = [15], separator = `${state.separator || ' |'} `;
 
diff --git a/js/src/ipc/node/reader.ts b/js/src/ipc/node/reader.ts
index 5cca84c..6284bea 100644
--- a/js/src/ipc/node/reader.ts
+++ b/js/src/ipc/node/reader.ts
@@ -77,7 +77,7 @@ class RecordBatchReaderDuplex<T extends { [key: string]: DataType } = any> exten
         while (this.readable && !(r = await reader.next()).done) {
             if (!this.push(r.value) || (size != null && --size <= 0)) { break; }
         }
-        if ((r && r.done || !this.readable)) {
+        if (!this.readable || (r && r.done && (reader.autoDestroy || (await reader.reset().open()).closed))) {
             this.push(null);
             await reader.cancel();
         }