You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@couchdb.apache.org by garrensmith <gi...@git.apache.org> on 2015/08/27 17:07:30 UTC
[GitHub] couchdb-nmo pull request: [WIP] Import csv
GitHub user garrensmith opened a pull request:
https://github.com/apache/couchdb-nmo/pull/1
[WIP] Import csv
This needs some clean up around error handling and capturing the arguments. But other than that it can now import a csv to CouchDB.
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/garrensmith/couchdb-nmo import-csv
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/couchdb-nmo/pull/1.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #1
----
commit d95f482d1ae3aa4919d437e6431e70c23bff9f10
Author: Garren Smith <ga...@gmail.com>
Date: 2015-08-26T12:54:44Z
Add jshint definitions
commit befe9feeee2fcbbfe70755eaeb4f5f00c730fb1d
Author: Garren Smith <ga...@gmail.com>
Date: 2015-08-27T15:05:57Z
First attempt at csv import
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on the pull request:
https://github.com/apache/couchdb-nmo/pull/1#issuecomment-137714888
awesome!
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38189452
--- Diff: src/import-csv.js ---
@@ -0,0 +1,65 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import _ from 'lodash';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...args) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
+ }
+
+ return importcsv.apply(importcsv, [file, url, convertArgsToKey(args)]);
+}
+
+export function convertArgsToKey (args) {
+ let lastKey = '';
+
+ return args.reduce((obj, val, i) => {
+ if (i % 2 === 0) {
+ lastKey = val;
+ } else {
+ obj[lastKey] = val;
+ }
+
+ return obj;
+
+ }, {});
+
+}
+
+export function importcsv (file, url, csvOptions) {
+ return new Promise((resolve, reject) => {
+ const input = fs.createReadStream(file);
+
+ const parser = parse(_.merge({
+ delimiter: ';',
+ columns: true
+ }, csvOptions));
+
+ input
+ .pipe(parser)
+ .pipe(new BulkBadger({chunksize: 2}))
+ .pipe(new CouchBulkImporter({
+ url: url
+ }))
+ .on('error', function (err) {
+ console.log('Error uploading - ', err);
+ reject(err);
+ })
+ .on('finish', function () {
+ console.log('Upload complete!');
--- End diff --
use `nmo.log.info` here
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by garrensmith <gi...@git.apache.org>.
Github user garrensmith commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38518425
--- Diff: src/import-csv.js ---
@@ -0,0 +1,65 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import _ from 'lodash';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...args) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
+ }
+
+ return importcsv.apply(importcsv, [file, url, convertArgsToKey(args)]);
+}
+
+export function convertArgsToKey (args) {
+ let lastKey = '';
+
+ return args.reduce((obj, val, i) => {
+ if (i % 2 === 0) {
+ lastKey = val;
+ } else {
+ obj[lastKey] = val;
+ }
+
+ return obj;
+
+ }, {});
+
+}
+
+export function importcsv (file, url, csvOptions) {
+ return new Promise((resolve, reject) => {
+ const input = fs.createReadStream(file);
+
+ const parser = parse(_.merge({
+ delimiter: ';',
+ columns: true
+ }, csvOptions));
--- End diff --
Unfortunately cannot use the default args because if an object is passed in then it automatically overrides the default object arguments. It doesn't merge them.
I then tried Object.assign as that should work but it seems to fail with lab-babel. I have no idea why.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on the pull request:
https://github.com/apache/couchdb-nmo/pull/1#issuecomment-137714878
+1
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on the pull request:
https://github.com/apache/couchdb-nmo/pull/1#issuecomment-137416834
very good work i am super amazed! looking forward to merge this today
i tried to explain how the tricky argument parsing works with values from the config files and arguments from the cli
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38189420
--- Diff: src/import-csv.js ---
@@ -0,0 +1,65 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import _ from 'lodash';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...args) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
+ }
+
+ return importcsv.apply(importcsv, [file, url, convertArgsToKey(args)]);
+}
+
+export function convertArgsToKey (args) {
--- End diff --
you don't need that function any more if you configure nopt to parse your options.
the passed options should be a usual object literal
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38635409
--- Diff: src/import-csv.js ---
@@ -0,0 +1,56 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...csvOptions) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
+ }
+
+ return importcsv.apply(importcsv, [file, url, createOpts(csvOptions)]);
+}
+
+export function createOpts (args) {
+ return args.reduce((obj, cmd) => {
+ let [key, val=true] = cmd.split('=');
+ obj[key] = val;
+ return obj;
+ }, {});
+}
+
+export function importcsv (file, url, {delimiter= ',', columns= 'true'}) {
+ return new Promise((resolve, reject) => {
+ const input = fs.createReadStream(file);
--- End diff --
try to add an `on('error')` here to catch non-existing / readable files, sadly error handling in node is still somehow uncomfortable
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38189518
--- Diff: src/import-csv.js ---
@@ -0,0 +1,65 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import _ from 'lodash';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...args) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
+ }
+
+ return importcsv.apply(importcsv, [file, url, convertArgsToKey(args)]);
+}
+
+export function convertArgsToKey (args) {
+ let lastKey = '';
+
+ return args.reduce((obj, val, i) => {
+ if (i % 2 === 0) {
+ lastKey = val;
+ } else {
+ obj[lastKey] = val;
+ }
+
+ return obj;
+
+ }, {});
+
+}
+
+export function importcsv (file, url, csvOptions) {
+ return new Promise((resolve, reject) => {
+ const input = fs.createReadStream(file);
+
+ const parser = parse(_.merge({
+ delimiter: ';',
+ columns: true
+ }, csvOptions));
+
+ input
+ .pipe(parser)
+ .pipe(new BulkBadger({chunksize: 2}))
--- End diff --
BulkBadger has a default chunksize of i think 300 - will make it a lot faster for big files, i i would remove setting it to 2
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38635076
--- Diff: doc/cli/nmo-import-csv.md ---
@@ -0,0 +1,27 @@
+nmo-import-csv(1) -- Bulk import CSV files
+=================================
+
+## SYNOPSIS
+
+ nmo import-csv [<file>], [<couchdb-url>] [delimiter=','] [columns=true]
--- End diff --
file and couchdb-url are not optional (remove the brackets)
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by garrensmith <gi...@git.apache.org>.
Github user garrensmith commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38513788
--- Diff: src/import-csv.js ---
@@ -0,0 +1,65 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import _ from 'lodash';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...args) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
--- End diff --
This part is copied from the config cli which also throws. Which would you prefer? I quite like throwing since the promise should still do a catch.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38635248
--- Diff: src/import-csv.js ---
@@ -0,0 +1,56 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...csvOptions) {
+ if (!file) {
--- End diff --
we can improve the error handling here:
test for:
- no url set
- url no http url
for both we already have a comfortable helper: https://github.com/apache/couchdb-nmo/blob/master/src/utils.js#L15
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38635034
--- Diff: doc/api/nmo-import-csv.md ---
@@ -0,0 +1,17 @@
+nmo-import-csv(3) -- configuration
+==============================
+
+## SYNOPSIS
+
+ nmo.commands.importcsv(file, url, csvOptions)
--- End diff --
`csvOptions` are optional:
```js
nmo.commands.importcsv(file, url, [csvOptions])
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38262525
--- Diff: src/import-csv.js ---
@@ -0,0 +1,65 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import _ from 'lodash';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...args) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
+ }
+
+ return importcsv.apply(importcsv, [file, url, convertArgsToKey(args)]);
+}
+
+export function convertArgsToKey (args) {
+ let lastKey = '';
+
+ return args.reduce((obj, val, i) => {
+ if (i % 2 === 0) {
+ lastKey = val;
+ } else {
+ obj[lastKey] = val;
+ }
+
+ return obj;
+
+ }, {});
+
+}
+
+export function importcsv (file, url, csvOptions) {
+ return new Promise((resolve, reject) => {
+ const input = fs.createReadStream(file);
+
+ const parser = parse(_.merge({
+ delimiter: ';',
+ columns: true
+ }, csvOptions));
+
+ input
+ .pipe(parser)
+ .pipe(new BulkBadger({chunksize: 2}))
+ .pipe(new CouchBulkImporter({
+ url: url
+ }))
+ .on('error', function (err) {
+ console.log('Error uploading - ', err);
--- End diff --
it will then get logged by the thingy that catches errors as error
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38254527
--- Diff: src/import-csv.js ---
@@ -0,0 +1,65 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import _ from 'lodash';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...args) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
+ }
+
+ return importcsv.apply(importcsv, [file, url, convertArgsToKey(args)]);
+}
+
+export function convertArgsToKey (args) {
+ let lastKey = '';
+
+ return args.reduce((obj, val, i) => {
+ if (i % 2 === 0) {
+ lastKey = val;
+ } else {
+ obj[lastKey] = val;
+ }
+
+ return obj;
+
+ }, {});
+
+}
+
+export function importcsv (file, url, csvOptions) {
+ return new Promise((resolve, reject) => {
+ const input = fs.createReadStream(file);
+
+ const parser = parse(_.merge({
+ delimiter: ';',
+ columns: true
+ }, csvOptions));
--- End diff --
i think you don't need a merge, just use default args:
```js
export function importcsv (file, url, csvOptions = {delimiter: ';', columns: true}) {
```
like in https://github.com/apache/couchdb-nmo/blob/master/src/config.js#L36
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by garrensmith <gi...@git.apache.org>.
Github user garrensmith closed the pull request at:
https://github.com/apache/couchdb-nmo/pull/1
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38635449
--- Diff: src/import-csv.js ---
@@ -0,0 +1,65 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import _ from 'lodash';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...args) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
--- End diff --
agreed
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by garrensmith <gi...@git.apache.org>.
Github user garrensmith commented on the pull request:
https://github.com/apache/couchdb-nmo/pull/1#issuecomment-137716155
Merged
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38189640
--- Diff: src/import-csv.js ---
@@ -0,0 +1,65 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import _ from 'lodash';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...args) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
+ }
+
+ return importcsv.apply(importcsv, [file, url, convertArgsToKey(args)]);
+}
+
+export function convertArgsToKey (args) {
+ let lastKey = '';
+
+ return args.reduce((obj, val, i) => {
+ if (i % 2 === 0) {
+ lastKey = val;
+ } else {
+ obj[lastKey] = val;
+ }
+
+ return obj;
+
+ }, {});
+
+}
+
+export function importcsv (file, url, csvOptions) {
+ return new Promise((resolve, reject) => {
+ const input = fs.createReadStream(file);
+
+ const parser = parse(_.merge({
+ delimiter: ';',
+ columns: true
+ }, csvOptions));
+
+ input
+ .pipe(parser)
+ .pipe(new BulkBadger({chunksize: 2}))
+ .pipe(new CouchBulkImporter({
+ url: url
+ }))
+ .on('error', function (err) {
+ console.log('Error uploading - ', err);
--- End diff --
I would modify the message of `err` - `err.message = 'Error uploading - ', err.message`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38189567
--- Diff: src/import-csv.js ---
@@ -0,0 +1,65 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import _ from 'lodash';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...args) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
--- End diff --
the cli functions usually return a promise, is there a reason to avoid it here?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38189383
--- Diff: package.json ---
@@ -24,12 +25,18 @@
"website": "./website"
},
"dependencies": {
+ "JSONStream": "^1.0.4",
--- End diff --
unused dep
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38637234
--- Diff: src/import-csv.js ---
@@ -0,0 +1,56 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...csvOptions) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
+ }
+
+ return importcsv.apply(importcsv, [file, url, createOpts(csvOptions)]);
--- End diff --
option parsing works a bit different:
you can access all passed commands from the cli or the nmo.conf file using:
```
nmo.config.get('delimiter')
```
cli commands have a higher priority than values from the config file.
would this work:
```js
const opts = {delimiter: nmo.config.get('delimiter'), columns: nmo.config.get('columns')};
return importcsv.apply(importcsv, [file, url, opts]);
```
and then in ./bin/cli.js:
```js
var parsed = nopt({
'json': [Boolean],
'force': [Boolean],
'delimiter': [String],
'columns': [String]
}, {'v': 'v'}, process.argv, 2);
```
you can then delete the custom option parsing `createOpts` and it has the advantage that power users can define
```
[nmo]
json=true
```
in their `.nmorc` as default value, and override it in special cases using the cli argument.
(they can use nmo to alter the config, using `nmo config set nmo json true`)
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-nmo pull request: [WIP] Import csv
Posted by robertkowalski <gi...@git.apache.org>.
Github user robertkowalski commented on a diff in the pull request:
https://github.com/apache/couchdb-nmo/pull/1#discussion_r38635291
--- Diff: src/import-csv.js ---
@@ -0,0 +1,56 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+
+export function cli (file, url, ...csvOptions) {
+ if (!file) {
--- End diff --
idea is to avoid these types of errors which will confuse the user:
```
(13:19:42) [robert@tequila-work] ~/apache/couchdb-nmo (pr/1 *) $ ./bin/nmo-cli.js import-csv sddf sdf dsf
events.js:85
throw er; // Unhandled 'error' event
^
Error: ENOENT, open 'sddf'
at Error (native)
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---