You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@couchdb.apache.org by ga...@apache.org on 2015/09/04 13:50:19 UTC
couchdb-nmo git commit: Import csv into couchdb
Repository: couchdb-nmo
Updated Branches:
refs/heads/master 6860b3985 -> 7b9bf5e5e
Import csv into couchdb
This adds import-csv command to import csv files into couchdb
Project: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/commit/7b9bf5e5
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/tree/7b9bf5e5
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/diff/7b9bf5e5
Branch: refs/heads/master
Commit: 7b9bf5e5e78cb6d6de7fe150e32b1fd14328e05e
Parents: 6860b39
Author: Garren Smith <ga...@gmail.com>
Authored: Thu Aug 27 17:05:57 2015 +0200
Committer: Garren Smith <ga...@gmail.com>
Committed: Fri Sep 4 13:42:34 2015 +0200
----------------------------------------------------------------------
bin/nmo-cli.js | 4 +-
doc/api/nmo-import-csv.md | 17 +++++++
doc/cli/nmo-import-csv.md | 27 ++++++++++
package.json | 9 +++-
src/import-csv.js | 60 ++++++++++++++++++++++
src/nmo.js | 3 +-
test/fixtures/fake.csv | 3 ++
test/import-csv.js | 110 +++++++++++++++++++++++++++++++++++++++++
8 files changed, 230 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/bin/nmo-cli.js
----------------------------------------------------------------------
diff --git a/bin/nmo-cli.js b/bin/nmo-cli.js
index 449e9b1..8696009 100755
--- a/bin/nmo-cli.js
+++ b/bin/nmo-cli.js
@@ -10,7 +10,9 @@ var fs = require('fs');
var nmo = require('../lib/nmo.js');
var parsed = nopt({
'json': [Boolean],
- 'force': [Boolean]
+ 'force': [Boolean],
+ 'delimiter': [String],
+ 'columns': [Boolean]
}, {'v': 'v'}, process.argv, 2);
var cmd = parsed.argv.remain.shift();
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/doc/api/nmo-import-csv.md
----------------------------------------------------------------------
diff --git a/doc/api/nmo-import-csv.md b/doc/api/nmo-import-csv.md
new file mode 100644
index 0000000..a056ce6
--- /dev/null
+++ b/doc/api/nmo-import-csv.md
@@ -0,0 +1,17 @@
+nmo-import-csv(3) -- configuration
+==============================
+
+## SYNOPSIS
+
+ nmo.commands.importcsv(file, url, [csvOptions])
+
+
+
+## DESCRIPTION
+
+Import csv file into CouchDB
+
+ - importcsv:
+
+Accepts the file, url and csv options to import a file into CouchDB.
+It returns a promise.
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/doc/cli/nmo-import-csv.md
----------------------------------------------------------------------
diff --git a/doc/cli/nmo-import-csv.md b/doc/cli/nmo-import-csv.md
new file mode 100644
index 0000000..498a1b4
--- /dev/null
+++ b/doc/cli/nmo-import-csv.md
@@ -0,0 +1,27 @@
+nmo-import-csv(1) -- Bulk import CSV files
+=================================
+
+## SYNOPSIS
+
+ nmo import-csv <file>, <couchdb-url> [delimiter=','] [columns=true]
+
+## DESCRIPTION
+
+Imports a csv file into CouchDB.
+
+## CONFIGURATION:
+
+ delimiter
+ * Default: ','
+ * Type: String
+ The csv delimiter
+
+ columns
+ * Default: true
+ * Type: Boolean
+ Whether to use the first row of the csv to define the key fields for each document
+
+
+## EXAMPLE
+
+ nmo import-csv /path/to/file.cvs http://couchdb-url:5984 --delimiter=',' --columns=true
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/package.json
----------------------------------------------------------------------
diff --git a/package.json b/package.json
index 701530d..ce4dad3 100644
--- a/package.json
+++ b/package.json
@@ -7,7 +7,8 @@
"scripts": {
"transpile": "babel src -d lib",
"prepublish": "make && npm run transpile",
- "test": "lab -v --transform node_modules/lab-babel -t 100 -S"
+ "test": "lab -v --transform node_modules/lab-babel -t 100 -S",
+ "lint": "jshint ./src ./test"
},
"author": "Robert Kowalski <ro...@kowalski.gd>",
"license": "Apache 2.0",
@@ -25,8 +26,12 @@
},
"dependencies": {
"bluebird": "~2.9.24",
+ "bulkbadger": "^1.0.0",
"config-chain": "~1.1.8",
+ "couchbulkimporter": "^1.0.0",
+ "csv-parse": "^1.0.0",
"ini": "~1.3.3",
+ "lodash": "^3.10.1",
"nopt": "~3.0.1",
"npmlog": "~1.2.0",
"osenv": "~0.1.0",
@@ -47,10 +52,12 @@
],
"devDependencies": {
"babel": "^5.1.10",
+ "jshint": "^2.8.0",
"lab": "~5.5.1",
"lab-babel": "1.0.0",
"less": "~2.5.0",
"marked-man": "~0.1.4",
+ "nock": "^2.10.0",
"node-doc-generator": "robertkowalski/node-documentation-generator#sloppy"
}
}
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/src/import-csv.js
----------------------------------------------------------------------
diff --git a/src/import-csv.js b/src/import-csv.js
new file mode 100644
index 0000000..5b72f1e
--- /dev/null
+++ b/src/import-csv.js
@@ -0,0 +1,60 @@
+import fs from 'fs';
+import Promise from 'bluebird';
+import CouchBulkImporter from 'couchbulkimporter';
+import parse from 'csv-parse';
+import BulkBadger from 'bulkbadger';
+import {checkUrl} from './utils';
+import nmo from './nmo.js';
+
+export function cli (file, url, ...csvOptions) {
+ if (!file) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo import-csv [file] [couchdb-url] [...<csv options> <pairs>]',
+ ].join('\n');
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+
+ throw err;
+ }
+
+ const er = checkUrl(url);
+ if (er) {
+ throw er;
+ }
+
+ const opts = {delimiter: nmo.config.get('delimiter'), columns: nmo.config.get('columns')};
+ return importcsv.apply(importcsv, [file, url, opts]);
+}
+
+export function importcsv (file, url, {delimiter= ',', columns= true}) {
+ return new Promise((resolve, reject) => {
+ const input = fs.createReadStream(file)
+ .on('error', (err) => {
+ err.message = 'Error reading file - ' + err.message;
+ reject(err);
+ });
+
+ const parser = parse({
+ delimiter: delimiter,
+ columns: columns
+ });
+
+ input
+ .pipe(parser)
+ .pipe(new BulkBadger())
+ .pipe(new CouchBulkImporter({
+ url: url
+ }))
+ .on('error', function (err) {
+ err.message = 'Error uploading - ' + err.message;
+ reject(err);
+ })
+ .on('finish', function () {
+ console.log('Upload complete!');
+ resolve();
+ });
+
+ });
+}
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/src/nmo.js
----------------------------------------------------------------------
diff --git a/src/nmo.js b/src/nmo.js
index 874cec9..c60c038 100644
--- a/src/nmo.js
+++ b/src/nmo.js
@@ -7,7 +7,8 @@ const commands = [
'help',
'config',
'cluster',
- 'v'
+ 'v',
+ 'import-csv'
];
const nmo = {
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/test/fixtures/fake.csv
----------------------------------------------------------------------
diff --git a/test/fixtures/fake.csv b/test/fixtures/fake.csv
new file mode 100644
index 0000000..3338248
--- /dev/null
+++ b/test/fixtures/fake.csv
@@ -0,0 +1,3 @@
+name,surname,email
+John, Rambo, john@rambo.com
+Eddie, Vedder, eddie@vedder.com
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/7b9bf5e5/test/import-csv.js
----------------------------------------------------------------------
diff --git a/test/import-csv.js b/test/import-csv.js
new file mode 100644
index 0000000..e1eac3c
--- /dev/null
+++ b/test/import-csv.js
@@ -0,0 +1,110 @@
+import assert from 'assert';
+
+import Lab from 'lab';
+export const lab = Lab.script();
+import nock from 'nock';
+import nmo from '../src/nmo.js';
+import {cli, importcsv} from '../src/import-csv.js';
+
+const docs = {
+ "docs":[
+ {
+ "name":"John",
+ "surname":" Rambo",
+ "email":" john@rambo.com"
+ },
+ {
+ name: 'Eddie',
+ surname: ' Vedder',
+ email: ' eddie@vedder.com'
+ }
+ ]};
+
+lab.experiment('import csv', () => {
+ lab.beforeEach((done) => {
+ nmo
+ .load({nmoconf: __dirname + '/fixtures/randomini'})
+ .then(() => done())
+ .catch(() => done());
+
+ });
+
+ lab.experiment('cli', () => {
+
+ lab.test('throws error if no inputs', (done) => {
+
+ try {
+ cli();
+ } catch(e) {
+ assert.deepEqual(e.type, 'EUSAGE');
+ }
+ done();
+ });
+
+ lab.test('throws error if bad url', (done) => {
+
+ try {
+ cli('file', 'bad-url');
+ } catch(e) {
+ assert.ok(/not a valid url/.test(e.message));
+ }
+ done();
+ });
+
+ lab.test('full integration works', (done) => {
+ nock('http://127.0.0.1:5984')
+ .put('/fake-csv')
+ .reply(200)
+ .post('/fake-csv/_bulk_docs')
+ .reply(200);
+
+ cli(__dirname + '/fixtures/fake.csv', 'http://127.0.0.1:5984/fake-csv', 'delimiter=","')
+ .then(done);
+ });
+ });
+
+ lab.experiment('upload to couchdb', () => {
+
+ lab.test('reports bad file', (done) => {
+ const url = 'http://127.0.0.1:5984';
+ importcsv('bad-fake.csv', url + '/csv-upload', {}).catch(function (err) {
+ assert.ok(/Error reading file -/.test(err));
+ done();
+ });
+
+ });
+
+ lab.test('logs error for failed request', (done) => {
+ const url = 'http://127.0.0.1:5984';
+
+ nock(url)
+ .put('/csv-upload')
+ .reply(501);
+
+ importcsv(__dirname + '/fixtures/fake.csv', url + '/csv-upload', {}).catch(function (err) {
+ assert.ok(/CouchDB server answered:/.test(err));
+ done();
+ });
+
+ });
+
+ lab.test('Uploads csv file to CouchDB', (done) => {
+ const url = 'http://127.0.0.1:5984';
+
+ nock(url)
+ .put('/csv-upload')
+ .reply(200)
+ .post('/csv-upload/_bulk_docs', docs)
+ .reply(200);
+
+ importcsv(__dirname + '/fixtures/fake.csv', url + '/csv-upload', {delimiter: ',', columns: true}).then(function () {
+ done();
+ }).catch(function (err) {
+ throw 'error ' + err;
+ });
+
+ });
+
+ });
+
+});