You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ga...@apache.org on 2015/10/20 09:31:41 UTC
couchdb-nmo git commit: Save database to file
Repository: couchdb-nmo
Updated Branches:
refs/heads/master 5d87e202c -> 123f68ee6
Save database to file
`savetofile` saves the docs in _all_docs to file. It can also gzip them
if specified
This fixes COUCHDB-2839
Project: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/commit/123f68ee
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/tree/123f68ee
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-nmo/diff/123f68ee
Branch: refs/heads/master
Commit: 123f68ee6dc68d97a346a74c57d39cb9ba0bc702
Parents: 5d87e20
Author: Garren Smith <ga...@gmail.com>
Authored: Tue Oct 13 16:45:32 2015 +0200
Committer: Garren Smith <ga...@gmail.com>
Committed: Wed Oct 14 15:49:43 2015 +0200
----------------------------------------------------------------------
LICENSE | 43 ++++++++++
doc/api/nmo-savetofile.md | 12 +++
doc/cli/nmo-savetofile.md | 20 +++++
package.json | 2 +
src/nmo.js | 3 +-
src/savetofile.js | 85 +++++++++++++++++++
test/savetofile.js | 182 +++++++++++++++++++++++++++++++++++++++++
7 files changed, 346 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/123f68ee/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 6a38c5c..7a034ec 100644
--- a/LICENSE
+++ b/LICENSE
@@ -844,3 +844,46 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The complete list of contributors can be found at: https://github.com/hapijs/hapi/graphs/contributors
Portions of this project were initially based on the Yahoo! Inc. Postmile project,
published at https://github.com/yahoo/postmile.
+
+node_modules/JSONStream
+
+The MIT License
+
+Copyright (c) 2011 Dominic Tarr
+
+Permission is hereby granted, free of charge,
+to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to
+deal in the Software without restriction, including
+without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom
+the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
+ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Apache License, Version 2.0
+
+Copyright (c) 2011 Dominic Tarr
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/123f68ee/doc/api/nmo-savetofile.md
----------------------------------------------------------------------
diff --git a/doc/api/nmo-savetofile.md b/doc/api/nmo-savetofile.md
new file mode 100644
index 0000000..d908013
--- /dev/null
+++ b/doc/api/nmo-savetofile.md
@@ -0,0 +1,12 @@
+nmo-savetofile(3) -- activetasks
+==============================
+
+## SYNOPSIS
+
+ nmo.commands.savetofile([<url> || <cluster>], [database], [filename])
+
+
+## DESCRIPTION
+
+Save the database found at the specified `database` found at `url` or `cluster` to the `filename`.
+If `compress` is true in the config it will gzip the file as well.
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/123f68ee/doc/cli/nmo-savetofile.md
----------------------------------------------------------------------
diff --git a/doc/cli/nmo-savetofile.md b/doc/cli/nmo-savetofile.md
new file mode 100644
index 0000000..1c99c52
--- /dev/null
+++ b/doc/cli/nmo-savetofile.md
@@ -0,0 +1,20 @@
+nmo-savetofile(1) -- save database to file
+===========================================
+
+## SYNOPSIS
+
+ nmo savetofile cluster database file [--compress]
+ nmo activetasks url database file [--compress]
+
+
+## DESCRIPTION
+
+Save a database's _all_docs to disk. If `--compress` is added the file will be gzipped.
+
+Example:
+
+This will save the database named `my-db` to the file named `my-data-dump.json`
+ nmo save mycluster my-db my-data-dump.json
+
+This will save the database and compress it.
+ nmo save mycluster my-db my-data-dump.json --compress
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/123f68ee/package.json
----------------------------------------------------------------------
diff --git a/package.json b/package.json
index 8f235e6..44d1c9c 100644
--- a/package.json
+++ b/package.json
@@ -25,6 +25,7 @@
"website": "./website"
},
"dependencies": {
+ "JSONStream": "^1.0.6",
"bluebird": "~2.9.24",
"bulkbadger": "^1.0.0",
"config-chain": "~1.1.8",
@@ -46,6 +47,7 @@
"couchbulkimporter",
"csv-parse",
"ini",
+ "JSONStream",
"nopt",
"npmlog",
"osenv",
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/123f68ee/src/nmo.js
----------------------------------------------------------------------
diff --git a/src/nmo.js b/src/nmo.js
index 7d6a2e0..6b76095 100644
--- a/src/nmo.js
+++ b/src/nmo.js
@@ -14,7 +14,8 @@ const commands = [
'v',
'import-csv',
'couch-config',
- 'activetasks'
+ 'activetasks',
+ 'savetofile'
];
const nmo = {
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/123f68ee/src/savetofile.js
----------------------------------------------------------------------
diff --git a/src/savetofile.js b/src/savetofile.js
new file mode 100644
index 0000000..4c1ba82
--- /dev/null
+++ b/src/savetofile.js
@@ -0,0 +1,85 @@
+import { createWriteStream } from 'fs';
+import { createGzip } from 'zlib';
+import log from 'npmlog';
+import Wreck from 'wreck';
+import Promise from 'bluebird';
+import JSONStream from 'JSONStream';
+import {getUrlFromCluster, validUrl, removeUsernamePw } from './utils';
+import nmo from './nmo.js';
+
+export function cli (url, dbname, file) {
+ return new Promise((resolve, reject) => {
+
+ if (!url || !file || !dbname) {
+ const msg = [
+ 'Usage:',
+ '',
+ 'nmo activetasks <cluster> <database name> <file> ',
+ 'nmo activetasks <url> <database name> <file>'
+ ].join('\n');
+
+ const err = new Error(msg);
+ err.type = 'EUSAGE';
+ return reject(err);
+ }
+
+ savetofile(url, dbname, file)
+ .then(() => {
+ console.log(dbname + ' saved to ' + file + '.');
+ resolve();
+ })
+ .catch(err => {
+ err.type = 'EUSAGE';
+ reject(err);
+ });
+
+ });
+}
+
+export function saveResToFile (res, file) {
+ return new Promise((resolve, reject) => {
+ let writestream = createWriteStream(file);
+ const compress = nmo.config.get('compress');
+
+ let pipeline = res
+ .pipe(JSONStream.parse('rows.*.doc'))
+ .pipe(JSONStream.stringify());
+
+ if (compress) {
+ const gzip = createGzip();
+ pipeline = pipeline.pipe(gzip);
+ writestream = createWriteStream(file + '.gzip');
+ }
+
+ pipeline
+ .pipe(writestream)
+ .on('error', function (err) {reject(err);})
+ .on('finish', () => {resolve(file);});
+ });
+}
+
+export default function savetofile (cluster, dbname, file) {
+ return new Promise((resolve, reject) => {
+
+ const url = getUrlFromCluster(cluster) + '/' + dbname + '/_all_docs?include_docs=true';
+ const er = validUrl(url);
+
+ if (er) {
+ er.type = 'EUSAGE';
+ return reject(er);
+ }
+ const cleanedUrl = removeUsernamePw(url);
+ log.http('request', 'GET', cleanedUrl);
+
+ Wreck.request('GET', url, {}, function (err, res) {
+ if (err) {
+ return reject(err);
+ }
+
+ log.http(res.statusCode, cleanedUrl);
+ saveResToFile(res, file)
+ .then(() => resolve())
+ .catch((err) => {reject(err);});
+ });
+ });
+}
http://git-wip-us.apache.org/repos/asf/couchdb-nmo/blob/123f68ee/test/savetofile.js
----------------------------------------------------------------------
diff --git a/test/savetofile.js b/test/savetofile.js
new file mode 100644
index 0000000..89f5bce
--- /dev/null
+++ b/test/savetofile.js
@@ -0,0 +1,182 @@
+import assert from 'assert';
+import Lab from 'lab';
+import {unlinkSync, readFileSync } from 'fs';
+
+import savetofile, {cli} from '../src/savetofile.js';
+import nmo from '../src/nmo.js';
+
+import nock from 'nock';
+import zlib from 'zlib';
+import { createConfigFile } from './common';
+
+export let lab = Lab.script();
+
+lab.experiment('savetofile', () => {
+ createConfigFile();
+ lab.beforeEach((done) => {
+ nmo
+ .load({nmoconf: __dirname + '/fixtures/randomini'})
+ .then(() => done())
+ .catch(() => done());
+ });
+
+ lab.experiment('cli', () => {
+
+ lab.test('Returns error for empty params', done => {
+ cli().catch(err => {
+ assert.ok(/Usage/.test(err.message));
+ done();
+ });
+ });
+
+ lab.test('Returns error for missing file', done => {
+ cli('a url').catch(err => {
+ assert.ok(/Usage/.test(err.message));
+ done();
+ });
+ });
+
+ lab.test('returns error with type EUSAGE for any error', done => {
+ cli('http://127.0.0.1:5555', 'the-db', 'a-file')
+ .catch(err => {
+ assert.deepEqual(err.type, 'EUSAGE');
+ done();
+ });
+
+ });
+ });
+
+ lab.experiment('savetofile', () => {
+ const file1 = __dirname + '/fixtures/delete-me-please.json';
+ const file2 = __dirname + '/fixtures/savetofiletest.json';
+ const file3 = __dirname + '/fixtures/savetofiletest.json';
+
+ function deleteFile (file) {
+ //try and delete file. Its ok if it fails
+ try {
+ unlinkSync(file);
+ } catch (e) {
+
+ }
+ }
+
+ lab.afterEach(done => {
+ deleteFile(file1);
+ deleteFile(file2);
+ deleteFile(file3);
+ deleteFile(file3 + '.gzip');
+ done();
+ });
+
+ lab.test('get all docs for db', done => {
+ const url = 'http://127.0.1.20';
+ nock(url)
+ .get('/test-db/_all_docs?include_docs=true')
+ .reply(200, []);
+
+ savetofile(url, 'test-db', file1)
+ .then(() => {
+ done();
+ });
+
+ });
+
+ lab.test('saves contents to file', done => {
+ const resp = {
+ "total_rows":2,
+ "offset":0,
+ "rows":[
+ {
+ "id":"0461444c-e60a-457d-a4bb-b8d811853f21",
+ "key":"0461444c-e60a-457d-a4bb-b8d811853f21",
+ "value":{"rev":"1-090ec379aecccc97f8d02bc024ba28e5"},
+ "doc":{
+ "_id":"0461444c-e60a-457d-a4bb-b8d811853f21",
+ "_rev":"1-090ec379aecccc97f8d02bc024ba28e5",
+ "name": "Garren"
+ }
+ },
+ {
+ "id":"0461444c-e60a-457d-a4bb-b8d811853123",
+ "key":"0461444c-e60a-457d-a4bb-b8d811853123",
+ "value":{"rev":"1-090ec379aecccc97f8d02bc024ba123"},
+ "doc":{
+ "_id":"0461444c-e60a-457d-a4bb-b8d811853f21",
+ "_rev":"1-090ec379aecccc97f8d02bc024ba28e5",
+ "name": "Robert"
+ }
+ }
+ ]
+ };
+ const url = 'http://127.0.1.20';
+
+ nock(url)
+ .get('/test-db/_all_docs?include_docs=true')
+ .reply(200, resp);
+
+ savetofile(url, 'test-db', file2)
+ .then(() => {
+ const fileContents = readFileSync(file2, 'utf8');
+ var rows = resp.rows.map(row => row.doc);
+ assert.deepEqual(JSON.parse(fileContents), rows);
+ done();
+ });
+ });
+
+ lab.test('saves contents to file and zips with --zip', done => {
+ const resp = {
+ "total_rows":2,
+ "offset":0,
+ "rows":[
+ {
+ "id":"0461444c-e60a-457d-a4bb-b8d811853f21",
+ "key":"0461444c-e60a-457d-a4bb-b8d811853f21",
+ "value":{"rev":"1-090ec379aecccc97f8d02bc024ba28e5"},
+ "doc":{
+ "_id":"0461444c-e60a-457d-a4bb-b8d811853f21",
+ "_rev":"1-090ec379aecccc97f8d02bc024ba28e5",
+ "name": "Garren"
+ }
+ },
+ {
+ "id":"0461444c-e60a-457d-a4bb-b8d811853123",
+ "key":"0461444c-e60a-457d-a4bb-b8d811853123",
+ "value":{"rev":"1-090ec379aecccc97f8d02bc024ba123"},
+ "doc":{
+ "_id":"0461444c-e60a-457d-a4bb-b8d811853f21",
+ "_rev":"1-090ec379aecccc97f8d02bc024ba28e5",
+ "name": "Robert"
+ }
+ }
+ ]
+ };
+ const url = 'http://127.0.1.20';
+
+ nock(url)
+ .get('/test-db/_all_docs?include_docs=true')
+ .reply(200, resp);
+ nmo
+ .load({nmoconf: __dirname + '/fixtures/randomini', compress: true})
+ .then(() => {
+ savetofile(url, 'test-db', file3)
+ .then(() => {
+ zlib.unzip(readFileSync(file3 + '.gzip'), function (err, fileContents) {
+ var rows = resp.rows.map(row => row.doc);
+ assert.deepEqual(JSON.parse(fileContents.toString()), rows);
+ done();
+ });
+ });
+ });
+ });
+
+ lab.test('returns error on failed fetch of data', done => {
+ savetofile('http://127.0.0.1:5555', 'db', 'the-file')
+ .catch(err => {
+ assert.ok(/ECONNREFUSED/.test(err.message));
+ done();
+ });
+
+ });
+ });
+
+});