You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cordova.apache.org by rk...@apache.org on 2016/03/10 19:53:38 UTC
cordova-plugin-file git commit: CB-7862: FileReader reads large files
in chunks with progress. This closes #168
Repository: cordova-plugin-file
Updated Branches:
refs/heads/master 3e09b31fd -> 3eae249cc
CB-7862: FileReader reads large files in chunks with progress. This closes #168
Project: http://git-wip-us.apache.org/repos/asf/cordova-plugin-file/repo
Commit: http://git-wip-us.apache.org/repos/asf/cordova-plugin-file/commit/3eae249c
Tree: http://git-wip-us.apache.org/repos/asf/cordova-plugin-file/tree/3eae249c
Diff: http://git-wip-us.apache.org/repos/asf/cordova-plugin-file/diff/3eae249c
Branch: refs/heads/master
Commit: 3eae249cca98de6793ed8af314a89952a912c464
Parents: 3e09b31
Author: Jason Ginchereau <ja...@microsoft.com>
Authored: Tue Mar 8 13:11:21 2016 -0800
Committer: Richard Knoll <ri...@gmail.com>
Committed: Thu Mar 10 10:51:08 2016 -0800
----------------------------------------------------------------------
tests/tests.js | 72 ++++++++---
www/FileReader.js | 326 +++++++++++++++++--------------------------------
2 files changed, 169 insertions(+), 229 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cordova-plugin-file/blob/3eae249c/tests/tests.js
----------------------------------------------------------------------
diff --git a/tests/tests.js b/tests/tests.js
index 3cdca3e..def599a 100644
--- a/tests/tests.js
+++ b/tests/tests.js
@@ -243,7 +243,7 @@ exports.defineAutoTests = function () {
it("file.spec.5 should be able to retrieve a TEMPORARY file system", function (done) {
var win = function (fileSystem) {
expect(fileSystem).toBeDefined();
- if (isChrome) {
+ if (isChrome) {
expect(fileSystem.name).toContain("Temporary");
} else {
expect(fileSystem.name).toBe("temporary");
@@ -2281,13 +2281,14 @@ exports.defineAutoTests = function () {
// create a file, write to it, and read it in again
createFile(fileName, createWriter, failed.bind(null, done, 'createFile - Error creating file: ' + fileName));
}
- function runReaderTest(funcName, writeBinary, done, verifierFunc, sliceStart, sliceEnd, fileContents) {
+ function runReaderTest(funcName, writeBinary, done, progressFunc, verifierFunc, sliceStart, sliceEnd, fileContents) {
writeDummyFile(writeBinary, function (fileEntry, file, fileData, fileDataAsBinaryString) {
var verifier = function (evt) {
expect(evt).toBeDefined();
verifierFunc(evt, fileData, fileDataAsBinaryString);
};
var reader = new FileReader();
+ reader.onprogress = progressFunc;
reader.onload = verifier;
reader.onerror = failed.bind(null, done, 'reader.onerror - Error reading file: ' + file + ' using function: ' + funcName);
if (sliceEnd !== undefined) {
@@ -2309,20 +2310,20 @@ exports.defineAutoTests = function () {
return match;
}
it("file.spec.84 should read file properly, readAsText", function (done) {
- runReaderTest('readAsText', false, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsText', false, done, null, function (evt, fileData, fileDataAsBinaryString) {
expect(evt.target.result).toBe(fileData);
done();
});
});
it("file.spec.84.1 should read JSON file properly, readAsText", function (done) {
var testObject = {key1: "value1", key2: 2};
- runReaderTest('readAsText', false, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsText', false, done, null, function (evt, fileData, fileDataAsBinaryString) {
expect(evt.target.result).toEqual(JSON.stringify(testObject));
done();
}, undefined, undefined, JSON.stringify(testObject));
});
it("file.spec.85 should read file properly, Data URI", function (done) {
- runReaderTest('readAsDataURL', true, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsDataURL', true, done, null, function (evt, fileData, fileDataAsBinaryString) {
/* `readAsDataURL` function is supported, but the mediatype in Chrome depends on entry name extension,
mediatype in IE is always empty (which is the same as `text-plain` according the specification),
the mediatype in Firefox is always `application/octet-stream`.
@@ -2346,7 +2347,7 @@ exports.defineAutoTests = function () {
pending();
}
- runReaderTest('readAsBinaryString', true, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsBinaryString', true, done, null, function (evt, fileData, fileDataAsBinaryString) {
expect(evt.target.result).toBe(fileDataAsBinaryString);
done();
});
@@ -2357,37 +2358,37 @@ exports.defineAutoTests = function () {
expect(true).toFailWithMessage('Platform does not supported this feature');
done();
}
- runReaderTest('readAsArrayBuffer', true, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsArrayBuffer', true, done, null, function (evt, fileData, fileDataAsBinaryString) {
expect(arrayBufferEqualsString(evt.target.result, fileDataAsBinaryString)).toBe(true);
done();
});
});
it("file.spec.88 should read sliced file: readAsText", function (done) {
- runReaderTest('readAsText', false, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsText', false, done, null, function (evt, fileData, fileDataAsBinaryString) {
expect(evt.target.result).toBe(fileDataAsBinaryString.slice(10, 40));
done();
}, 10, 40);
});
it("file.spec.89 should read sliced file: slice past eof", function (done) {
- runReaderTest('readAsText', false, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsText', false, done, null, function (evt, fileData, fileDataAsBinaryString) {
expect(evt.target.result).toBe(fileData.slice(-5, 9999));
done();
}, -5, 9999);
});
it("file.spec.90 should read sliced file: slice to eof", function (done) {
- runReaderTest('readAsText', false, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsText', false, done, null, function (evt, fileData, fileDataAsBinaryString) {
expect(evt.target.result).toBe(fileData.slice(-5));
done();
}, -5);
});
it("file.spec.91 should read empty slice", function (done) {
- runReaderTest('readAsText', false, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsText', false, done, null, function (evt, fileData, fileDataAsBinaryString) {
expect(evt.target.result).toBe('');
done();
}, 0, 0);
});
it("file.spec.92 should read sliced file properly, readAsDataURL", function (done) {
- runReaderTest('readAsDataURL', true, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsDataURL', true, done, null, function (evt, fileData, fileDataAsBinaryString) {
/* `readAsDataURL` function is supported, but the mediatype in Chrome depends on entry name extension,
mediatype in IE is always empty (which is the same as `text-plain` according the specification),
the mediatype in Firefox is always `application/octet-stream`.
@@ -2411,7 +2412,7 @@ exports.defineAutoTests = function () {
pending();
}
- runReaderTest('readAsBinaryString', true, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsBinaryString', true, done, null, function (evt, fileData, fileDataAsBinaryString) {
expect(evt.target.result).toBe(fileDataAsBinaryString.slice(-10, -5));
done();
}, -10, -5);
@@ -2422,12 +2423,53 @@ exports.defineAutoTests = function () {
expect(true).toFailWithMessage('Platform does not supported this feature');
done();
}
- runReaderTest('readAsArrayBuffer', true, done, function (evt, fileData, fileDataAsBinaryString) {
+ runReaderTest('readAsArrayBuffer', true, done, null, function (evt, fileData, fileDataAsBinaryString) {
expect(arrayBufferEqualsString(evt.target.result, fileDataAsBinaryString.slice(0, -1))).toBe(true);
done();
}, 0, -1);
});
- });
+ it("file.spec.94.5 should read large file in multiple chunks, readAsArrayBuffer", function (done) {
+ // Skip test if ArrayBuffers are not supported (e.g.: Android 2.3).
+ if (typeof window.ArrayBuffer == 'undefined') {
+ expect(true).toFailWithMessage('Platform does not supported this feature');
+ done();
+ }
+
+ var largeText = "";
+ for (var i = 0; i < 1000; i++) {
+ largeText += "Test " + i + "\n";
+ }
+
+ // Set the chunk size so that the read will take 5 chunks
+ FileReader.READ_CHUNK_SIZE = largeText.length / 4 + 1;
+
+ var chunkCount = 0;
+ var lastProgressValue = -1;
+ var progressFunc = function (evt) {
+ expect(evt.loaded).toBeDefined();
+ expect(evt.total).toBeDefined();
+
+ expect(evt.total >= largeText.length).toBe(true);
+ expect(evt.total <= largeText.length + 5).toBe(true);
+ expect(evt.loaded > lastProgressValue).toBe(true);
+ expect(evt.loaded <= evt.total).toBe(true);
+
+ lastProgressValue = evt.loaded;
+ chunkCount++;
+ };
+
+ runReaderTest(
+ 'readAsArrayBuffer', true, done, progressFunc,
+ function (evt, fileData, fileDataAsBinaryString) {
+ expect(arrayBufferEqualsString(evt.target.result, fileDataAsBinaryString.slice(0, -1))).toBe(true);
+ expect(lastProgressValue >= largeText.length).toBe(true);
+ expect(lastProgressValue <= largeText.length + 5).toBe(true);
+ expect(chunkCount).toBe(5);
+ done();
+ },
+ 0, -1, largeText);
+ });
+ });
//Read method
describe('FileWriter', function () {
it("file.spec.95 should have correct methods", function (done) {
http://git-wip-us.apache.org/repos/asf/cordova-plugin-file/blob/3eae249c/www/FileReader.js
----------------------------------------------------------------------
diff --git a/www/FileReader.js b/www/FileReader.js
index 30c9825..1b841d8 100644
--- a/www/FileReader.js
+++ b/www/FileReader.js
@@ -38,10 +38,19 @@ var FileReader = function() {
this._readyState = 0;
this._error = null;
this._result = null;
+ this._progress = null;
this._localURL = '';
this._realReader = origFileReader ? new origFileReader() : {};
};
+/**
+ * Defines the maximum size to read at a time via the native API. The default value is a compromise between
+ * minimizing the overhead of many exec() calls while still reporting progress frequently enough for large files.
+ * (Note attempts to allocate more than a few MB of contiguous memory on the native side are likely to cause
+ * OOM exceptions, while the JS engine seems to have fewer problems managing large strings or ArrayBuffers.)
+ */
+FileReader.READ_CHUNK_SIZE = 256*1024;
+
// States
FileReader.EMPTY = 0;
FileReader.LOADING = 1;
@@ -81,6 +90,7 @@ function initRead(reader, file) {
reader._result = null;
reader._error = null;
+ reader._progress = 0;
reader._readyState = FileReader.LOADING;
if (typeof file.localURL == 'string') {
@@ -96,6 +106,78 @@ function initRead(reader, file) {
}
/**
+ * Callback used by the following read* functions to handle incremental or final success.
+ * Must be bound to the FileReader's this along with all but the last parameter,
+ * e.g. readSuccessCallback.bind(this, "readAsText", "UTF-8", offset, totalSize, accumulate)
+ * @param readType The name of the read function to call.
+ * @param encoding Text encoding, or null if this is not a text type read.
+ * @param offset Starting offset of the read.
+ * @param totalSize Total number of bytes or chars to read.
+ * @param accumulate A function that takes the callback result and accumulates it in this._result.
+ * @param r Callback result returned by the last read exec() call, or null to begin reading.
+ */
+function readSuccessCallback(readType, encoding, offset, totalSize, accumulate, r) {
+ if (this._readyState === FileReader.DONE) {
+ return;
+ }
+
+ if (typeof r !== "undefined") {
+ accumulate(r);
+ this._progress = Math.min(this._progress + FileReader.READ_CHUNK_SIZE, totalSize);
+
+ if (typeof this.onprogress === "function") {
+ this.onprogress(new ProgressEvent("progress", {loaded:this._progress, total:totalSize}));
+ }
+ }
+
+ if (typeof r === "undefined" || this._progress < totalSize) {
+ var execArgs = [
+ this._localURL,
+ offset + this._progress,
+ offset + this._progress + Math.min(totalSize - this._progress, FileReader.READ_CHUNK_SIZE)];
+ if (encoding) {
+ execArgs.splice(1, 0, encoding);
+ }
+ exec(
+ readSuccessCallback.bind(this, readType, encoding, offset, totalSize, accumulate),
+ readFailureCallback.bind(this),
+ "File", readType, execArgs);
+ } else {
+ this._readyState = FileReader.DONE;
+
+ if (typeof this.onload === "function") {
+ this.onload(new ProgressEvent("load", {target:this}));
+ }
+
+ if (typeof this.onloadend === "function") {
+ this.onloadend(new ProgressEvent("loadend", {target:this}));
+ }
+ }
+}
+
+/**
+ * Callback used by the following read* functions to handle errors.
+ * Must be bound to the FileReader's this, e.g. readFailureCallback.bind(this)
+ */
+function readFailureCallback(e) {
+ if (this._readyState === FileReader.DONE) {
+ return;
+ }
+
+ this._readyState = FileReader.DONE;
+ this._result = null;
+ this._error = new FileError(e);
+
+ if (typeof this.onerror === "function") {
+ this.onerror(new ProgressEvent("error", {target:this}));
+ }
+
+ if (typeof this.onloadend === "function") {
+ this.onloadend(new ProgressEvent("loadend", {target:this}));
+ }
+}
+
+/**
* Abort reading file.
*/
FileReader.prototype.abort = function() {
@@ -133,60 +215,14 @@ FileReader.prototype.readAsText = function(file, encoding) {
// Default encoding is UTF-8
var enc = encoding ? encoding : "UTF-8";
- var me = this;
- var execArgs = [this._localURL, enc, file.start, file.end];
-
- // Read file
- exec(
- // Success callback
- function(r) {
- // If DONE (cancelled), then don't do anything
- if (me._readyState === FileReader.DONE) {
- return;
- }
-
- // DONE state
- me._readyState = FileReader.DONE;
-
- // Save result
- me._result = r;
-
- // If onload callback
- if (typeof me.onload === "function") {
- me.onload(new ProgressEvent("load", {target:me}));
- }
-
- // If onloadend callback
- if (typeof me.onloadend === "function") {
- me.onloadend(new ProgressEvent("loadend", {target:me}));
- }
- },
- // Error callback
- function(e) {
- // If DONE (cancelled), then don't do anything
- if (me._readyState === FileReader.DONE) {
- return;
- }
-
- // DONE state
- me._readyState = FileReader.DONE;
-
- // null result
- me._result = null;
-
- // Save error
- me._error = new FileError(e);
-
- // If onerror callback
- if (typeof me.onerror === "function") {
- me.onerror(new ProgressEvent("error", {target:me}));
- }
-
- // If onloadend callback
- if (typeof me.onloadend === "function") {
- me.onloadend(new ProgressEvent("loadend", {target:me}));
- }
- }, "File", "readAsText", execArgs);
+
+ var totalSize = file.end - file.start;
+ readSuccessCallback.bind(this)("readAsText", enc, file.start, totalSize, function(r) {
+ if (this._progress === 0) {
+ this._result = "";
+ }
+ this._result += r;
+ }.bind(this));
};
@@ -202,59 +238,15 @@ FileReader.prototype.readAsDataURL = function(file) {
return this._realReader.readAsDataURL(file);
}
- var me = this;
- var execArgs = [this._localURL, file.start, file.end];
-
- // Read file
- exec(
- // Success callback
- function(r) {
- // If DONE (cancelled), then don't do anything
- if (me._readyState === FileReader.DONE) {
- return;
- }
-
- // DONE state
- me._readyState = FileReader.DONE;
-
- // Save result
- me._result = r;
-
- // If onload callback
- if (typeof me.onload === "function") {
- me.onload(new ProgressEvent("load", {target:me}));
- }
-
- // If onloadend callback
- if (typeof me.onloadend === "function") {
- me.onloadend(new ProgressEvent("loadend", {target:me}));
- }
- },
- // Error callback
- function(e) {
- // If DONE (cancelled), then don't do anything
- if (me._readyState === FileReader.DONE) {
- return;
- }
-
- // DONE state
- me._readyState = FileReader.DONE;
-
- me._result = null;
-
- // Save error
- me._error = new FileError(e);
-
- // If onerror callback
- if (typeof me.onerror === "function") {
- me.onerror(new ProgressEvent("error", {target:me}));
- }
-
- // If onloadend callback
- if (typeof me.onloadend === "function") {
- me.onloadend(new ProgressEvent("loadend", {target:me}));
- }
- }, "File", "readAsDataURL", execArgs);
+ var totalSize = file.end - file.start;
+ readSuccessCallback.bind(this)("readAsDataURL", null, file.start, totalSize, function(r) {
+ var commaIndex = r.indexOf(',');
+ if (this._progress === 0) {
+ this._result = r;
+ } else {
+ this._result += r.substring(commaIndex + 1);
+ }
+ }.bind(this));
};
/**
@@ -267,58 +259,13 @@ FileReader.prototype.readAsBinaryString = function(file) {
return this._realReader.readAsBinaryString(file);
}
- var me = this;
- var execArgs = [this._localURL, file.start, file.end];
-
- // Read file
- exec(
- // Success callback
- function(r) {
- // If DONE (cancelled), then don't do anything
- if (me._readyState === FileReader.DONE) {
- return;
- }
-
- // DONE state
- me._readyState = FileReader.DONE;
-
- me._result = r;
-
- // If onload callback
- if (typeof me.onload === "function") {
- me.onload(new ProgressEvent("load", {target:me}));
- }
-
- // If onloadend callback
- if (typeof me.onloadend === "function") {
- me.onloadend(new ProgressEvent("loadend", {target:me}));
- }
- },
- // Error callback
- function(e) {
- // If DONE (cancelled), then don't do anything
- if (me._readyState === FileReader.DONE) {
- return;
- }
-
- // DONE state
- me._readyState = FileReader.DONE;
-
- me._result = null;
-
- // Save error
- me._error = new FileError(e);
-
- // If onerror callback
- if (typeof me.onerror === "function") {
- me.onerror(new ProgressEvent("error", {target:me}));
- }
-
- // If onloadend callback
- if (typeof me.onloadend === "function") {
- me.onloadend(new ProgressEvent("loadend", {target:me}));
- }
- }, "File", "readAsBinaryString", execArgs);
+ var totalSize = file.end - file.start;
+ readSuccessCallback.bind(this)("readAsBinaryString", null, file.start, totalSize, function(r) {
+ if (this._progress === 0) {
+ this._result = "";
+ }
+ this._result += r;
+ }.bind(this));
};
/**
@@ -331,61 +278,12 @@ FileReader.prototype.readAsArrayBuffer = function(file) {
return this._realReader.readAsArrayBuffer(file);
}
- var me = this;
- var execArgs = [this._localURL, file.start, file.end];
-
- // Read file
- exec(
- // Success callback
- function(r) {
- // If DONE (cancelled), then don't do anything
- if (me._readyState === FileReader.DONE) {
- return;
- }
-
- // DONE state
- me._readyState = FileReader.DONE;
-
- if (r instanceof Array) {
- r = new Uint8Array(r).buffer;
- }
- me._result = r;
-
- // If onload callback
- if (typeof me.onload === "function") {
- me.onload(new ProgressEvent("load", {target:me}));
- }
-
- // If onloadend callback
- if (typeof me.onloadend === "function") {
- me.onloadend(new ProgressEvent("loadend", {target:me}));
- }
- },
- // Error callback
- function(e) {
- // If DONE (cancelled), then don't do anything
- if (me._readyState === FileReader.DONE) {
- return;
- }
-
- // DONE state
- me._readyState = FileReader.DONE;
-
- me._result = null;
-
- // Save error
- me._error = new FileError(e);
-
- // If onerror callback
- if (typeof me.onerror === "function") {
- me.onerror(new ProgressEvent("error", {target:me}));
- }
-
- // If onloadend callback
- if (typeof me.onloadend === "function") {
- me.onloadend(new ProgressEvent("loadend", {target:me}));
- }
- }, "File", "readAsArrayBuffer", execArgs);
+ var totalSize = file.end - file.start;
+ readSuccessCallback.bind(this)("readAsArrayBuffer", null, file.start, totalSize, function(r) {
+ var resultArray = (this._progress === 0 ? new Uint8Array(totalSize) : new Uint8Array(this._result));
+ resultArray.set(new Uint8Array(r), this._progress);
+ this._result = resultArray.buffer;
+ }.bind(this));
};
module.exports = FileReader;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cordova.apache.org
For additional commands, e-mail: commits-help@cordova.apache.org