You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by "CharlieFRuan (via GitHub)" <gi...@apache.org> on 2024/03/25 17:16:02 UTC
Re: [PR] [Web] Support web indexDB cache for larger model storage [tvm]
CharlieFRuan commented on code in PR #16733:
URL: https://github.com/apache/tvm/pull/16733#discussion_r1537852720
##########
web/src/index.ts:
##########
@@ -22,7 +22,7 @@ export {
PackedFunc, Module, NDArray,
TVMArray, TVMObject, VirtualMachine,
InitProgressCallback, InitProgressReport,
- ArtifactCache, Instance, instantiate, hasNDArrayInCache, deleteNDArrayCache
+ ArtifactCache, ArtifactIndexDBCache, Instance, instantiate, hasNDArrayInCache, deleteNDArrayCache
Review Comment:
I think the cache is called `indexedDB` instead of `indexDB`, let's change all the namings.
##########
web/src/runtime.ts:
##########
@@ -1052,6 +1062,191 @@ export class ArtifactCache implements ArtifactCacheTemplate {
}
}
+/**
+ * Cache by IndexDB to support caching model data
+ */
+export class ArtifactIndexDBCache implements ArtifactCacheTemplate {
+ private dbName?: string;
+ private dbVersion = 1;
+ private db: IDBDatabase | undefined;
+
+ private async initDB() {
+ if (this.db != null){
+ return; // the db is already inialized
+ }
+ return new Promise<void>((resolve, reject) => {
+ const request = indexedDB.open(this.dbName, this.dbVersion);
+ request.onupgradeneeded = (event) => {
+ this.db = (event.target as IDBOpenDBRequest).result;
+ if (!this.db.objectStoreNames.contains('urls')) {
+ this.db.createObjectStore('urls', { keyPath: 'url' });
+ }
+ };
+ request.onsuccess = (event) => {
+ this.db = (event.target as IDBOpenDBRequest).result;
+ resolve();
+ };
+ request.onerror = (event) => {
+ console.error("Database error: ", (event.target as IDBOpenDBRequest).error);
+ reject((event.target as IDBOpenDBRequest).error);
+ };
+ });
+ }
+
+ /* Check if the URL is in DB or not */
+ private async isUrlInDB(url: string): Promise<boolean> {
+ return new Promise<boolean>((resolve, reject) => {
+ const transaction = this.db?.transaction(['urls'], 'readonly');
+ if (transaction === undefined){
+ return false;
+ }
+ const store = transaction.objectStore('urls');
+ const request = store.get(url);
+ request.onsuccess = () => {
+ resolve(request.result !== undefined);
+ };
+ request.onerror = (event) => {
+ reject((event.target as IDBRequest).error);
+ };
+ });
+ }
+
+ constructor(dbName: string){
+ this.dbName = dbName;
+ }
Review Comment:
Let's move the constructor to the top
##########
web/src/runtime.ts:
##########
@@ -1052,6 +1062,191 @@ export class ArtifactCache implements ArtifactCacheTemplate {
}
}
+/**
+ * Cache by IndexDB to support caching model data
+ */
+export class ArtifactIndexDBCache implements ArtifactCacheTemplate {
+ private dbName?: string;
+ private dbVersion = 1;
+ private db: IDBDatabase | undefined;
+
+ private async initDB() {
+ if (this.db != null){
+ return; // the db is already inialized
+ }
+ return new Promise<void>((resolve, reject) => {
+ const request = indexedDB.open(this.dbName, this.dbVersion);
+ request.onupgradeneeded = (event) => {
+ this.db = (event.target as IDBOpenDBRequest).result;
+ if (!this.db.objectStoreNames.contains('urls')) {
+ this.db.createObjectStore('urls', { keyPath: 'url' });
+ }
+ };
+ request.onsuccess = (event) => {
+ this.db = (event.target as IDBOpenDBRequest).result;
+ resolve();
+ };
+ request.onerror = (event) => {
+ console.error("Database error: ", (event.target as IDBOpenDBRequest).error);
+ reject((event.target as IDBOpenDBRequest).error);
+ };
+ });
+ }
+
+ /* Check if the URL is in DB or not */
+ private async isUrlInDB(url: string): Promise<boolean> {
+ return new Promise<boolean>((resolve, reject) => {
+ const transaction = this.db?.transaction(['urls'], 'readonly');
+ if (transaction === undefined){
+ return false;
+ }
+ const store = transaction.objectStore('urls');
+ const request = store.get(url);
+ request.onsuccess = () => {
+ resolve(request.result !== undefined);
+ };
+ request.onerror = (event) => {
+ reject((event.target as IDBRequest).error);
+ };
+ });
+ }
+
+ constructor(dbName: string){
+ this.dbName = dbName;
+ }
+
+ async asyncGetHelper(url: string){
+ return new Promise((resolve, reject) => {
+ let result: any;
+ const transaction = this.db?.transaction(['urls'], 'readonly');
+ if (transaction === undefined){
+ return false;
+ }
+ transaction.oncomplete = () => resolve(result);
+ transaction.onerror = () => reject(transaction.error);
+ const objectStore = transaction.objectStore('urls');
+ const getRequest = objectStore.get(url);
+ getRequest.onsuccess = () => {
+ result = getRequest.result;
+ }
+ })
+ }
+
+ async fetchWithCache(url: string, storetype?: string) {
+ await this.initDB(); // await the initDB process
+ const isInDB = await this.isUrlInDB(url);
+ if (!isInDB) {
+ const response = await this.addToCache(url, storetype);
+ return response;
+ } else {
+ // URL found in DB, just fetch without storing
+ const result = await this.asyncGetHelper(url);
+ if (result != null && typeof result === "object" && "data" in result){
+ return result.data;
+ } else if (result === null){
+ // previously null data in cache!
+ await this.deleteInCache(url);
+ const response = await this.addToCache(url, storetype);
+ return response;
+ }
+ return null;
+ }
+ }
+
+ async addToIndexDB(url: string, response: any, storetype?: string){
+ await this.initDB();
+ let data: any;
+ if (storetype != undefined){
+ if (storetype.toLowerCase() === "json"){
+ data = await response.json();
+ } else if (storetype.toLocaleLowerCase() === "arraybuffer"){
+ data = await response.arrayBuffer();
+ } else {
+ console.error("Unsupported Type in IndexDB");
+ }
+ }
+ return new Promise<void>((resolve, reject) => {
+ const transaction = this.db?.transaction(['urls'], 'readwrite');
+ if (transaction === undefined){
+ return;
+ }
+ const store = transaction.objectStore('urls');
+ const request = store.add({data, url}); // Index DB follows a {value, key} format, instead of {key, value} format!
+ request.onsuccess = () => resolve();
+ request.onerror = (event) => reject((event.target as IDBRequest).error);
+ });
+ }
+
+ async addToCache(url: string, storetype?: string) :Promise<any>{
+ let response: Response;
+ try {
+ response = await fetch(url);
+ if (!response.ok) {
+ throw new Error('Network response was not ok');
+ }
+ const response_copy = response.clone();
+ await this.addToIndexDB(url, response_copy, storetype);
+ if (storetype.toLowerCase() === "arraybuffer"){
+ return await response.arrayBuffer();
+ } else if (storetype.toLowerCase() == "json"){
+ return await response.json();
+ } else {
+ return response;
+ }
+ } catch (error) {
+ console.error("There was a problem fetching the data:", error);
+ }
+ }
+
+ async hasAllKeys(keys: string[]) :Promise<boolean> {
Review Comment:
When testing end-to-end with WebLLM, this seems to be always returning true (we can tell by how the `simple-chat` example always prints `Loading model from cache` rather than `fetching param cache`, even if we are downloading the model for the first time. Which means that the `cacheOnly` variable in `fetchNDArrayCacheInternal` is always `true`, which is why the downloading speed is quite slow -- it is never entering `downloadCache()` for parallel download.
Let's check the implementation of this.
##########
web/src/runtime.ts:
##########
@@ -1052,6 +1062,191 @@ export class ArtifactCache implements ArtifactCacheTemplate {
}
}
+/**
+ * Cache by IndexDB to support caching model data
+ */
+export class ArtifactIndexDBCache implements ArtifactCacheTemplate {
+ private dbName?: string;
+ private dbVersion = 1;
+ private db: IDBDatabase | undefined;
+
+ private async initDB() {
+ if (this.db != null){
+ return; // the db is already inialized
+ }
+ return new Promise<void>((resolve, reject) => {
+ const request = indexedDB.open(this.dbName, this.dbVersion);
+ request.onupgradeneeded = (event) => {
+ this.db = (event.target as IDBOpenDBRequest).result;
+ if (!this.db.objectStoreNames.contains('urls')) {
+ this.db.createObjectStore('urls', { keyPath: 'url' });
+ }
+ };
+ request.onsuccess = (event) => {
+ this.db = (event.target as IDBOpenDBRequest).result;
+ resolve();
+ };
+ request.onerror = (event) => {
+ console.error("Database error: ", (event.target as IDBOpenDBRequest).error);
+ reject((event.target as IDBOpenDBRequest).error);
+ };
+ });
+ }
+
+ /* Check if the URL is in DB or not */
+ private async isUrlInDB(url: string): Promise<boolean> {
+ return new Promise<boolean>((resolve, reject) => {
+ const transaction = this.db?.transaction(['urls'], 'readonly');
+ if (transaction === undefined){
+ return false;
+ }
+ const store = transaction.objectStore('urls');
+ const request = store.get(url);
+ request.onsuccess = () => {
+ resolve(request.result !== undefined);
+ };
+ request.onerror = (event) => {
+ reject((event.target as IDBRequest).error);
+ };
+ });
+ }
+
+ constructor(dbName: string){
+ this.dbName = dbName;
+ }
+
+ async asyncGetHelper(url: string){
+ return new Promise((resolve, reject) => {
+ let result: any;
+ const transaction = this.db?.transaction(['urls'], 'readonly');
+ if (transaction === undefined){
+ return false;
+ }
+ transaction.oncomplete = () => resolve(result);
+ transaction.onerror = () => reject(transaction.error);
+ const objectStore = transaction.objectStore('urls');
+ const getRequest = objectStore.get(url);
+ getRequest.onsuccess = () => {
+ result = getRequest.result;
+ }
+ })
+ }
+
+ async fetchWithCache(url: string, storetype?: string) {
+ await this.initDB(); // await the initDB process
+ const isInDB = await this.isUrlInDB(url);
+ if (!isInDB) {
+ const response = await this.addToCache(url, storetype);
+ return response;
+ } else {
+ // URL found in DB, just fetch without storing
+ const result = await this.asyncGetHelper(url);
+ if (result != null && typeof result === "object" && "data" in result){
+ return result.data;
+ } else if (result === null){
+ // previously null data in cache!
+ await this.deleteInCache(url);
+ const response = await this.addToCache(url, storetype);
+ return response;
+ }
+ return null;
+ }
+ }
+
+ async addToIndexDB(url: string, response: any, storetype?: string){
+ await this.initDB();
+ let data: any;
+ if (storetype != undefined){
+ if (storetype.toLowerCase() === "json"){
+ data = await response.json();
+ } else if (storetype.toLocaleLowerCase() === "arraybuffer"){
+ data = await response.arrayBuffer();
+ } else {
+ console.error("Unsupported Type in IndexDB");
+ }
+ }
+ return new Promise<void>((resolve, reject) => {
+ const transaction = this.db?.transaction(['urls'], 'readwrite');
+ if (transaction === undefined){
+ return;
+ }
+ const store = transaction.objectStore('urls');
+ const request = store.add({data, url}); // Index DB follows a {value, key} format, instead of {key, value} format!
+ request.onsuccess = () => resolve();
+ request.onerror = (event) => reject((event.target as IDBRequest).error);
+ });
+ }
+
+ async addToCache(url: string, storetype?: string) :Promise<any>{
+ let response: Response;
+ try {
+ response = await fetch(url);
+ if (!response.ok) {
+ throw new Error('Network response was not ok');
+ }
+ const response_copy = response.clone();
+ await this.addToIndexDB(url, response_copy, storetype);
+ if (storetype.toLowerCase() === "arraybuffer"){
+ return await response.arrayBuffer();
Review Comment:
Should `addToCache()` return the result, or just simply add the artifact to the cache? The behavior here differs from that of `ArtifactCache`. We should unify the behavior; adding return types to the interface in `artifact_cache.ts` would be good.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@tvm.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org