[jira] [Created] (COUCHDB-3393) Function lookup() - script can ask for documents!

["Ondřej Novák (JIRA)" <ji...@apache.org>]

Ondřej Novák created COUCHDB-3393:
-------------------------------------

             Summary: Function lookup() - script can ask for documents!
                 Key: COUCHDB-3393
                 URL: https://issues.apache.org/jira/browse/COUCHDB-3393
             Project: CouchDB
          Issue Type: New Feature
          Components: Database Core
            Reporter: Ondřej Novák


This is not just a feature request, this is proof of concept. Working with query servers, i wondered why the functions list() and show() cannot asks for additional documents. Sticking with one document (show) or one view(list) very limiting these rendering functions. 

CouchDB also doesn't support join operation with more then one reference hop and only with include_docs (which is not supported for reduce). Giving to the script some lookup function can break many these limitations (INCLUDING REDUCED RESULTS!)

After some research, i found a way, how the query server can ask the the database for the documents. So I downloaded couchdb sources and tried to implement my idea. And it worked.

From the javascript, there is a new function available for the Render section (show, list, update).
{noformat}
function lookup(docs[])  -> array
{noformat}

The function accepts list of doc-ids. It returns whole document for every doc-id in the array. Asking for non-existing documen, or deleted document causes that null is returned.

How is this implemented?

The query server can return a command instead of a regular response.
{noformat}
 ["lookup", [docs]] 
{noformat}
The couchdb performs lookup for the specified documents and returns them through the proc_prompt back to the query server. Then the couchdb repeats the waiting for the regular response. The above situation can repeat as many times as needed.

The following log was created on my development version of couchdb. You can see "lookup" function in the action.

{noformat}
[debug] [<0.163.0>] OS Process #Port<0.3023> Input  :: ["ddoc","_design/example",["shows","test"],[null,{..}]]
[debug] [<0.163.0>] OS Process #Port<0.3023> Output :: ["lookup",["doc1","doc2","doc3"]]
[debug] [<0.163.0>] OS Process #Port<0.3023> Input  :: [true,[{"_id":"doc1","_rev":"1-7ef1a0ad6b5aae5c716ed7969c87fe97","payload":"aaa"},null,null]]
[debug] [<0.163.0>] OS Process #Port<0.3023> Output :: ["resp",{"body":"[{\"_id\":\"doc1\",\"_rev\":\"1-7ef1a0ad6b5aae5c716ed7969c87fe97\",\"payload\":\"aaa\"},null,null]"}]
{noformat}

I hope this new feature can improve scripting a lot!

I have implemented this feature in the branch 1.6.x, because i have this version in the production. You can find the patch below. It would be probably easy to port the patch to the master branch.

I would appreciate if somebody look at this concept and perhaps improve my implementation and include it to the future version.

Diff to 1.6.x
{noformat}
diff --git a/share/server/loop.js b/share/server/loop.js
index 644d89b..886afb3 100644
--- a/share/server/loop.js
+++ b/share/server/loop.js
@@ -28,6 +28,7 @@ function init_sandbox() {
     sandbox.send = Render.send;
     sandbox.getRow = Render.getRow;
     sandbox.isArray = isArray;
+    sandbox.lookup = Render.lookup;
   } catch (e) {
     //log(e.toSource());
   }
diff --git a/share/server/render.js b/share/server/render.js
index 49b0863..3384fd0 100644
--- a/share/server/render.js
+++ b/share/server/render.js
@@ -195,6 +195,18 @@ var Render = (function() {
     return json[1];
   };
 
+
+  function lookup(docs) {
+      respond(["lookup", docs]);
+      var json = JSON.parse(readline());
+      if (json[0] == true) {
+         return json[1];
+      } else {
+         return null;
+      }
+    };
+
+  
   
   function maybeWrapResponse(resp) {
     var type = typeof resp;
@@ -337,6 +349,7 @@ var Render = (function() {
     start : start,
     send : send,
     getRow : getRow,
+    lookup: lookup,
     show : function(fun, ddoc, args) {
       // var showFun = Couch.compileFunction(funSrc);
       runShow(fun, ddoc, args);
diff --git a/src/couch_mrview/src/couch_mrview_show.erl b/src/couch_mrview/src/couch_mrview_show.erl
index f8fa837..fe1eaca 100644
--- a/src/couch_mrview/src/couch_mrview_show.erl
+++ b/src/couch_mrview/src/couch_mrview_show.erl
@@ -86,8 +86,7 @@ handle_doc_show(Req, Db, DDoc, ShowName, Doc, DocId) ->
     couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
         JsonReq = couch_httpd_external:json_req_obj(Req, Db, DocId),
         JsonDoc = couch_query_servers:json_doc(Doc),
-        [<<"resp">>, ExternalResp] =
-            couch_query_servers:ddoc_prompt(DDoc, [<<"shows">>, ShowName],
+        [<<"resp">>, ExternalResp] = couch_query_servers:ddoc_prompt_wlookup(Db,DDoc, [<<"shows">>, ShowName],
                 [JsonDoc, JsonReq]),
         JsonResp = apply_etag(ExternalResp, CurrentEtag),
         couch_httpd_external:send_external_response(Req, JsonResp)
@@ -132,7 +131,7 @@ send_doc_update_response(Req, Db, DDoc, UpdateName, Doc, DocId) ->
     JsonReq = couch_httpd_external:json_req_obj(Req, Db, DocId),
     JsonDoc = couch_query_servers:json_doc(Doc),
     Cmd = [<<"updates">>, UpdateName],
-    UpdateResp = couch_query_servers:ddoc_prompt(DDoc, Cmd, [JsonDoc, JsonReq]),
+    UpdateResp = couch_query_servers:ddoc_prompt_wlookup(Db,DDoc, Cmd, [JsonDoc, JsonReq]),
     JsonResp = case UpdateResp of
         [<<"up">>, {NewJsonDoc}, {JsonResp0}] ->
             case couch_httpd:header_value(
@@ -238,13 +237,13 @@ list_cb({row, Row}, #lacc{code=undefined} = Acc) ->
 list_cb({row, Row}, Acc) ->
     send_list_row(Row, Acc);
 list_cb(complete, Acc) ->
-    #lacc{qserver = {Proc, _}, resp = Resp0} = Acc,
+    #lacc{qserver = {Proc, _}, resp = Resp0, db = Db} = Acc,
     if Resp0 =:= nil ->
         {ok, #lacc{resp = Resp}} = start_list_resp({[]}, Acc);
     true ->
         Resp = Resp0
     end,
-    case couch_query_servers:proc_prompt(Proc, [<<"list_end">>]) of
+    case couch_query_servers:proc_prompt_wlookup(Db, Proc, [<<"list_end">>]) of
         [<<"end">>, Data, Headers] ->
             Acc2 = fixup_headers(Headers, Acc#lacc{resp=Resp}),
             #lacc{resp = Resp2} = send_non_empty_chunk(Acc2, Data);
@@ -258,7 +257,7 @@ start_list_resp(Head, Acc) ->
     #lacc{db=Db, req=Req, qserver=QServer, lname=LName} = Acc,
     JsonReq = couch_httpd_external:json_req_obj(Req, Db),
 
-    [<<"start">>,Chunk,JsonResp] = couch_query_servers:ddoc_proc_prompt(QServer,
+    [<<"start">>,Chunk,JsonResp] = couch_query_servers:ddoc_proc_prompt_wlookup(Db, QServer,
         [<<"lists">>, LName], [Head, JsonReq]),
     Acc2 = send_non_empty_chunk(fixup_headers(JsonResp, Acc), Chunk),
     {ok, Acc2}.
@@ -273,7 +272,7 @@ fixup_headers(Headers, #lacc{etag=ETag} = Acc) ->
     Headers3 = couch_httpd_external:default_or_content_type(CType, ExtHeaders),
     Acc#lacc{code=Code, headers=Headers3}.
 
-send_list_row(Row, #lacc{qserver = {Proc, _}, resp = Resp} = Acc) ->
+send_list_row(Row, #lacc{qserver = {Proc, _}, resp = Resp, db=Db} = Acc) ->
     RowObj = case couch_util:get_value(id, Row) of
         undefined -> [];
         Id -> [{id, Id}]
@@ -287,7 +286,7 @@ send_list_row(Row, #lacc{qserver = {Proc, _}, resp = Resp} = Acc) ->
         undefined -> [];
         Doc -> [{doc, Doc}]
     end,
-    try couch_query_servers:proc_prompt(Proc, [<<"list_row">>, {RowObj}]) of
+    try couch_query_servers:proc_prompt_wlookup(Db,Proc, [<<"list_row">>, {RowObj}]) of
     [<<"chunks">>, Chunk, Headers] ->
         Acc2 = send_non_empty_chunk(fixup_headers(Headers, Acc), Chunk),
         {ok, Acc2};
diff --git a/src/couchdb/couch_query_servers.erl b/src/couchdb/couch_query_servers.erl
index 3b58cbe..959fd41 100644
--- a/src/couchdb/couch_query_servers.erl
+++ b/src/couchdb/couch_query_servers.erl
@@ -22,6 +22,7 @@
 -export([filter_view/3]).
 
 -export([with_ddoc_proc/2, proc_prompt/2, ddoc_prompt/3, ddoc_proc_prompt/3, json_doc/1]).
+-export([proc_prompt_wlookup/3, ddoc_prompt_wlookup/4, ddoc_proc_prompt_wlookup/4]).
 
 % For 210-os-proc-pool.t
 -export([get_os_process/1, ret_os_process/1]).
@@ -237,6 +238,10 @@ json_doc(nil) -> null;
 json_doc(Doc) ->
     couch_doc:to_json_obj(Doc, [revs]).
 
+json_doc(nil, Options) -> null;
+json_doc(Doc, Options) ->
+    couch_doc:to_json_obj(Doc, Options).
+
 filter_view(DDoc, VName, Docs) ->
     JsonDocs = [couch_doc:to_json_obj(Doc, [revs]) || Doc <- Docs],
     [true, Passes] = ddoc_prompt(DDoc, [<<"views">>, VName, <<"map">>], [JsonDocs]),
@@ -257,6 +262,14 @@ filter_docs(Req, Db, DDoc, FName, Docs) ->
 ddoc_proc_prompt({Proc, DDocId}, FunPath, Args) ->
     proc_prompt(Proc, [<<"ddoc">>, DDocId, FunPath, Args]).
 
+ddoc_proc_prompt_wlookup(Db, {Proc, DDocId}, FunPath, Args) ->
+    proc_prompt_wlookup(Db, Proc, [<<"ddoc">>, DDocId, FunPath, Args]).
+
+ddoc_prompt_wlookup(Db, DDoc, FunPath, Args) ->
+    with_ddoc_proc(DDoc, fun({Proc, DDocId}) ->
+        proc_prompt_wlookup(Db, Proc, [<<"ddoc">>, DDocId, FunPath, Args])
+    end).
+	
 ddoc_prompt(DDoc, FunPath, Args) ->
     with_ddoc_proc(DDoc, fun({Proc, DDocId}) ->
         proc_prompt(Proc, [<<"ddoc">>, DDocId, FunPath, Args])
@@ -507,6 +520,25 @@ proc_with_ddoc(DDoc, DDocKey, LangProcs) ->
             {ok, SmartProc}
     end.
 
+maybe_open_doc(Db, DocId) ->
+    case catch couch_httpd_db:couch_doc_open(Db, DocId, nil, [conflicts]) of
+        #doc{} = Doc -> Doc;
+        {not_found, _} -> nil
+    end.
+
+
+do_lookup(Db, DocList) ->
+    lists:map(fun(X) -> json_doc(maybe_open_doc(Db, X),[]) end, DocList ).
+
+proc_prompt_wlookup(Db, Proc, Args) ->
+	case proc_prompt(Proc,Args) of
+		[<<"lookup">>, DocList] ->
+			proc_prompt_wlookup(Db,Proc,[true, do_lookup(Db, DocList)]);
+		Other ->
+			Other
+	end.
+	
+
 proc_prompt(Proc, Args) ->
      case proc_prompt_raw(Proc, Args) of
      {json, Json} ->


{noformat}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)