[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...

[gilv <gi...@git.apache.org>]

Github user gilv commented on a diff in the pull request:

    https://github.com/apache/couchdb-fabric/pull/33#discussion_r48705157
  
    --- Diff: src/fabric_attachments_handler.erl ---
    @@ -0,0 +1,333 @@
    +%% @author gilv
    +%% @doc @todo Add description to fabric_swift_handler.
    +
    +
    +-module(fabric_attachments_handler).
    +
    +-include_lib("fabric/include/fabric.hrl").
    +-include_lib("couch/include/couch_db.hrl").
    +
    +-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
    +
    +
    +%% ====================================================================
    +%% External API functions. I try to keep them as general as possible, 
    +%% without correlation to specific object store that might be internally used.
    +%% ====================================================================
    +
    +normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
    +    ContainerName = container_name(Db),
    +    couch_log:debug("Standard attachment handler",[]),
    +    couch_log:debug("Going to store ~p of length is ~p,  ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
    +    %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
    +    Data = couch_httpd:recv(Req, ContentLen),
    +    case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
    +        {ok,{201,NewUrl}} ->
    +            couch_log:debug("Created. Swift response code is 201 ~n",[]),
    +            {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
    +            {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
    +        {_,{Code,_}} ->
    +            couch_log:debug("Swift response code is ~p~n",[]),
    +            {Data, -1}
    +    end.
    +
    +inline_att_store(Db, Docs) ->
    +    DbName = container_name(Db),
    +    couch_log:debug("Store inline base64 encoded attachment ~n",[]),
    +    if 
    +        is_list(Docs)  ->
    +            couch_log:debug("Going to handle document list",[]),
    +            DocsArray = Docs;
    +        true ->
    +            couch_log:debug("Going to handle single document",[]),
    +            DocsArray = [Docs]
    +    end,
    +    [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
    +    if 
    +        length(Atts0) > 0 ->
    +            couch_log:debug("Att length is larger than 0",[]),
    +            Atts = [att_processor(DbName,Att) || Att <- Atts0],
    +            if 
    +                is_list(Docs)  ->
    +                    [Doc#doc{atts = Atts} | Rest];
    +            true ->
    +                Doc#doc{atts = Atts}
    +            end;
    +        true ->
    +            couch_log:debug("No attachments to handle",[]),
    +            Docs
    +    end.
    +
    +inline_att_handler(get, Db, Att) ->
    +    DbName = container_name(Db),
    +    couch_log:debug("Retrieve attachment",[]),
    +    [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
    +    couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
    +    NewData = swift_get_object(DbName, Name),
    +    NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
    +    NewAtt;
    +
    +inline_att_handler(delete,Db, Att) ->
    +    DbName = container_name(Db),
    +    couch_log:debug("Delete attachment ~p~n",[]).
    +
    +container_handler(create,DbName) ->
    +    couch_log:debug("Create container ~p~n",[DbName]),
    +    case swift_create_container(DbName) of
    +        {ok,{201,Container}} ->
    +            couch_log:debug("Container ~p created succesfully ~n",[Container]),
    +            {ok,Container};
    +        {error,_} ->
    +            couch_log:debug("Container ~p creation failed ~n",[DbName]),
    +            {error,DbName}
    +    end;
    +container_handler(get,DbName) ->
    +    couch_log:debug("Get container ~p~n",[DbName]);
    +container_handler(delete,DbName)->
    +    couch_log:debug("Delete container ~p~n",[DbName]),
    +    swift_delete_container(DbName).
    +
    +externalize_att(Db) ->
    +    Res = config:get("swift","attachments_offload","false"),
    +    Res.
    +
    +%% ====================================================================
    +%% Internal general functions
    +%% ====================================================================
    +
    +container_name(Db) ->
    +    Suffix = list_to_binary(mem3:shard_suffix(Db)),
    +    DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
    +    couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),	
    +    DbNameSuffix.
    +
    +hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
    --- End diff --
    
    @kxepal I am using hex_to_bin to transform Etag returned from Swift to the same format as it used by CouchDB.
    
    I did some tests
        A = hex_to_bin("39345d76817774864639d2bafb7c4551"),
        B = couch_util: to_hex("39345d76817774864639d2bafb7c4551"),
    
    They produce different results. B is not what is expected.
    A  = 4]v�wt�F9Һ� 
    B = 3339333435643736383137373734383634363339643262616662376334353531


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---