You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@couchdb.apache.org by gilv <gi...@git.apache.org> on 2015/10/14 13:04:49 UTC
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
GitHub user gilv opened a pull request:
https://github.com/apache/couchdb-fabric/pull/33
COUCHDB-769: Store attachments in the external storage.
Initial implementation that allows CouchDB to store attachments outside of the database file.
This implementation supports OpenStack Swift and SoftLayer Object store
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/gilv/couchdb-fabric store_attachments_external
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/couchdb-fabric/pull/33.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #33
----
commit d029d3ae64a047131b022f52126a8bf7f765f823
Author: Gil Vernik <gi...@il.ibm.com>
Date: 2015-10-14T11:02:40Z
COUCHDB-769: Store attachments in external storage. This is initial implementation, supports OpenStack Swift and SoftLayer Object store
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48718488
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
+ {error, _} ->
+ {error, Data}
+ end;
+ {error, _} ->
+ {error, Data}
+ end.
+
+att_store(Db, #doc{}=Doc) ->
+ att_store(Db, [Doc]);
+att_store(Db, Docs) when is_list(Docs) ->
+ DbName = container_name(Db),
+ [#doc{atts=Atts0} = Doc | Rest] = Docs,
+ if
+ length(Atts0) > 0 ->
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Docs
+ end.
+
+att(get, Db, Att) ->
+ DbName = container_name(Db),
+ [Name,AttLen,AttLenUser] = couch_att:fetch([name, att_len,
+ att_external_size], Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, "
+ ++ "stored length: ~p~n", [Name, AttLen, AttLenUser]),
+ NewData = (get_backend_driver()):get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser},
+ {disk_len, AttLenUser}], Att),
+ NewAtt;
+att(delete,Db, Att) ->
+ DbName = container_name(Db),
+ [Name] = couch_att:fetch([name],Att),
+ couch_log:debug("Delete ~p from ~p", [Name, DbName]).
+
+container(create,DbName) ->
+ couch_log:debug("Create container ~p~n", [DbName]),
+ case (get_backend_driver()):create_container(DbName) of
+ {ok, {"201", Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n", [Container]),
+ {ok, Container};
+ {error, _} ->
+ couch_log:debug("Container ~p creation failed ~n", [DbName]),
+ {error, DbName}
+ end;
+container(get, DbName) ->
+ couch_log:debug("Get container ~p~n", [DbName]);
+container(delete, DbName)->
+ couch_log:debug("Delete container ~p~n", [DbName]),
+ (get_backend_driver()):delete_container(DbName).
+
+external_store() ->
+ config:get_boolean("ext_store", "attachments_offload", false).
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+get_backend_driver() ->
+ case config:get("ext_store", "active_store") of
+ "swift" ->
+ fabric_swift_driver;
+ undefined ->
+ fabric_swift_driver
+ end.
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ DbNameSuffix.
+
+att_processor(DbName, Att) ->
+ [Name, Data, Type, Enc] = couch_att:fetch([name, data, type, encoding], Att),
+ couch_log:debug("Att name: ~p, Type: ~p, Encoding: ~p ~n", [Name, Type, Enc]),
+ case is_binary(Data) of
+ true ->
+ case (get_backend_driver()):put_object(DbName, Name, Type, [], Data) of
+ {ok, {"201", NewUrl}} ->
--- End diff --
Another backend leak: 201 status (and the fact of http request behind) is an backend implementation details. Others may use different statuses and different protocols.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48718748
--- Diff: src/fabric_swift_driver.erl ---
@@ -0,0 +1,269 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_swift_driver).
+
+-export([put_object/5, delete_object/2, get_object/2, head_object/2]).
+-export([store_id/0, create_container/1, delete_container/1 ]).
+%% ====================================================================
+%% OpenStack Swift implementation
+%% ====================================================================
+
+store_id() ->
+ "swift".
+
+put_object(Container, ObjName, ContextType, CustomHeaders, Data) ->
+ couch_log:debug("Swift: PUT ~p/~s object ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Headers = CustomHeaders ++ [{"X-Auth-Token", Storage_Token},
+ {"Content-Type",
+ unicode:characters_to_list(ContextType)}],
+ Method = put,
+ couch_log:debug("Swift: PUT : ~p ~p ~p ~p~n",
+ [Url,ContextType, ObjNameEncoded, Container]),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: PUT url: ~p~n",[NewUrl]),
+ R = ibrowse:send_req(NewUrl, Headers, Method, Data),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ {ok, {ReturnCode, NewUrl}};
+ Error ->
+ couch_log:debug("PUT object failed ~p ~n", [element(2, Error)]),
+ {error, "PUT object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+delete_object(Container, ObjName) ->
+ %TO-DO: should be called during database compaction.
+ couch_log:debug("Swift: Delete ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = delete,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/"
+ ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ couch_log:debug("Swift: Delete ~p. Return code ~p ~n",
+ [NewUrl, ReturnCode]),
+ {ok, ReturnCode};
+ Error ->
+ couch_log:debug("Swift: Delete ~p failed. ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error, "Delete object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+get_object(Container, ObjName) ->
+ couch_log:debug("Swift: get object ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = get,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/"
+ ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, Body} ->
+ couch_log:debug("Swift: GET ~p with return code ~p ~n", [NewUrl, ReturnCode]),
+ Body;
+ Error ->
+ couch_log:debug("Swift: GET ~p failed. ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error, "Get object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+head_object(Container, ObjName) ->
+ couch_log:debug("Swift: head object ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ Method = head,
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container)
+ ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, Head, _} ->
+ couch_log:debug("Swift: ~p~p~n", [ReturnCode, Head]),
+ ObjectSize = lists:filter(fun ({"Content-Length", _}) -> true; (_) -> false end, Head),
+ EtagHeader = lists:filter(fun ({"Etag", _}) -> true ; (_) -> false end, Head),
+ Etag = element(2, lists:nth(1, EtagHeader)),
+ {ObjectSizeNumeric, _} = string:to_integer(element(2, lists:nth(1, ObjectSize))),
+ couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n",
+ [ObjectSizeNumeric, Etag]),
+ EtagDecode = hex_to_bin(Etag),
+ EtagMD5 = base64:encode(EtagDecode),
+ couch_log:debug("Etag in base64 ~p~n", [EtagMD5]),
+ {ok, {ObjectSizeNumeric, EtagMD5}};
+ Error ->
+ couch_log:debug("Swift: Head ~p failed ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error,"HEAD object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+create_container(DbName) ->
+ couch_log:debug("Swift : create container ~p~n", [DbName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ Container = DbName,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++"/",
+ couch_log:debug("Swift: url ~p ~n",[NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ couch_log:debug("Swift: container ~p created with code : ~p~n",[Container, ReturnCode]),
+ {ok,{ReturnCode,Container}};
+ Error ->
+ couch_log:debug("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]),
+ {error, "Failed to create container"}
+ end;
+ {not_authenticated,_} ->
+ {error, "Failed to create container. Not authenticated"}
+ end.
+
+delete_container(Container) ->
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = delete,
+ NewUrl = Url ++ "/" ++ Container ++"/",
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ {ok,{ReturnCode,Container}};
+ Error ->
+ {error,{element(2,Error),Container}}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+%% ====================================================================
+%% Internal functions
+%% ====================================================================
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+authenticate() ->
+ couch_log:debug("Going to authenticate in Swift",[]),
+ case config:get("swift", "auth_model", "tempauth") of
+ "tempauth" ->
+ swift_v1_auth();
+ "keystone" ->
+ keystone_auth();
+ Error ->
+ couch_log:debug("Authentication method is not supported: ~p", element(2,Error)),
+ {not_authenticated, ""}
+ end.
+
+keystone_auth() ->
+ Method = post,
+ URL = config:get("swift", "auth_url"),
+ Header = [{"Content-Type", "application/json"}],
+ Tenant = config:get("swift", "account_tenant"),
+ User = config:get("swift", "username"),
+ Psw = config:get("swift", "password"),
+ Body = "{\"auth\": {\"tenantName\": \"" ++ Tenant
+ ++ "\", \"passwordCredentials\": {\"username\": \""
+ ++ User ++ "\", \"password\": \"" ++ Psw ++ "\"}}}",
--- End diff --
How would it works if my password will contain non ascii characters or even just double quote?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-148027523
Would be nice to see some tests for this to prove that this all works.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48717679
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
+ {error, _} ->
+ {error, Data}
+ end;
+ {error, _} ->
+ {error, Data}
+ end.
+
+att_store(Db, #doc{}=Doc) ->
+ att_store(Db, [Doc]);
+att_store(Db, Docs) when is_list(Docs) ->
+ DbName = container_name(Db),
+ [#doc{atts=Atts0} = Doc | Rest] = Docs,
+ if
+ length(Atts0) > 0 ->
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Docs
+ end.
+
+att(get, Db, Att) ->
+ DbName = container_name(Db),
+ [Name,AttLen,AttLenUser] = couch_att:fetch([name, att_len,
+ att_external_size], Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, "
+ ++ "stored length: ~p~n", [Name, AttLen, AttLenUser]),
+ NewData = (get_backend_driver()):get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser},
+ {disk_len, AttLenUser}], Att),
+ NewAtt;
+att(delete,Db, Att) ->
+ DbName = container_name(Db),
+ [Name] = couch_att:fetch([name],Att),
+ couch_log:debug("Delete ~p from ~p", [Name, DbName]).
+
+container(create,DbName) ->
+ couch_log:debug("Create container ~p~n", [DbName]),
+ case (get_backend_driver()):create_container(DbName) of
+ {ok, {"201", Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n", [Container]),
+ {ok, Container};
+ {error, _} ->
+ couch_log:debug("Container ~p creation failed ~n", [DbName]),
+ {error, DbName}
+ end;
+container(get, DbName) ->
+ couch_log:debug("Get container ~p~n", [DbName]);
+container(delete, DbName)->
+ couch_log:debug("Delete container ~p~n", [DbName]),
+ (get_backend_driver()):delete_container(DbName).
+
+external_store() ->
+ config:get_boolean("ext_store", "attachments_offload", false).
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+get_backend_driver() ->
+ case config:get("ext_store", "active_store") of
+ "swift" ->
+ fabric_swift_driver;
+ undefined ->
--- End diff --
Need a clause for unknown backend and the way to gracefully handle that.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41982119
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
--- End diff --
Use `couch_util:to_hex` instead.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r49944498
--- Diff: src/fabric_swift_driver.erl ---
@@ -0,0 +1,269 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_swift_driver).
+
+-export([put_object/5, delete_object/2, get_object/2, head_object/2]).
+-export([store_id/0, create_container/1, delete_container/1 ]).
+%% ====================================================================
+%% OpenStack Swift implementation
+%% ====================================================================
+
+store_id() ->
+ "swift".
+
+put_object(Container, ObjName, ContextType, CustomHeaders, Data) ->
+ couch_log:debug("Swift: PUT ~p/~s object ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Headers = CustomHeaders ++ [{"X-Auth-Token", Storage_Token},
+ {"Content-Type",
+ unicode:characters_to_list(ContextType)}],
+ Method = put,
+ couch_log:debug("Swift: PUT : ~p ~p ~p ~p~n",
+ [Url,ContextType, ObjNameEncoded, Container]),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: PUT url: ~p~n",[NewUrl]),
+ R = ibrowse:send_req(NewUrl, Headers, Method, Data),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ {ok, {ReturnCode, NewUrl}};
+ Error ->
+ couch_log:debug("PUT object failed ~p ~n", [element(2, Error)]),
+ {error, "PUT object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+delete_object(Container, ObjName) ->
+ %TO-DO: should be called during database compaction.
+ couch_log:debug("Swift: Delete ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = delete,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/"
+ ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ couch_log:debug("Swift: Delete ~p. Return code ~p ~n",
+ [NewUrl, ReturnCode]),
+ {ok, ReturnCode};
+ Error ->
+ couch_log:debug("Swift: Delete ~p failed. ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error, "Delete object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+get_object(Container, ObjName) ->
+ couch_log:debug("Swift: get object ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = get,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/"
+ ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, Body} ->
+ couch_log:debug("Swift: GET ~p with return code ~p ~n", [NewUrl, ReturnCode]),
+ Body;
+ Error ->
+ couch_log:debug("Swift: GET ~p failed. ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error, "Get object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+head_object(Container, ObjName) ->
+ couch_log:debug("Swift: head object ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ Method = head,
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container)
+ ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, Head, _} ->
+ couch_log:debug("Swift: ~p~p~n", [ReturnCode, Head]),
+ ObjectSize = lists:filter(fun ({"Content-Length", _}) -> true; (_) -> false end, Head),
+ EtagHeader = lists:filter(fun ({"Etag", _}) -> true ; (_) -> false end, Head),
+ Etag = element(2, lists:nth(1, EtagHeader)),
+ {ObjectSizeNumeric, _} = string:to_integer(element(2, lists:nth(1, ObjectSize))),
+ couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n",
+ [ObjectSizeNumeric, Etag]),
+ EtagDecode = hex_to_bin(Etag),
+ EtagMD5 = base64:encode(EtagDecode),
+ couch_log:debug("Etag in base64 ~p~n", [EtagMD5]),
+ {ok, {ObjectSizeNumeric, EtagMD5}};
+ Error ->
+ couch_log:debug("Swift: Head ~p failed ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error,"HEAD object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+create_container(DbName) ->
+ couch_log:debug("Swift : create container ~p~n", [DbName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ Container = DbName,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++"/",
+ couch_log:debug("Swift: url ~p ~n",[NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ couch_log:debug("Swift: container ~p created with code : ~p~n",[Container, ReturnCode]),
+ {ok,{ReturnCode,Container}};
+ Error ->
+ couch_log:debug("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]),
+ {error, "Failed to create container"}
+ end;
+ {not_authenticated,_} ->
+ {error, "Failed to create container. Not authenticated"}
+ end.
+
+delete_container(Container) ->
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = delete,
+ NewUrl = Url ++ "/" ++ Container ++"/",
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ {ok,{ReturnCode,Container}};
+ Error ->
+ {error,{element(2,Error),Container}}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+%% ====================================================================
+%% Internal functions
+%% ====================================================================
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+authenticate() ->
+ couch_log:debug("Going to authenticate in Swift",[]),
+ case config:get("swift", "auth_model", "tempauth") of
+ "tempauth" ->
+ swift_v1_auth();
+ "keystone" ->
+ keystone_auth();
+ Error ->
+ couch_log:debug("Authentication method is not supported: ~p", element(2,Error)),
+ {not_authenticated, ""}
+ end.
+
+keystone_auth() ->
+ Method = post,
+ URL = config:get("swift", "auth_url"),
+ Header = [{"Content-Type", "application/json"}],
+ Tenant = config:get("swift", "account_tenant"),
+ User = config:get("swift", "username"),
+ Psw = config:get("swift", "password"),
+ Body = "{\"auth\": {\"tenantName\": \"" ++ Tenant
+ ++ "\", \"passwordCredentials\": {\"username\": \""
+ ++ User ++ "\", \"password\": \"" ++ Psw ++ "\"}}}",
--- End diff --
@gilv It's 2016 today, right? Still, build correct JSON isn't much harder than deal with possible issues and ask users to manually JSON escape their credentials in config.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41983311
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
+%% ====================================================================
+
+extractAuthInfo([{_,_}|_] = Obj,[])->
+ Storage_URL = proplists:get_value(<<"publicURL">>, Obj),
+ Storage_Token = proplists:get_value(<<"id">>, Obj),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{[{_,_}|_]}] = Obj,[])->
+ [{ObjNext}] = Obj,
+ Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext),
+ Storage_Token = proplists:get_value(<<"id">>, ObjNext),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{_,_}|L]=Obj,KeyValPath)->
+ [{Key, Val}|KeyValPathNext] = KeyValPath,
+ ObjVal = proplists:get_value(Key,Obj),
+ case Val of
+ [] -> extractAuthInfo(ObjVal, KeyValPathNext);
+ ObjVal -> extractAuthInfo(Obj, KeyValPathNext);
+ _ -> ["", "", false]
+ end;
+extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)->
+ [ObjCur| _] = Obj,
+ [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur, KeyValPath),
+ case Status of
+ false -> extractAuthInfo(L, KeyValPath);
+ _ -> [Storage_URL, Storage_Token, Status]
+ end;
+extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)->
+ {Doc} = Obj,
+ extractAuthInfo(Doc, KeyValPath).
+
+att_processor(DbName,Att) ->
+ couch_log:debug("Swift: attachment processor",[]),
+ [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding, disk_len, att_len], Att),
+ [Name, Data] = couch_att:fetch([name, data], Att),
+ couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk len: ~p~n",[Name,Type,Enc,DiskLen]),
+ case is_binary(Data) of
+ true ->
+ couch_log:debug("Swift: binary attachment exists",[]),
+ case swift_put_object(DbName, Name, Type, [], Data) of
+ {ok,{201,NewUrl}} ->
+ N1 = unicode:characters_to_binary(NewUrl, unicode, utf8),
+ couch_log:debug("~p ~p~n",[N1, NewUrl]),
+ NewAtt = couch_att:store(data,N1,Att),
+ couch_log:debug("Swift. testing store in original length ~p~n",[AttLen]),
+ {ObjectSize,EtagMD5} = swift_head_object(DbName, Name),
+ NewAtt1 = couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt),
--- End diff --
Why you store "external" as list, no at atom, or at least as binary?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by lazedo <gi...@git.apache.org>.
Github user lazedo commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-179789279
@kxepal maybe i'm missing the objective of this feature. as i see it, a external attachment would still create the stub information about how to reach the `real` attachment. the `stub`would have the information needed to get the attachment back, so, you would need to store the `driver` in case the configuration is changed and you start using another driver but still want to reach the already stored attachments. the `options` would allow more flexibility to the driver creating the necessary stub information needed to get the attachment back and also to choose which driver we want for the attachment.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r49949144
--- Diff: src/fabric_swift_driver.erl ---
@@ -0,0 +1,269 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_swift_driver).
+
+-export([put_object/5, delete_object/2, get_object/2, head_object/2]).
+-export([store_id/0, create_container/1, delete_container/1 ]).
+%% ====================================================================
+%% OpenStack Swift implementation
+%% ====================================================================
+
+store_id() ->
+ "swift".
+
+put_object(Container, ObjName, ContextType, CustomHeaders, Data) ->
+ couch_log:debug("Swift: PUT ~p/~s object ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Headers = CustomHeaders ++ [{"X-Auth-Token", Storage_Token},
+ {"Content-Type",
+ unicode:characters_to_list(ContextType)}],
+ Method = put,
+ couch_log:debug("Swift: PUT : ~p ~p ~p ~p~n",
+ [Url,ContextType, ObjNameEncoded, Container]),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: PUT url: ~p~n",[NewUrl]),
+ R = ibrowse:send_req(NewUrl, Headers, Method, Data),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ {ok, {ReturnCode, NewUrl}};
+ Error ->
+ couch_log:debug("PUT object failed ~p ~n", [element(2, Error)]),
+ {error, "PUT object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+delete_object(Container, ObjName) ->
+ %TO-DO: should be called during database compaction.
+ couch_log:debug("Swift: Delete ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = delete,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/"
+ ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ couch_log:debug("Swift: Delete ~p. Return code ~p ~n",
+ [NewUrl, ReturnCode]),
+ {ok, ReturnCode};
+ Error ->
+ couch_log:debug("Swift: Delete ~p failed. ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error, "Delete object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+get_object(Container, ObjName) ->
+ couch_log:debug("Swift: get object ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = get,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/"
+ ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, Body} ->
+ couch_log:debug("Swift: GET ~p with return code ~p ~n", [NewUrl, ReturnCode]),
+ Body;
+ Error ->
+ couch_log:debug("Swift: GET ~p failed. ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error, "Get object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+head_object(Container, ObjName) ->
+ couch_log:debug("Swift: head object ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ Method = head,
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container)
+ ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, Head, _} ->
+ couch_log:debug("Swift: ~p~p~n", [ReturnCode, Head]),
+ ObjectSize = lists:filter(fun ({"Content-Length", _}) -> true; (_) -> false end, Head),
+ EtagHeader = lists:filter(fun ({"Etag", _}) -> true ; (_) -> false end, Head),
+ Etag = element(2, lists:nth(1, EtagHeader)),
+ {ObjectSizeNumeric, _} = string:to_integer(element(2, lists:nth(1, ObjectSize))),
+ couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n",
+ [ObjectSizeNumeric, Etag]),
+ EtagDecode = hex_to_bin(Etag),
+ EtagMD5 = base64:encode(EtagDecode),
+ couch_log:debug("Etag in base64 ~p~n", [EtagMD5]),
+ {ok, {ObjectSizeNumeric, EtagMD5}};
+ Error ->
+ couch_log:debug("Swift: Head ~p failed ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error,"HEAD object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+create_container(DbName) ->
+ couch_log:debug("Swift : create container ~p~n", [DbName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ Container = DbName,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++"/",
+ couch_log:debug("Swift: url ~p ~n",[NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ couch_log:debug("Swift: container ~p created with code : ~p~n",[Container, ReturnCode]),
+ {ok,{ReturnCode,Container}};
+ Error ->
+ couch_log:debug("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]),
+ {error, "Failed to create container"}
+ end;
+ {not_authenticated,_} ->
+ {error, "Failed to create container. Not authenticated"}
+ end.
+
+delete_container(Container) ->
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = delete,
+ NewUrl = Url ++ "/" ++ Container ++"/",
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ {ok,{ReturnCode,Container}};
+ Error ->
+ {error,{element(2,Error),Container}}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+%% ====================================================================
+%% Internal functions
+%% ====================================================================
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+authenticate() ->
+ couch_log:debug("Going to authenticate in Swift",[]),
+ case config:get("swift", "auth_model", "tempauth") of
+ "tempauth" ->
+ swift_v1_auth();
+ "keystone" ->
+ keystone_auth();
+ Error ->
+ couch_log:debug("Authentication method is not supported: ~p", element(2,Error)),
+ {not_authenticated, ""}
+ end.
+
+keystone_auth() ->
+ Method = post,
+ URL = config:get("swift", "auth_url"),
+ Header = [{"Content-Type", "application/json"}],
+ Tenant = config:get("swift", "account_tenant"),
+ User = config:get("swift", "username"),
+ Psw = config:get("swift", "password"),
+ Body = "{\"auth\": {\"tenantName\": \"" ++ Tenant
+ ++ "\", \"passwordCredentials\": {\"username\": \""
+ ++ User ++ "\", \"password\": \"" ++ Psw ++ "\"}}}",
--- End diff --
@gilv If service accepts ASCII only passwords I'm in doubt if it's safe to use it.
I'm not aware about such code, but all what you need to do is to construct proper property list and encode it to JSON. That's all.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by rnewson <gi...@git.apache.org>.
Github user rnewson commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48727391
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
+ {error, _} ->
+ {error, Data}
+ end;
+ {error, _} ->
+ {error, Data}
+ end.
+
+att_store(Db, #doc{}=Doc) ->
+ att_store(Db, [Doc]);
+att_store(Db, Docs) when is_list(Docs) ->
+ DbName = container_name(Db),
+ [#doc{atts=Atts0} = Doc | Rest] = Docs,
+ if
+ length(Atts0) > 0 ->
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Docs
+ end.
+
+att(get, Db, Att) ->
--- End diff --
agree, I'm not a fan either. These are distinct functions, they should be expressed as separate functions.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48717599
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
+ {error, _} ->
+ {error, Data}
+ end;
+ {error, _} ->
+ {error, Data}
+ end.
+
+att_store(Db, #doc{}=Doc) ->
+ att_store(Db, [Doc]);
+att_store(Db, Docs) when is_list(Docs) ->
+ DbName = container_name(Db),
+ [#doc{atts=Atts0} = Doc | Rest] = Docs,
+ if
+ length(Atts0) > 0 ->
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Docs
+ end.
+
+att(get, Db, Att) ->
+ DbName = container_name(Db),
+ [Name,AttLen,AttLenUser] = couch_att:fetch([name, att_len,
+ att_external_size], Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, "
+ ++ "stored length: ~p~n", [Name, AttLen, AttLenUser]),
+ NewData = (get_backend_driver()):get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser},
+ {disk_len, AttLenUser}], Att),
+ NewAtt;
+att(delete,Db, Att) ->
+ DbName = container_name(Db),
+ [Name] = couch_att:fetch([name],Att),
+ couch_log:debug("Delete ~p from ~p", [Name, DbName]).
+
+container(create,DbName) ->
--- End diff --
I think there is terminology leak: containers is a thing of open stack swift world and it doesn't applicable for, say, AWS S3. The frontend module should be backend agnostic and, probably, operate with own universal (xkcd: lets make yet another standard) terms that fits everyone or try to avoid knowledge of these bits: just put attachment, get attachment, don't care how it will be stored on backend.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48718264
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
+ {error, _} ->
+ {error, Data}
+ end;
+ {error, _} ->
+ {error, Data}
+ end.
+
+att_store(Db, #doc{}=Doc) ->
+ att_store(Db, [Doc]);
+att_store(Db, Docs) when is_list(Docs) ->
+ DbName = container_name(Db),
+ [#doc{atts=Atts0} = Doc | Rest] = Docs,
+ if
+ length(Atts0) > 0 ->
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Docs
+ end.
+
+att(get, Db, Att) ->
--- End diff --
I'm not sure that this idea with atoms is good one (here and below), but let's keep the same style: if we have att_store, then there better see att_get and att_delete rather than att(get, ...) or otherwise, not have att_store.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by gilv <gi...@git.apache.org>.
Github user gilv commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48724446
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
--- End diff --
@kxepal . Thanks for review it.
fabric_at_handler is a layer between CouchDB and the backend store. It's not aware how backend store works or what it does internally. I completely agree with your other comments, for example "201" is something that belongs to the backend store and should be here.
Data = couch_httpd:recv(Req, ContentLen) is suppose to read data from the HTTP request, using the same mechanism as CouchDB works, so i use the same function to read data. Otherwise i will need to copy-paste the same code here.
Why is it different to function att_processor(DbName, Att)? That function also "aware" of CouchDB's internal functions, for example it uses couch_at module to update attachments.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41981289
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
--- End diff --
This option looks as boolean, but you use "external" value elsewhere in place of "true". If it pretend to be a flag, use config:get_boolean. Otherwise better pick less confusing name for default value.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by gilv <gi...@git.apache.org>.
Github user gilv commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48705157
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
--- End diff --
@kxepal I am using hex_to_bin to transform Etag returned from Swift to the same format as it used by CouchDB.
I did some tests
A = hex_to_bin("39345d76817774864639d2bafb7c4551"),
B = couch_util: to_hex("39345d76817774864639d2bafb7c4551"),
They produce different results. B is not what is expected.
A = 4]v�wt�F9Һ�
B = 3339333435643736383137373734383634363339643262616662376334353531
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-179780756
@lazedo
`Options` argument assumes that it will be passed from the outside. Who will form it and how? And if it contains some url to there attachment have to be stored (by your example, as I get it right), how it would be preserved on the following attachment read? Have to be we need to store these options somewhere and link them by the database-document-attachment triple id. In `att_store` we already have all these information that need to lookup such options (assuming that they are stored outside of the document).
Per-attachment configuration involves custom attachment stub fields feature, so they all will be available for the passed Doc argument (in anyway it's a reasonable to store them within the document itself). But that would be a question how to store private information there and how they should be replicated.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41982798
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
--- End diff --
May be move OpenStack integration logic to separate module, or even into separate project which gets plugged in via couch_epi? This place is not the best one for it.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41982990
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
+%% ====================================================================
+
+extractAuthInfo([{_,_}|_] = Obj,[])->
+ Storage_URL = proplists:get_value(<<"publicURL">>, Obj),
+ Storage_Token = proplists:get_value(<<"id">>, Obj),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{[{_,_}|_]}] = Obj,[])->
+ [{ObjNext}] = Obj,
+ Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext),
+ Storage_Token = proplists:get_value(<<"id">>, ObjNext),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{_,_}|L]=Obj,KeyValPath)->
+ [{Key, Val}|KeyValPathNext] = KeyValPath,
+ ObjVal = proplists:get_value(Key,Obj),
+ case Val of
+ [] -> extractAuthInfo(ObjVal, KeyValPathNext);
+ ObjVal -> extractAuthInfo(Obj, KeyValPathNext);
+ _ -> ["", "", false]
+ end;
+extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)->
+ [ObjCur| _] = Obj,
+ [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur, KeyValPath),
+ case Status of
+ false -> extractAuthInfo(L, KeyValPath);
+ _ -> [Storage_URL, Storage_Token, Status]
+ end;
+extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)->
+ {Doc} = Obj,
+ extractAuthInfo(Doc, KeyValPath).
+
+att_processor(DbName,Att) ->
+ couch_log:debug("Swift: attachment processor",[]),
+ [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding, disk_len, att_len], Att),
+ [Name, Data] = couch_att:fetch([name, data], Att),
+ couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk len: ~p~n",[Name,Type,Enc,DiskLen]),
+ case is_binary(Data) of
+ true ->
+ couch_log:debug("Swift: binary attachment exists",[]),
+ case swift_put_object(DbName, Name, Type, [], Data) of
+ {ok,{201,NewUrl}} ->
+ N1 = unicode:characters_to_binary(NewUrl, unicode, utf8),
+ couch_log:debug("~p ~p~n",[N1, NewUrl]),
+ NewAtt = couch_att:store(data,N1,Att),
+ couch_log:debug("Swift. testing store in original length ~p~n",[AttLen]),
+ {ObjectSize,EtagMD5} = swift_head_object(DbName, Name),
+ NewAtt1 = couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt),
+ NewAtt1;
+ {_,{Code,_}} ->
+ couch_log:debug("Swift: response code is ~p~n",[Code]),
+ Att
+ end;
+ _ ->
+ Att
+ end.
+
+swift_put_object(Container, ObjName, ContextType, CustomHeaders, Data) ->
+ couch_log:debug("Swift: PUT ~p/~s object method ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Headers = CustomHeaders ++ [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ couch_log:debug("Swift: going to upload : ~p ~p ~p ~p ~p~n",[Url, Headers, ContextType,ObjNameEncoded, Container]),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url: ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Headers, unicode:characters_to_list(ContextType), Data}, [], []),
+ couch_log:debug("~p~n",[R]),
+ case R of
+ {ok,_} ->
+ {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R,
+ {ok,{ReturnCode,NewUrl}};
+ Error ->
+ {error,{element(2,Error),NewUrl}}
+ end;
+ {not_authenticated,_} ->
+ {error,"Not authenticated",""};
+ {auth_not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+swift_delete_object(Container, ObjName) ->
+ couch_log:debug("Swift: Delete ~p/~p object method ~n",[Container, ObjName]).
+
+swift_get_object(Container, ObjName) ->
+ couch_log:debug("Swift: get object ~p/~p ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = get,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Header}, [], []),
+ {ok, {{"HTTP/1.1",ReturnCode, _}, Head, Body}} = R,
+ couch_log:debug("Swift: ~p~p ~n",[ReturnCode, Head]),
+ Body;
+ {not_authenticated,_} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+swift_head_object(Container, ObjName) ->
+ couch_log:debug("Swift: head object ~p/~p ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = head,
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Header}, [], []),
+ {ok, {{"HTTP/1.1",ReturnCode, _}, Head, _}} = R,
+ couch_log:debug("Swift: ~p~p~n",[ReturnCode, Head]),
+ ObjectSize = lists:filter(fun ({"content-length",_}) -> true ; (_) -> false end, Head),
+ EtagHeader = lists:filter(fun ({"etag",_}) -> true ; (_) -> false end, Head),
+ Etag = element(2,lists:nth(1,EtagHeader)),
+ {ObjectSizeNumeric,_} = string:to_integer(element(2,lists:nth(1,ObjectSize))),
+ couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n",[ObjectSizeNumeric, Etag]),
+ couch_log:debug("Etag in base16 ~p~n",[Etag]),
+ EtagDecode = hex_to_bin(Etag),
+ EtagMD5 = base64:encode(EtagDecode),
+ couch_log:debug("Etag in base64 ~p~n",[EtagMD5]),
+ {ObjectSizeNumeric,EtagMD5};
+ {auth_not_supported, Error} ->
+ {error,{element(2,Error),""}};
+ {not_authenticated, _} ->
+ {-1,""}
+ end.
+
+swift_create_container(DbName) ->
+ couch_log:debug("Swift : create container ~p~n",[DbName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ Container = DbName,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++"/",
+ couch_log:debug("Swift: url ~p ~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Header, [], []}, [], []),
+ case R of
+ {ok,_} ->
+ {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R,
+ couch_log:debug("Swift: container ~p created with code : ~p~n",[Container, ReturnCode]),
+ {ok,{ReturnCode,Container}};
+ Error ->
+ couch_log:debug("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]),
+ {error,{element(2,Error),Container}}
+ end;
+ {not_authenticated,_} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+swift_delete_container(Container) ->
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = delete,
+ NewUrl = Url ++ "/" ++ Container ++"/",
+ R = httpc:request(Method, {NewUrl, Header, [], []}, [], []),
+ case R of
+ {ok,_} ->
+ {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R,
+ {ok,{ReturnCode,Container}};
+ Error ->
+ {error,{element(2,Error),Container}}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+authenticate() ->
--- End diff --
This function needs in refactoring: no need to put all the eggs into a single function.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by gilv <gi...@git.apache.org>.
Github user gilv commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-168609248
@kxepal @rnewson
I provided another patch, implementing most of the comments.
Here is what is not implemented in this patch:
1. Remark on the AttFun. Not yet implemented. Original remark: “AttFun could be actually an attachment data or some fold function? Something went wrong if you have to do this.”
2. usage of couch_util:to_hex
3. still no unitests
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41989841
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
--- End diff --
Oh, right. And we need ASF license header here.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-169215816
@gilv
2. I'm afraid there is no any document that describes whole set of rules. I want to setup elvis to define and control that, but it's a future. For now, better just follow the comments.
3. Tests need to cover whole attachments API to ensure that everything works same as before + I think some test with replication would be nice to have to see how it goes. Here is the main question is how to ensure that swift integration code is really works. I'm not sure that it's possible to verify that without Open Stack cluster. Hopefully, today, we have docker thing for the resque.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by gilv <gi...@git.apache.org>.
Github user gilv commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-173119045
@kxepal @rnewson
Few notes related recent patch
1. Please ignore all debug / info print statements. At some point i will remove them at once.
2. I addressed most of the comments.
3. Remark on the AttFun. Not yet implemented. Original remark: “AttFun could be actually an attachment data or some fold function? Something went wrong if you have to do this.”
4. Still no unitests / functional tests.
5. Attachment retrieval from the object store - keep all data in memory. I need to figure out how to use streaming there.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by lazedo <gi...@git.apache.org>.
Github user lazedo commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-179751996
@gilv i think `att_store/2` should be `att_store/3` with `Options` param so that we can have different drivers for different databases and/or documents. `get_backend_driver` should try to read it from Options mentioned above and get it from config ( maybe add a section `external_storage_drivers` for mapping) with default to `active_store`. the `Options` would also allow passing specific driver options.
thoughts ?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48717312
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
--- End diff --
I think this should be in swift backend module, at least. And certainly not couch_httpd (even if these two does the same thing) - just chttpd.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48717371
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
--- End diff --
Why get backend driver three times? What if it will be eventually changed in the middle of this?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-148025302
Please fix:
- Style: try to fit 80 chars line length, put space after comma, no tabs, correctly aligned clauses etc.
- Reduce amount of debug logs: debug logs should help you understand what going on in your code and in which state it is in places where problems may happens, not produce noise about every logical step.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41983817
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
--- End diff --
Why you pass here Db which is not get used?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48718511
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
+ {error, _} ->
+ {error, Data}
+ end;
+ {error, _} ->
+ {error, Data}
+ end.
+
+att_store(Db, #doc{}=Doc) ->
+ att_store(Db, [Doc]);
+att_store(Db, Docs) when is_list(Docs) ->
+ DbName = container_name(Db),
+ [#doc{atts=Atts0} = Doc | Rest] = Docs,
+ if
+ length(Atts0) > 0 ->
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Docs
+ end.
+
+att(get, Db, Att) ->
+ DbName = container_name(Db),
+ [Name,AttLen,AttLenUser] = couch_att:fetch([name, att_len,
+ att_external_size], Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, "
+ ++ "stored length: ~p~n", [Name, AttLen, AttLenUser]),
+ NewData = (get_backend_driver()):get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser},
+ {disk_len, AttLenUser}], Att),
+ NewAtt;
+att(delete,Db, Att) ->
+ DbName = container_name(Db),
+ [Name] = couch_att:fetch([name],Att),
+ couch_log:debug("Delete ~p from ~p", [Name, DbName]).
+
+container(create,DbName) ->
+ couch_log:debug("Create container ~p~n", [DbName]),
+ case (get_backend_driver()):create_container(DbName) of
+ {ok, {"201", Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n", [Container]),
+ {ok, Container};
+ {error, _} ->
+ couch_log:debug("Container ~p creation failed ~n", [DbName]),
+ {error, DbName}
+ end;
+container(get, DbName) ->
+ couch_log:debug("Get container ~p~n", [DbName]);
+container(delete, DbName)->
+ couch_log:debug("Delete container ~p~n", [DbName]),
+ (get_backend_driver()):delete_container(DbName).
+
+external_store() ->
+ config:get_boolean("ext_store", "attachments_offload", false).
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+get_backend_driver() ->
+ case config:get("ext_store", "active_store") of
+ "swift" ->
+ fabric_swift_driver;
+ undefined ->
+ fabric_swift_driver
+ end.
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ DbNameSuffix.
+
+att_processor(DbName, Att) ->
+ [Name, Data, Type, Enc] = couch_att:fetch([name, data, type, encoding], Att),
+ couch_log:debug("Att name: ~p, Type: ~p, Encoding: ~p ~n", [Name, Type, Enc]),
+ case is_binary(Data) of
+ true ->
+ case (get_backend_driver()):put_object(DbName, Name, Type, [], Data) of
+ {ok, {"201", NewUrl}} ->
+ N1 = unicode:characters_to_binary(NewUrl, unicode, utf8),
+ NewAtt = couch_att:store(data, N1, Att),
+ case (get_backend_driver()):head_object(DbName, Name) of
+ {ok, {ObjectSize,EtagMD5}} ->
+ NewAtt1 = couch_att:store([{att_extstore_size, ObjectSize},
+ {att_extstore_id, (get_backend_driver()):store_id()},
--- End diff --
Same question about triple get backend driver.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by gilv <gi...@git.apache.org>.
Github user gilv commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r49943389
--- Diff: src/fabric_swift_driver.erl ---
@@ -0,0 +1,269 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_swift_driver).
+
+-export([put_object/5, delete_object/2, get_object/2, head_object/2]).
+-export([store_id/0, create_container/1, delete_container/1 ]).
+%% ====================================================================
+%% OpenStack Swift implementation
+%% ====================================================================
+
+store_id() ->
+ "swift".
+
+put_object(Container, ObjName, ContextType, CustomHeaders, Data) ->
+ couch_log:debug("Swift: PUT ~p/~s object ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Headers = CustomHeaders ++ [{"X-Auth-Token", Storage_Token},
+ {"Content-Type",
+ unicode:characters_to_list(ContextType)}],
+ Method = put,
+ couch_log:debug("Swift: PUT : ~p ~p ~p ~p~n",
+ [Url,ContextType, ObjNameEncoded, Container]),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: PUT url: ~p~n",[NewUrl]),
+ R = ibrowse:send_req(NewUrl, Headers, Method, Data),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ {ok, {ReturnCode, NewUrl}};
+ Error ->
+ couch_log:debug("PUT object failed ~p ~n", [element(2, Error)]),
+ {error, "PUT object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+delete_object(Container, ObjName) ->
+ %TO-DO: should be called during database compaction.
+ couch_log:debug("Swift: Delete ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = delete,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/"
+ ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ couch_log:debug("Swift: Delete ~p. Return code ~p ~n",
+ [NewUrl, ReturnCode]),
+ {ok, ReturnCode};
+ Error ->
+ couch_log:debug("Swift: Delete ~p failed. ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error, "Delete object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+get_object(Container, ObjName) ->
+ couch_log:debug("Swift: get object ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = get,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/"
+ ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, Body} ->
+ couch_log:debug("Swift: GET ~p with return code ~p ~n", [NewUrl, ReturnCode]),
+ Body;
+ Error ->
+ couch_log:debug("Swift: GET ~p failed. ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error, "Get object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+head_object(Container, ObjName) ->
+ couch_log:debug("Swift: head object ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ Method = head,
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container)
+ ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, Head, _} ->
+ couch_log:debug("Swift: ~p~p~n", [ReturnCode, Head]),
+ ObjectSize = lists:filter(fun ({"Content-Length", _}) -> true; (_) -> false end, Head),
+ EtagHeader = lists:filter(fun ({"Etag", _}) -> true ; (_) -> false end, Head),
+ Etag = element(2, lists:nth(1, EtagHeader)),
+ {ObjectSizeNumeric, _} = string:to_integer(element(2, lists:nth(1, ObjectSize))),
+ couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n",
+ [ObjectSizeNumeric, Etag]),
+ EtagDecode = hex_to_bin(Etag),
+ EtagMD5 = base64:encode(EtagDecode),
+ couch_log:debug("Etag in base64 ~p~n", [EtagMD5]),
+ {ok, {ObjectSizeNumeric, EtagMD5}};
+ Error ->
+ couch_log:debug("Swift: Head ~p failed ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error,"HEAD object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+create_container(DbName) ->
+ couch_log:debug("Swift : create container ~p~n", [DbName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ Container = DbName,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++"/",
+ couch_log:debug("Swift: url ~p ~n",[NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ couch_log:debug("Swift: container ~p created with code : ~p~n",[Container, ReturnCode]),
+ {ok,{ReturnCode,Container}};
+ Error ->
+ couch_log:debug("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]),
+ {error, "Failed to create container"}
+ end;
+ {not_authenticated,_} ->
+ {error, "Failed to create container. Not authenticated"}
+ end.
+
+delete_container(Container) ->
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = delete,
+ NewUrl = Url ++ "/" ++ Container ++"/",
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ {ok,{ReturnCode,Container}};
+ Error ->
+ {error,{element(2,Error),Container}}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+%% ====================================================================
+%% Internal functions
+%% ====================================================================
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+authenticate() ->
+ couch_log:debug("Going to authenticate in Swift",[]),
+ case config:get("swift", "auth_model", "tempauth") of
+ "tempauth" ->
+ swift_v1_auth();
+ "keystone" ->
+ keystone_auth();
+ Error ->
+ couch_log:debug("Authentication method is not supported: ~p", element(2,Error)),
+ {not_authenticated, ""}
+ end.
+
+keystone_auth() ->
+ Method = post,
+ URL = config:get("swift", "auth_url"),
+ Header = [{"Content-Type", "application/json"}],
+ Tenant = config:get("swift", "account_tenant"),
+ User = config:get("swift", "username"),
+ Psw = config:get("swift", "password"),
+ Body = "{\"auth\": {\"tenantName\": \"" ++ Tenant
+ ++ "\", \"passwordCredentials\": {\"username\": \""
+ ++ User ++ "\", \"password\": \"" ++ Psw ++ "\"}}}",
--- End diff --
@kxepal It actually depends on the Keystone. I couldn't understand completely if Keystone permits such passwords or they should be encoded. My guess Keystone does not permits such passwords , so password will be ascii. Do you think i should add some encoding here? I can document your comment in the code, so we can handle it at later stage.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48718337
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
+ {error, _} ->
+ {error, Data}
+ end;
+ {error, _} ->
+ {error, Data}
+ end.
+
+att_store(Db, #doc{}=Doc) ->
+ att_store(Db, [Doc]);
+att_store(Db, Docs) when is_list(Docs) ->
+ DbName = container_name(Db),
+ [#doc{atts=Atts0} = Doc | Rest] = Docs,
+ if
+ length(Atts0) > 0 ->
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Docs
+ end.
+
+att(get, Db, Att) ->
+ DbName = container_name(Db),
+ [Name,AttLen,AttLenUser] = couch_att:fetch([name, att_len,
+ att_external_size], Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, "
+ ++ "stored length: ~p~n", [Name, AttLen, AttLenUser]),
+ NewData = (get_backend_driver()):get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser},
+ {disk_len, AttLenUser}], Att),
+ NewAtt;
+att(delete,Db, Att) ->
+ DbName = container_name(Db),
+ [Name] = couch_att:fetch([name],Att),
+ couch_log:debug("Delete ~p from ~p", [Name, DbName]).
+
+container(create,DbName) ->
+ couch_log:debug("Create container ~p~n", [DbName]),
+ case (get_backend_driver()):create_container(DbName) of
+ {ok, {"201", Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n", [Container]),
+ {ok, Container};
+ {error, _} ->
+ couch_log:debug("Container ~p creation failed ~n", [DbName]),
+ {error, DbName}
+ end;
+container(get, DbName) ->
+ couch_log:debug("Get container ~p~n", [DbName]);
+container(delete, DbName)->
+ couch_log:debug("Delete container ~p~n", [DbName]),
+ (get_backend_driver()):delete_container(DbName).
+
+external_store() ->
+ config:get_boolean("ext_store", "attachments_offload", false).
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+get_backend_driver() ->
+ case config:get("ext_store", "active_store") of
+ "swift" ->
+ fabric_swift_driver;
+ undefined ->
+ fabric_swift_driver
+ end.
+
+container_name(Db) ->
--- End diff --
Just curious why not `container(name, Db)` ? (:
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41983087
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
+%% ====================================================================
+
+extractAuthInfo([{_,_}|_] = Obj,[])->
+ Storage_URL = proplists:get_value(<<"publicURL">>, Obj),
+ Storage_Token = proplists:get_value(<<"id">>, Obj),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{[{_,_}|_]}] = Obj,[])->
+ [{ObjNext}] = Obj,
+ Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext),
+ Storage_Token = proplists:get_value(<<"id">>, ObjNext),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{_,_}|L]=Obj,KeyValPath)->
+ [{Key, Val}|KeyValPathNext] = KeyValPath,
+ ObjVal = proplists:get_value(Key,Obj),
+ case Val of
+ [] -> extractAuthInfo(ObjVal, KeyValPathNext);
+ ObjVal -> extractAuthInfo(Obj, KeyValPathNext);
+ _ -> ["", "", false]
+ end;
+extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)->
+ [ObjCur| _] = Obj,
+ [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur, KeyValPath),
+ case Status of
+ false -> extractAuthInfo(L, KeyValPath);
+ _ -> [Storage_URL, Storage_Token, Status]
+ end;
+extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)->
+ {Doc} = Obj,
+ extractAuthInfo(Doc, KeyValPath).
+
+att_processor(DbName,Att) ->
+ couch_log:debug("Swift: attachment processor",[]),
+ [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding, disk_len, att_len], Att),
+ [Name, Data] = couch_att:fetch([name, data], Att),
+ couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk len: ~p~n",[Name,Type,Enc,DiskLen]),
+ case is_binary(Data) of
+ true ->
+ couch_log:debug("Swift: binary attachment exists",[]),
+ case swift_put_object(DbName, Name, Type, [], Data) of
+ {ok,{201,NewUrl}} ->
+ N1 = unicode:characters_to_binary(NewUrl, unicode, utf8),
+ couch_log:debug("~p ~p~n",[N1, NewUrl]),
+ NewAtt = couch_att:store(data,N1,Att),
+ couch_log:debug("Swift. testing store in original length ~p~n",[AttLen]),
+ {ObjectSize,EtagMD5} = swift_head_object(DbName, Name),
+ NewAtt1 = couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt),
+ NewAtt1;
+ {_,{Code,_}} ->
+ couch_log:debug("Swift: response code is ~p~n",[Code]),
+ Att
+ end;
+ _ ->
+ Att
+ end.
+
+swift_put_object(Container, ObjName, ContextType, CustomHeaders, Data) ->
+ couch_log:debug("Swift: PUT ~p/~s object method ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Headers = CustomHeaders ++ [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ couch_log:debug("Swift: going to upload : ~p ~p ~p ~p ~p~n",[Url, Headers, ContextType,ObjNameEncoded, Container]),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url: ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Headers, unicode:characters_to_list(ContextType), Data}, [], []),
+ couch_log:debug("~p~n",[R]),
+ case R of
+ {ok,_} ->
+ {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R,
+ {ok,{ReturnCode,NewUrl}};
+ Error ->
+ {error,{element(2,Error),NewUrl}}
+ end;
+ {not_authenticated,_} ->
+ {error,"Not authenticated",""};
+ {auth_not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+swift_delete_object(Container, ObjName) ->
+ couch_log:debug("Swift: Delete ~p/~p object method ~n",[Container, ObjName]).
+
+swift_get_object(Container, ObjName) ->
+ couch_log:debug("Swift: get object ~p/~p ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = get,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Header}, [], []),
+ {ok, {{"HTTP/1.1",ReturnCode, _}, Head, Body}} = R,
+ couch_log:debug("Swift: ~p~p ~n",[ReturnCode, Head]),
+ Body;
+ {not_authenticated,_} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
--- End diff --
Why return error as a tuple with empty string?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by gilv <gi...@git.apache.org>.
Github user gilv commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-169077329
@kxepal @rnewson
1. Please ignore the debug statements in my code. When my code will be stable enough - i will just remove those debug prints. Since i am changing code rapidly - i still need those prints.
2. You both wrote that the code style is not good. Can you please tell me what should be the code style? Is there any template i can use? Is there any documentation in CouchDB where it's written what should be coding style? Perhaps i missed it.
3. Tests: What would you like to have exactly? To fully test that my patch is working there is need to access the Swift cluster. I have some functional tests, that upload attachments to CouchDB. Those tests access Swift cluster directly and verify that attachments stored correctly. But i assume it's not what you mean. Can you please point me to some existing tests in CouchDB so i will see what i should implement in my case?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-179792020
@lazedo I think I miss the case as well. Anyway, to give the proper answer on this need to implement few more different drivers against very different backends for point of their API. Then we can shape our API more-or-less right. Now it's more like a proof-of-concept, but need to start from something.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by rnewson <gi...@git.apache.org>.
Github user rnewson commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-168670628
Given the nature of the patch, this cannot merge without tests, assuming all style and other issues are resolved.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-179763345
@gilv I can read the code and leave few more comments about styling, naming and else not-much-important-but-annoying bits, but I think that would be much better see any tests and the way to try this at home. That's more important than yet-another-spacing issue.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by rnewson <gi...@git.apache.org>.
Github user rnewson commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48727492
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
+ {error, _} ->
+ {error, Data}
+ end;
+ {error, _} ->
+ {error, Data}
+ end.
+
+att_store(Db, #doc{}=Doc) ->
+ att_store(Db, [Doc]);
+att_store(Db, Docs) when is_list(Docs) ->
+ DbName = container_name(Db),
+ [#doc{atts=Atts0} = Doc | Rest] = Docs,
+ if
+ length(Atts0) > 0 ->
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Docs
+ end.
+
+att(get, Db, Att) ->
+ DbName = container_name(Db),
+ [Name,AttLen,AttLenUser] = couch_att:fetch([name, att_len,
+ att_external_size], Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, "
+ ++ "stored length: ~p~n", [Name, AttLen, AttLenUser]),
+ NewData = (get_backend_driver()):get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser},
+ {disk_len, AttLenUser}], Att),
+ NewAtt;
+att(delete,Db, Att) ->
+ DbName = container_name(Db),
+ [Name] = couch_att:fetch([name],Att),
+ couch_log:debug("Delete ~p from ~p", [Name, DbName]).
+
+container(create,DbName) ->
+ couch_log:debug("Create container ~p~n", [DbName]),
+ case (get_backend_driver()):create_container(DbName) of
+ {ok, {"201", Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n", [Container]),
+ {ok, Container};
+ {error, _} ->
+ couch_log:debug("Container ~p creation failed ~n", [DbName]),
+ {error, DbName}
+ end;
+container(get, DbName) ->
+ couch_log:debug("Get container ~p~n", [DbName]);
--- End diff --
??
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by rnewson <gi...@git.apache.org>.
Github user rnewson commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-152233728
I'm not going to have time to review this in detail, but I agree it needs tests to go in, as well as considerable cleanup.
Noting here that this should not be merged before 2.0 is released whatever happens.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41984239
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
--- End diff --
You can actually concat binaries instead or not use list_to_binary cast above.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41982873
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
+%% ====================================================================
+
+extractAuthInfo([{_,_}|_] = Obj,[])->
+ Storage_URL = proplists:get_value(<<"publicURL">>, Obj),
+ Storage_Token = proplists:get_value(<<"id">>, Obj),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{[{_,_}|_]}] = Obj,[])->
+ [{ObjNext}] = Obj,
+ Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext),
+ Storage_Token = proplists:get_value(<<"id">>, ObjNext),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{_,_}|L]=Obj,KeyValPath)->
+ [{Key, Val}|KeyValPathNext] = KeyValPath,
+ ObjVal = proplists:get_value(Key,Obj),
+ case Val of
+ [] -> extractAuthInfo(ObjVal, KeyValPathNext);
+ ObjVal -> extractAuthInfo(Obj, KeyValPathNext);
+ _ -> ["", "", false]
+ end;
+extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)->
+ [ObjCur| _] = Obj,
+ [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur, KeyValPath),
+ case Status of
+ false -> extractAuthInfo(L, KeyValPath);
+ _ -> [Storage_URL, Storage_Token, Status]
+ end;
+extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)->
+ {Doc} = Obj,
+ extractAuthInfo(Doc, KeyValPath).
+
+att_processor(DbName,Att) ->
+ couch_log:debug("Swift: attachment processor",[]),
+ [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding, disk_len, att_len], Att),
+ [Name, Data] = couch_att:fetch([name, data], Att),
+ couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk len: ~p~n",[Name,Type,Enc,DiskLen]),
+ case is_binary(Data) of
+ true ->
+ couch_log:debug("Swift: binary attachment exists",[]),
+ case swift_put_object(DbName, Name, Type, [], Data) of
+ {ok,{201,NewUrl}} ->
+ N1 = unicode:characters_to_binary(NewUrl, unicode, utf8),
+ couch_log:debug("~p ~p~n",[N1, NewUrl]),
+ NewAtt = couch_att:store(data,N1,Att),
+ couch_log:debug("Swift. testing store in original length ~p~n",[AttLen]),
+ {ObjectSize,EtagMD5} = swift_head_object(DbName, Name),
+ NewAtt1 = couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt),
+ NewAtt1;
+ {_,{Code,_}} ->
+ couch_log:debug("Swift: response code is ~p~n",[Code]),
+ Att
+ end;
+ _ ->
+ Att
+ end.
+
+swift_put_object(Container, ObjName, ContextType, CustomHeaders, Data) ->
+ couch_log:debug("Swift: PUT ~p/~s object method ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Headers = CustomHeaders ++ [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ couch_log:debug("Swift: going to upload : ~p ~p ~p ~p ~p~n",[Url, Headers, ContextType,ObjNameEncoded, Container]),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url: ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Headers, unicode:characters_to_list(ContextType), Data}, [], []),
--- End diff --
We use ibrowse as a client for internal http requests. No need to introduce yet another one.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by rnewson <gi...@git.apache.org>.
Github user rnewson commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48727560
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
+ {error, _} ->
+ {error, Data}
+ end;
+ {error, _} ->
+ {error, Data}
+ end.
+
+att_store(Db, #doc{}=Doc) ->
+ att_store(Db, [Doc]);
+att_store(Db, Docs) when is_list(Docs) ->
+ DbName = container_name(Db),
+ [#doc{atts=Atts0} = Doc | Rest] = Docs,
+ if
+ length(Atts0) > 0 ->
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Docs
+ end.
+
+att(get, Db, Att) ->
+ DbName = container_name(Db),
+ [Name,AttLen,AttLenUser] = couch_att:fetch([name, att_len,
+ att_external_size], Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, "
+ ++ "stored length: ~p~n", [Name, AttLen, AttLenUser]),
+ NewData = (get_backend_driver()):get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser},
+ {disk_len, AttLenUser}], Att),
+ NewAtt;
+att(delete,Db, Att) ->
+ DbName = container_name(Db),
+ [Name] = couch_att:fetch([name],Att),
+ couch_log:debug("Delete ~p from ~p", [Name, DbName]).
+
+container(create,DbName) ->
+ couch_log:debug("Create container ~p~n", [DbName]),
+ case (get_backend_driver()):create_container(DbName) of
+ {ok, {"201", Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n", [Container]),
+ {ok, Container};
+ {error, _} ->
+ couch_log:debug("Container ~p creation failed ~n", [DbName]),
--- End diff --
every log statement appears to be at the debug level. This is below the default level, and no one can really enable debug level logging for long due to the volume of it. These messages will not help anyone in production.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41983524
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
+%% ====================================================================
+
+extractAuthInfo([{_,_}|_] = Obj,[])->
+ Storage_URL = proplists:get_value(<<"publicURL">>, Obj),
+ Storage_Token = proplists:get_value(<<"id">>, Obj),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{[{_,_}|_]}] = Obj,[])->
+ [{ObjNext}] = Obj,
+ Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext),
+ Storage_Token = proplists:get_value(<<"id">>, ObjNext),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{_,_}|L]=Obj,KeyValPath)->
+ [{Key, Val}|KeyValPathNext] = KeyValPath,
+ ObjVal = proplists:get_value(Key,Obj),
+ case Val of
+ [] -> extractAuthInfo(ObjVal, KeyValPathNext);
+ ObjVal -> extractAuthInfo(Obj, KeyValPathNext);
+ _ -> ["", "", false]
+ end;
+extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)->
+ [ObjCur| _] = Obj,
+ [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur, KeyValPath),
+ case Status of
+ false -> extractAuthInfo(L, KeyValPath);
+ _ -> [Storage_URL, Storage_Token, Status]
+ end;
+extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)->
+ {Doc} = Obj,
+ extractAuthInfo(Doc, KeyValPath).
+
+att_processor(DbName,Att) ->
+ couch_log:debug("Swift: attachment processor",[]),
+ [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding, disk_len, att_len], Att),
+ [Name, Data] = couch_att:fetch([name, data], Att),
+ couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk len: ~p~n",[Name,Type,Enc,DiskLen]),
+ case is_binary(Data) of
+ true ->
+ couch_log:debug("Swift: binary attachment exists",[]),
+ case swift_put_object(DbName, Name, Type, [], Data) of
+ {ok,{201,NewUrl}} ->
+ N1 = unicode:characters_to_binary(NewUrl, unicode, utf8),
+ couch_log:debug("~p ~p~n",[N1, NewUrl]),
+ NewAtt = couch_att:store(data,N1,Att),
+ couch_log:debug("Swift. testing store in original length ~p~n",[AttLen]),
+ {ObjectSize,EtagMD5} = swift_head_object(DbName, Name),
+ NewAtt1 = couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt),
--- End diff --
I also think that external is not the best choice. Since field is already carries external word in the key, better specify which exact external we assume here. Today it's only swift, tomorrow it could be also S3/NFS/whatever which will have to use different integration logic.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by rnewson <gi...@git.apache.org>.
Github user rnewson commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48727108
--- Diff: src/fabric_db_create.erl ---
@@ -164,6 +164,19 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix) ->
{[[<<"add">>, Range, Node] | Raw], orddict:append(Node, Range, ByNode),
orddict:append(Range, Node, ByRange)}
end, {[], [], []}, Shards),
+
+ case fabric_att_handler:external_store() of
+ true ->
+ DbNameSuffix = unicode:characters_to_list(DbName) ++ Suffix,
+ case fabric_att_handler:container(create,DbNameSuffix) of
+ {ok, Container} ->
+ couch_log:debug("Container ~p created", [Container]);
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed", [DbNameSuffix])
--- End diff --
You don't handle an error by logging, surely this is fatal?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by lazedo <gi...@git.apache.org>.
Github user lazedo commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-179773968
@kxepal i'm thinking in a way to have different provider per attachment/document. the use case is a document with two or more attachments where one could stored in a public accessible manner (dropbox, gdrive) and others more sensitive or private could be stored in another like evernote.
the `per-db driver configuration` makes sense but i believe it doesn't respond to this?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48725942
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
--- End diff --
The reason why it's different is simple: couch_httpd/chttpd are frontend modules that provides HTTP API for internal stuff like couch_att.
The reason why couch_httpd better to avoid use here as that this module is for backdoor interface while chttpd that used for the front, cluster one, may eventually start work by different rules.
The reason why chttpd/couch_httpd shouldn't be here is that fabric is a layer to run cluster wide operations after http request get processed.
Also, have you checked `fabric:att_receiver/2`? Looks like exactly what you need here.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by gilv <gi...@git.apache.org>.
Github user gilv commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r49948952
--- Diff: src/fabric_swift_driver.erl ---
@@ -0,0 +1,269 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_swift_driver).
+
+-export([put_object/5, delete_object/2, get_object/2, head_object/2]).
+-export([store_id/0, create_container/1, delete_container/1 ]).
+%% ====================================================================
+%% OpenStack Swift implementation
+%% ====================================================================
+
+store_id() ->
+ "swift".
+
+put_object(Container, ObjName, ContextType, CustomHeaders, Data) ->
+ couch_log:debug("Swift: PUT ~p/~s object ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Headers = CustomHeaders ++ [{"X-Auth-Token", Storage_Token},
+ {"Content-Type",
+ unicode:characters_to_list(ContextType)}],
+ Method = put,
+ couch_log:debug("Swift: PUT : ~p ~p ~p ~p~n",
+ [Url,ContextType, ObjNameEncoded, Container]),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: PUT url: ~p~n",[NewUrl]),
+ R = ibrowse:send_req(NewUrl, Headers, Method, Data),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ {ok, {ReturnCode, NewUrl}};
+ Error ->
+ couch_log:debug("PUT object failed ~p ~n", [element(2, Error)]),
+ {error, "PUT object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+delete_object(Container, ObjName) ->
+ %TO-DO: should be called during database compaction.
+ couch_log:debug("Swift: Delete ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = delete,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/"
+ ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ couch_log:debug("Swift: Delete ~p. Return code ~p ~n",
+ [NewUrl, ReturnCode]),
+ {ok, ReturnCode};
+ Error ->
+ couch_log:debug("Swift: Delete ~p failed. ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error, "Delete object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+get_object(Container, ObjName) ->
+ couch_log:debug("Swift: get object ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = get,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/"
+ ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, Body} ->
+ couch_log:debug("Swift: GET ~p with return code ~p ~n", [NewUrl, ReturnCode]),
+ Body;
+ Error ->
+ couch_log:debug("Swift: GET ~p failed. ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error, "Get object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+head_object(Container, ObjName) ->
+ couch_log:debug("Swift: head object ~p/~p ~n", [Container, ObjName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token", Storage_Token}],
+ Method = head,
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container)
+ ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n", [NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, Head, _} ->
+ couch_log:debug("Swift: ~p~p~n", [ReturnCode, Head]),
+ ObjectSize = lists:filter(fun ({"Content-Length", _}) -> true; (_) -> false end, Head),
+ EtagHeader = lists:filter(fun ({"Etag", _}) -> true ; (_) -> false end, Head),
+ Etag = element(2, lists:nth(1, EtagHeader)),
+ {ObjectSizeNumeric, _} = string:to_integer(element(2, lists:nth(1, ObjectSize))),
+ couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n",
+ [ObjectSizeNumeric, Etag]),
+ EtagDecode = hex_to_bin(Etag),
+ EtagMD5 = base64:encode(EtagDecode),
+ couch_log:debug("Etag in base64 ~p~n", [EtagMD5]),
+ {ok, {ObjectSizeNumeric, EtagMD5}};
+ Error ->
+ couch_log:debug("Swift: Head ~p failed ~n", [NewUrl]),
+ couch_log:debug("Swift: ~p ~n", [element(2, Error)]),
+ {error,"HEAD object failed"}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"}
+ end.
+
+create_container(DbName) ->
+ couch_log:debug("Swift : create container ~p~n", [DbName]),
+ case authenticate() of
+ {ok, {Url, Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ Container = DbName,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++"/",
+ couch_log:debug("Swift: url ~p ~n",[NewUrl]),
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ couch_log:debug("Swift: container ~p created with code : ~p~n",[Container, ReturnCode]),
+ {ok,{ReturnCode,Container}};
+ Error ->
+ couch_log:debug("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]),
+ {error, "Failed to create container"}
+ end;
+ {not_authenticated,_} ->
+ {error, "Failed to create container. Not authenticated"}
+ end.
+
+delete_container(Container) ->
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = delete,
+ NewUrl = Url ++ "/" ++ Container ++"/",
+ R = ibrowse:send_req(NewUrl, Header, Method),
+ case R of
+ {ok, ReturnCode, _, _} ->
+ {ok,{ReturnCode,Container}};
+ Error ->
+ {error,{element(2,Error),Container}}
+ end;
+ {not_authenticated, _} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+%% ====================================================================
+%% Internal functions
+%% ====================================================================
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+authenticate() ->
+ couch_log:debug("Going to authenticate in Swift",[]),
+ case config:get("swift", "auth_model", "tempauth") of
+ "tempauth" ->
+ swift_v1_auth();
+ "keystone" ->
+ keystone_auth();
+ Error ->
+ couch_log:debug("Authentication method is not supported: ~p", element(2,Error)),
+ {not_authenticated, ""}
+ end.
+
+keystone_auth() ->
+ Method = post,
+ URL = config:get("swift", "auth_url"),
+ Header = [{"Content-Type", "application/json"}],
+ Tenant = config:get("swift", "account_tenant"),
+ User = config:get("swift", "username"),
+ Psw = config:get("swift", "password"),
+ Body = "{\"auth\": {\"tenantName\": \"" ++ Tenant
+ ++ "\", \"passwordCredentials\": {\"username\": \""
+ ++ User ++ "\", \"password\": \"" ++ Psw ++ "\"}}}",
--- End diff --
@kxepal I didn't mean to ask users to manually encode password. In any case "who is the user" here that should encode or not encode password? The "user" is actually CouchDB administrator that also own the Keystone account in Swift. I assume that his account already has ASCII password. Anyway, it's not the point. I can gladly add encoding here. I assume CouchDB already has similar code fragments to encode double quotes or non ASCII characters. Can you please point me to the existing code where i can find something like this?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41982165
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
+%% ====================================================================
+
+extractAuthInfo([{_,_}|_] = Obj,[])->
+ Storage_URL = proplists:get_value(<<"publicURL">>, Obj),
--- End diff --
For key-value proplists, use `couch_util:get_value` instead. It's faster.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by gilv <gi...@git.apache.org>.
Github user gilv commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-176590688
@kxepal Can you please review the recent code?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by rnewson <gi...@git.apache.org>.
Github user rnewson commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-168670040
there's a lot to review here. The huge number of :debug calls seems inappropriate, and often appears to be placeholders / reminders for future work (like error handling).
I note also that indentation rules are still not followed (4 spaces, no 'pretty' alignment).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41983028
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
+%% ====================================================================
+
+extractAuthInfo([{_,_}|_] = Obj,[])->
+ Storage_URL = proplists:get_value(<<"publicURL">>, Obj),
+ Storage_Token = proplists:get_value(<<"id">>, Obj),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{[{_,_}|_]}] = Obj,[])->
+ [{ObjNext}] = Obj,
+ Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext),
+ Storage_Token = proplists:get_value(<<"id">>, ObjNext),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{_,_}|L]=Obj,KeyValPath)->
+ [{Key, Val}|KeyValPathNext] = KeyValPath,
+ ObjVal = proplists:get_value(Key,Obj),
+ case Val of
+ [] -> extractAuthInfo(ObjVal, KeyValPathNext);
+ ObjVal -> extractAuthInfo(Obj, KeyValPathNext);
+ _ -> ["", "", false]
+ end;
+extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)->
+ [ObjCur| _] = Obj,
+ [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur, KeyValPath),
+ case Status of
+ false -> extractAuthInfo(L, KeyValPath);
+ _ -> [Storage_URL, Storage_Token, Status]
+ end;
+extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)->
+ {Doc} = Obj,
+ extractAuthInfo(Doc, KeyValPath).
+
+att_processor(DbName,Att) ->
+ couch_log:debug("Swift: attachment processor",[]),
+ [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding, disk_len, att_len], Att),
+ [Name, Data] = couch_att:fetch([name, data], Att),
+ couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk len: ~p~n",[Name,Type,Enc,DiskLen]),
+ case is_binary(Data) of
+ true ->
+ couch_log:debug("Swift: binary attachment exists",[]),
+ case swift_put_object(DbName, Name, Type, [], Data) of
+ {ok,{201,NewUrl}} ->
+ N1 = unicode:characters_to_binary(NewUrl, unicode, utf8),
+ couch_log:debug("~p ~p~n",[N1, NewUrl]),
+ NewAtt = couch_att:store(data,N1,Att),
+ couch_log:debug("Swift. testing store in original length ~p~n",[AttLen]),
+ {ObjectSize,EtagMD5} = swift_head_object(DbName, Name),
+ NewAtt1 = couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt),
+ NewAtt1;
+ {_,{Code,_}} ->
+ couch_log:debug("Swift: response code is ~p~n",[Code]),
+ Att
+ end;
+ _ ->
+ Att
+ end.
+
+swift_put_object(Container, ObjName, ContextType, CustomHeaders, Data) ->
+ couch_log:debug("Swift: PUT ~p/~s object method ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Headers = CustomHeaders ++ [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ couch_log:debug("Swift: going to upload : ~p ~p ~p ~p ~p~n",[Url, Headers, ContextType,ObjNameEncoded, Container]),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url: ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Headers, unicode:characters_to_list(ContextType), Data}, [], []),
+ couch_log:debug("~p~n",[R]),
+ case R of
+ {ok,_} ->
+ {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R,
+ {ok,{ReturnCode,NewUrl}};
+ Error ->
+ {error,{element(2,Error),NewUrl}}
+ end;
+ {not_authenticated,_} ->
+ {error,"Not authenticated",""};
+ {auth_not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+swift_delete_object(Container, ObjName) ->
+ couch_log:debug("Swift: Delete ~p/~p object method ~n",[Container, ObjName]).
+
+swift_get_object(Container, ObjName) ->
+ couch_log:debug("Swift: get object ~p/~p ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = get,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Header}, [], []),
+ {ok, {{"HTTP/1.1",ReturnCode, _}, Head, Body}} = R,
+ couch_log:debug("Swift: ~p~p ~n",[ReturnCode, Head]),
+ Body;
+ {not_authenticated,_} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+swift_head_object(Container, ObjName) ->
+ couch_log:debug("Swift: head object ~p/~p ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = head,
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Header}, [], []),
+ {ok, {{"HTTP/1.1",ReturnCode, _}, Head, _}} = R,
+ couch_log:debug("Swift: ~p~p~n",[ReturnCode, Head]),
+ ObjectSize = lists:filter(fun ({"content-length",_}) -> true ; (_) -> false end, Head),
+ EtagHeader = lists:filter(fun ({"etag",_}) -> true ; (_) -> false end, Head),
+ Etag = element(2,lists:nth(1,EtagHeader)),
+ {ObjectSizeNumeric,_} = string:to_integer(element(2,lists:nth(1,ObjectSize))),
+ couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n",[ObjectSizeNumeric, Etag]),
+ couch_log:debug("Etag in base16 ~p~n",[Etag]),
+ EtagDecode = hex_to_bin(Etag),
+ EtagMD5 = base64:encode(EtagDecode),
+ couch_log:debug("Etag in base64 ~p~n",[EtagMD5]),
+ {ObjectSizeNumeric,EtagMD5};
+ {auth_not_supported, Error} ->
+ {error,{element(2,Error),""}};
+ {not_authenticated, _} ->
+ {-1,""}
--- End diff --
What -1 stand for?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by rnewson <gi...@git.apache.org>.
Github user rnewson commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r48727510
--- Diff: src/fabric_att_handler.erl ---
@@ -0,0 +1,136 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(fabric_att_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([att_store/2, att/3, container/2, att_store/5, external_store/0]).
+
+
+%% ====================================================================
+%% External API functions. Exposes a generic API that can be used with
+%% other backend drivers
+%% ====================================================================
+
+att_store(FileName, Db, ContentLen, MimeType, Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Going to store ~p of length is ~p,"
+ ++ " ~p in the container: ~n",[FileName, ContentLen,
+ ContainerName]),
+ %TO-DO: No chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case (get_backend_driver()):put_object(ContainerName, FileName, MimeType,
+ [], Data) of
+ {ok,{"201",NewUrl}} ->
+ couch_log:debug("Object ~p created.Response code is 201 ~n",
+ [FileName]),
+ case (get_backend_driver()):head_object(ContainerName, FileName) of
+ {ok, {ObjectSize, EtagMD5}} ->
+ {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8),
+ ObjectSize, EtagMD5, (get_backend_driver()):store_id()}};
+ {error, _} ->
+ {error, Data}
+ end;
+ {error, _} ->
+ {error, Data}
+ end.
+
+att_store(Db, #doc{}=Doc) ->
+ att_store(Db, [Doc]);
+att_store(Db, Docs) when is_list(Docs) ->
+ DbName = container_name(Db),
+ [#doc{atts=Atts0} = Doc | Rest] = Docs,
+ if
+ length(Atts0) > 0 ->
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Docs
+ end.
+
+att(get, Db, Att) ->
+ DbName = container_name(Db),
+ [Name,AttLen,AttLenUser] = couch_att:fetch([name, att_len,
+ att_external_size], Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, "
+ ++ "stored length: ~p~n", [Name, AttLen, AttLenUser]),
+ NewData = (get_backend_driver()):get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser},
+ {disk_len, AttLenUser}], Att),
+ NewAtt;
+att(delete,Db, Att) ->
+ DbName = container_name(Db),
+ [Name] = couch_att:fetch([name],Att),
+ couch_log:debug("Delete ~p from ~p", [Name, DbName]).
+
+container(create,DbName) ->
+ couch_log:debug("Create container ~p~n", [DbName]),
+ case (get_backend_driver()):create_container(DbName) of
+ {ok, {"201", Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n", [Container]),
+ {ok, Container};
+ {error, _} ->
--- End diff --
why is the error type ignored?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41982564
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
--- End diff --
Imho, better split this function into multiple ones:
```
inline_att_store(Db, Doc) when is_record(Doc, #doc) ->
inline_atts_store(Db, [Doc]);
inline_att_store(Db, Docs) when is_list(Docs) ->
...
```
Same for attachment filtering below. Makes easy to follow the flow.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41981566
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully ~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
--- End diff --
Sorry, was confused. So it's flag. Definitely use `config:get_boolean` and true/false atoms elsewhere.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by rnewson <gi...@git.apache.org>.
Github user rnewson commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41989338
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
--- End diff --
general note that we don't include attributions in the code.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by gilv <gi...@git.apache.org>.
Github user gilv commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-168674762
@rnewson what is the style convention i should use? For example, there was a comment that my code should use 80 chars per line, but there is bunch of existing code with more than 80 in a line.
Is there any template for Eclipse so i can use it for style? spaces, tabs, etc..
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by lazedo <gi...@git.apache.org>.
Github user lazedo commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-179777024
@kxepal passing the `Options` to the driver would also allow for example to for form the complete url in a google drive driver implementation (folder name).
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on the pull request:
https://github.com/apache/couchdb-fabric/pull/33#issuecomment-179768803
@lazedo I think there is no practically need: these options could be fetched within att_store/2 transparently for the API user during driver resolve and without abstraction leaks.
Assume we have per-db driver configuration. Where such options will be stored? First case: in config. Then we do `Driver = get_backend_driver(Db)` and `Options = get_backend_driver_options(Db, Driver)` right before do any driver API call. Another case: db options are stored in _metadata db - but this changes nothing, only configuration backend. What else Options could be passed here?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] couchdb-fabric pull request: COUCHDB-769: Store attachments in the...
Posted by kxepal <gi...@git.apache.org>.
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41982028
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should be fixed.
--- End diff --
Start this comment with TODO for easy grep.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---