You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ni...@apache.org on 2014/03/07 09:17:38 UTC
couchdb commit: updated refs/heads/1.6.x to ce3e89d
Repository: couchdb
Updated Branches:
refs/heads/1.6.x af196a472 -> ce3e89dc9
Speed up and move couch_httpd:find_in_binary.
See https://issues.apache.org/jira/browse/COUCHDB-1953
(cherry picked from commit 824869c3c059d887da0dbe1cd04eb244c931c27b)
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/ce3e89dc
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/ce3e89dc
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/ce3e89dc
Branch: refs/heads/1.6.x
Commit: ce3e89dc9fef25aa0e038e5a621a1e7c732e66e2
Parents: af196a4
Author: NickNorth <No...@gmail.com>
Authored: Tue Dec 3 20:58:53 2013 +0000
Committer: NickNorth <No...@gmail.com>
Committed: Fri Mar 7 08:15:06 2014 +0000
----------------------------------------------------------------------
src/couchdb/couch_httpd.erl | 30 +---------------
src/couchdb/couch_util.erl | 32 +++++++++++++++++
test/etap/043-find-in-binary.t | 69 +++++++++++++++++++++++++++++++++++++
3 files changed, 102 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/ce3e89dc/src/couchdb/couch_httpd.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_httpd.erl b/src/couchdb/couch_httpd.erl
index 465bc7a..9245f4b 100644
--- a/src/couchdb/couch_httpd.erl
+++ b/src/couchdb/couch_httpd.erl
@@ -1003,7 +1003,7 @@ split_header(Line) ->
mochiweb_util:parse_header(Value)}].
read_until(#mp{data_fun=DataFun, buffer=Buffer}=Mp, Pattern, Callback) ->
- case find_in_binary(Pattern, Buffer) of
+ case couch_util:find_in_binary(Pattern, Buffer) of
not_found ->
Callback2 = Callback(Buffer),
{Buffer2, DataFun2} = DataFun(),
@@ -1079,34 +1079,6 @@ check_for_last(#mp{buffer=Buffer, data_fun=DataFun}=Mp) ->
data_fun = DataFun2})
end.
-find_in_binary(_B, <<>>) ->
- not_found;
-
-find_in_binary(B, Data) ->
- case binary:match(Data, [B], []) of
- nomatch ->
- partial_find(binary:part(B, {0, byte_size(B) - 1}),
- binary:part(Data, {byte_size(Data), -byte_size(Data) + 1}), 1);
- {Pos, _Len} ->
- {exact, Pos}
- end.
-
-partial_find(<<>>, _Data, _Pos) ->
- not_found;
-
-partial_find(B, Data, N) when byte_size(Data) > 0 ->
- case binary:match(Data, [B], []) of
- nomatch ->
- partial_find(binary:part(B, {0, byte_size(B) - 1}),
- binary:part(Data, {byte_size(Data), -byte_size(Data) + 1}), N + 1);
- {Pos, _Len} ->
- {partial, N + Pos}
- end;
-
-partial_find(_B, _Data, _N) ->
- not_found.
-
-
validate_bind_address(Address) ->
case inet_parse:address(Address) of
{ok, _} -> ok;
http://git-wip-us.apache.org/repos/asf/couchdb/blob/ce3e89dc/src/couchdb/couch_util.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_util.erl b/src/couchdb/couch_util.erl
index afe3528..2509bef 100644
--- a/src/couchdb/couch_util.erl
+++ b/src/couchdb/couch_util.erl
@@ -29,6 +29,7 @@
-export([encode_doc_id/1]).
-export([with_db/2]).
-export([rfc1123_date/0, rfc1123_date/1]).
+-export([find_in_binary/2]).
-include("couch_db.hrl").
@@ -487,3 +488,34 @@ month(9) -> "Sep";
month(10) -> "Oct";
month(11) -> "Nov";
month(12) -> "Dec".
+
+
+find_in_binary(_B, <<>>) ->
+ not_found;
+
+find_in_binary(B, Data) ->
+ case binary:match(Data, [B], []) of
+ nomatch ->
+ MatchLength = erlang:min(byte_size(B), byte_size(Data)),
+ match_prefix_at_end(binary:part(B, {0, MatchLength}),
+ binary:part(Data, {byte_size(Data), -MatchLength}),
+ MatchLength, byte_size(Data) - MatchLength);
+ {Pos, _Len} ->
+ {exact, Pos}
+ end.
+
+match_prefix_at_end(Prefix, Data, PrefixLength, N) ->
+ FirstCharMatches = binary:matches(Data, [binary:part(Prefix, {0, 1})], []),
+ match_rest_of_prefix(FirstCharMatches, Prefix, Data, PrefixLength, N).
+
+match_rest_of_prefix([], _Prefix, _Data, _PrefixLength, _N) ->
+ not_found;
+
+match_rest_of_prefix([{Pos, _Len} | Rest], Prefix, Data, PrefixLength, N) ->
+ case binary:match(binary:part(Data, {PrefixLength, Pos - PrefixLength}),
+ [binary:part(Prefix, {0, PrefixLength - Pos})], []) of
+ nomatch ->
+ match_rest_of_prefix(Rest, Prefix, Data, PrefixLength, N);
+ {_Pos, _Len1} ->
+ {partial, N + Pos}
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb/blob/ce3e89dc/test/etap/043-find-in-binary.t
----------------------------------------------------------------------
diff --git a/test/etap/043-find-in-binary.t b/test/etap/043-find-in-binary.t
new file mode 100755
index 0000000..d29aa8a
--- /dev/null
+++ b/test/etap/043-find-in-binary.t
@@ -0,0 +1,69 @@
+#!/usr/bin/env escript
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
+%% -*- erlang -*-
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+ test_util:init_code_path(),
+
+ etap:plan(length(cases())),
+ case (catch test()) of
+ ok ->
+ etap:end_tests();
+ Other ->
+ etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+ etap:bail(Other)
+ end,
+ ok.
+
+
+test() ->
+ lists:foreach(fun({Needle, Haystack, Result}) ->
+ try
+ Msg = io_lib:format("Looking for ~s in ~s", [Needle, Haystack]),
+ etap:is(couch_util:find_in_binary(Needle, Haystack), Result, Msg)
+ catch _T:_R ->
+ etap:diag("~p", [{_T, _R}])
+ end
+ end, cases()),
+ ok.
+
+
+cases() ->
+ [
+ {<<"foo">>, <<"foobar">>, {exact, 0}},
+ {<<"foo">>, <<"foofoo">>, {exact, 0}},
+ {<<"foo">>, <<"barfoo">>, {exact, 3}},
+ {<<"foo">>, <<"barfo">>, {partial, 3}},
+ {<<"f">>, <<"fobarfff">>, {exact, 0}},
+ {<<"f">>, <<"obarfff">>, {exact, 4}},
+ {<<"f">>, <<"obarggf">>, {exact, 6}},
+ {<<"f">>, <<"f">>, {exact, 0}},
+ {<<"f">>, <<"g">>, not_found},
+ {<<"foo">>, <<"f">>, {partial, 0}},
+ {<<"foo">>, <<"g">>, not_found},
+ {<<"foo">>, <<"">>, not_found},
+ {<<"fofo">>, <<"foofo">>, {partial, 3}},
+ {<<"foo">>, <<"gfobarfo">>, {partial, 6}},
+ {<<"foo">>, <<"gfobarf">>, {partial, 6}},
+ {<<"foo">>, <<"gfobar">>, not_found},
+ {<<"fog">>, <<"gbarfogquiz">>, {exact, 4}},
+ {<<"ggg">>, <<"ggg">>, {exact, 0}},
+ {<<"ggg">>, <<"ggggg">>, {exact, 0}},
+ {<<"ggg">>, <<"bggg">>, {exact, 1}},
+ {<<"ggg">>, <<"bbgg">>, {partial, 2}},
+ {<<"ggg">>, <<"bbbg">>, {partial, 3}},
+ {<<"ggg">>, <<"bgbggbggg">>, {exact, 6}},
+ {<<"ggg">>, <<"bgbggb">>, not_found}
+ ].
Re: couchdb commit: updated refs/heads/1.6.x to ce3e89d
Posted by Andy Wenk <an...@apache.org>.
On 7 March 2014 23:44, Dirkjan Ochtman <di...@ochtman.nl> wrote:
> On Fri, Mar 7, 2014 at 10:17 AM, Nick North <no...@gmail.com> wrote:
> > When Dave has time to post more detailed findings someone may look at
> them
> > and spot the problem quickly. Otherwise I'm not sure of the best way
> > forward: the community could take as long as it takes to get to the
> bottom
> > of the problem, or we could say that the issue probably relates only to
> > tests, remove the ones that break and put out a release while working on
> a
> > solution in the background. Neither option is great.
>
> Yeah, it's tricky.
>
> I'm going to be out of the country for a week or so, so I'll be less
> responsive.
>
> Cheers,
>
> Dirkjan
>
I have tested the 1.6.x branch at commit ce3e89dc9fef on my MBP:
All tests successful.
Files=51, Tests=1214, 254 wallclock secs ( 0.55 usr 0.11 sys + 84.92 cusr
19.49 csys = 105.07 CPU)
Result: PASS
Yay :)
Erlang R16B03-1 (erts-5.10.4)
I am +1 for starting a vote on 1.6-rc2
Cheers
Andy
--
Andy Wenk
Hamburg - Germany
RockIt!
GPG fingerprint: C044 8322 9E12 1483 4FEC 9452 B65D 6BE3 9ED3 9588
https://people.apache.org/keys/committer/andywenk.asc
Re: couchdb commit: updated refs/heads/1.6.x to ce3e89d
Posted by Dirkjan Ochtman <di...@ochtman.nl>.
On Fri, Mar 7, 2014 at 10:17 AM, Nick North <no...@gmail.com> wrote:
> When Dave has time to post more detailed findings someone may look at them
> and spot the problem quickly. Otherwise I'm not sure of the best way
> forward: the community could take as long as it takes to get to the bottom
> of the problem, or we could say that the issue probably relates only to
> tests, remove the ones that break and put out a release while working on a
> solution in the background. Neither option is great.
Yeah, it's tricky.
I'm going to be out of the country for a week or so, so I'll be less responsive.
Cheers,
Dirkjan
Re: couchdb commit: updated refs/heads/1.6.x to ce3e89d
Posted by Nick North <no...@gmail.com>.
Dave and I spent some time on this yesterday. He has the notes so can give
a better summary than I can, but our feeling is that something bad is going
on with replication and this patch at best masks it. However the problem
only seems to arise within etap tests where database creation and
replication happens in Erlang - do everything via the HTTP API and
something happens between operations that allows it to work OK.
When Dave has time to post more detailed findings someone may look at them
and spot the problem quickly. Otherwise I'm not sure of the best way
forward: the community could take as long as it takes to get to the bottom
of the problem, or we could say that the issue probably relates only to
tests, remove the ones that break and put out a release while working on a
solution in the background. Neither option is great.
Nick
On 7 March 2014 08:48, Dirkjan Ochtman <di...@ochtman.nl> wrote:
> On Fri, Mar 7, 2014 at 9:43 AM, Alexander Shorin <kx...@gmail.com> wrote:
> > I'll check it on FreeBSD to make sure for that and need someone to
> > confirm that problem is gone on OS X - just to avoid rc3 if we miss
> > something.
>
> Great, thanks!
>
Re: couchdb commit: updated refs/heads/1.6.x to ce3e89d
Posted by Dirkjan Ochtman <di...@ochtman.nl>.
On Fri, Mar 7, 2014 at 9:43 AM, Alexander Shorin <kx...@gmail.com> wrote:
> I'll check it on FreeBSD to make sure for that and need someone to
> confirm that problem is gone on OS X - just to avoid rc3 if we miss
> something.
Great, thanks!
Re: couchdb commit: updated refs/heads/1.6.x to ce3e89d
Posted by Alexander Shorin <kx...@gmail.com>.
I'll check it on FreeBSD to make sure for that and need someone to
confirm that problem is gone on OS X - just to avoid rc3 if we miss
something.
--
,,,^..^,,,
On Fri, Mar 7, 2014 at 12:32 PM, Dirkjan Ochtman <dj...@apache.org> wrote:
> On Fri, Mar 7, 2014 at 9:17 AM, <ni...@apache.org> wrote:
>> Repository: couchdb
>> Updated Branches:
>> refs/heads/1.6.x af196a472 -> ce3e89dc9
>>
>>
>> Speed up and move couch_httpd:find_in_binary.
>>
>> See https://issues.apache.org/jira/browse/COUCHDB-1953
>>
>> (cherry picked from commit 824869c3c059d887da0dbe1cd04eb244c931c27b)
>
> Cool, so we think is enough to base an 1.6-rc2 on, right?
>
> Cheers,
>
> Dirkjan
Re: couchdb commit: updated refs/heads/1.6.x to ce3e89d
Posted by Dirkjan Ochtman <dj...@apache.org>.
On Fri, Mar 7, 2014 at 9:17 AM, <ni...@apache.org> wrote:
> Repository: couchdb
> Updated Branches:
> refs/heads/1.6.x af196a472 -> ce3e89dc9
>
>
> Speed up and move couch_httpd:find_in_binary.
>
> See https://issues.apache.org/jira/browse/COUCHDB-1953
>
> (cherry picked from commit 824869c3c059d887da0dbe1cd04eb244c931c27b)
Cool, so we think is enough to base an 1.6-rc2 on, right?
Cheers,
Dirkjan