You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kudu.apache.org by "Alexey Serbin (JIRA)" <ji...@apache.org> on 2017/01/09 23:46:58 UTC
[jira] [Comment Edited] (KUDU-1825) kudu-jepsen reports
non-linearizable history for 'kill-all-tservers' scenario
[ https://issues.apache.org/jira/browse/KUDU-1825?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15813237#comment-15813237 ]
Alexey Serbin edited comment on KUDU-1825 at 1/9/17 11:46 PM:
--------------------------------------------------------------
Thank you for attaching those files already.
I'm carving out the appropriate parts of the kudu-tserver and kudu-master logs. Will update the script to put those into archive of the kudu-jenkins job artifacts, so less monkey work while working with tserver and master log files.
From the {{history.txt}}, the following snippet correspond to the operations in the SVG file (spanning between '1 :invoke :write 7' and '1 :ok :write 7'):
{noformat}
1 :invoke :write 7
5 :invoke :read nil
2 :invoke :write 4
4 :invoke :read nil
9 :invoke :read nil
0 :invoke :read nil
:nemesis :info :start {"aserbin-kudu-jepsen-ts-1.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-5.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-2.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-3.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-4.gce.cloudera.com" [:killed :kudu-tserver]}
3 :invoke :write 8
:nemesis :info :stop nil
9 :ok :read 2
8 :ok :read 2
:nemesis :info :stop {"aserbin-kudu-jepsen-ts-1.gce.cloudera.com" [:started :kudu-tserver], "aserbin-kudu-jepsen-ts-5.gce.cloudera.com" [:started :kudu-tserver], "aserbin-kudu-jepsen-ts-2.gce.cloudera.com" [:started :kudu-tserver], "aserbin-kudu-jepsen-ts-3.gce.cloudera.com" [:started :kudu-tserver], "aserbin-kudu-jepsen-ts-4.gce.cloudera.com" [:started :kudu-tserver]}
9 :invoke :read nil
9 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
7 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
6 :ok :read 2
0 :ok :read 2
0 :invoke :write 3
6 :invoke :read nil
6 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
4 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
5 :ok :read 2
4 :invoke :read nil
4 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
4 :invoke :read nil
4 :ok :read 2
2 :ok :write 4
8 :invoke :read nil
8 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
4 :invoke :read nil
4 :ok :read 2
4 :invoke :read nil
4 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
2 :invoke :write 4
2 :ok :write 4
8 :invoke :read nil
8 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
2 :invoke :read nil
2 :ok :read 4
4 :invoke :write 0
9 :invoke :read nil
9 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
2 :invoke :write 5
2 :ok :write 5
6 :invoke :read nil
6 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
2 :invoke :write 7
2 :ok :write 7
6 :invoke :read nil
6 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
2 :invoke :read nil
2 :ok :read 7
5 :invoke :read nil
5 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
2 :invoke :read nil
2 :ok :read 7
9 :invoke :read nil
9 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
9 :invoke :read nil
0 :ok :write 3
9 :ok :read 7
4 :ok :write 0
4 :invoke :write 7
4 :ok :write 7
5 :invoke :read nil
:nemesis :info :start nil
5 :ok :read 7
3 :ok :write 8
0 :invoke :read nil
0 :ok :read 8
2 :invoke :write 2
2 :ok :write 2
8 :invoke :read nil
8 :ok :read 2
6 :invoke :read nil
0 :invoke :write 7
0 :ok :write 7
6 :ok :read 2
0 :invoke :read nil
0 :ok :read 7
0 :invoke :write 9
2 :invoke :read nil
5 :invoke :read nil
2 :ok :read 9
0 :ok :write 9
7 :invoke :read nil
8 :invoke :read nil
5 :ok :read 9
8 :ok :read 9
7 :ok :read 9
4 :invoke :read nil
4 :ok :read 9
5 :invoke :read nil
9 :invoke :read nil
6 :invoke :read nil
7 :invoke :read nil
3 :invoke :write 7
4 :invoke :read nil
2 :invoke :write 7
8 :invoke :read nil
0 :invoke :write 0
:nemesis :info :start {"aserbin-kudu-jepsen-ts-4.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-3.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-2.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-5.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-1.gce.cloudera.com" [:killed :kudu-tserver]}
:nemesis :info :stop nil
8 :ok :read 9
4 :ok :read 9
8 :invoke :read nil
8 :ok :read 9
4 :invoke :write 4
4 :ok :write 4
3 :ok :write 7
3 :invoke :write 3
3 :ok :write 3
4 :invoke :read nil
4 :ok :read 3
6 :ok :read 3
8 :invoke :read nil
8 :ok :read 3
3 :invoke :read nil
3 :ok :read 3
4 :invoke :read nil
4 :ok :read 3
0 :ok :write 0
0 :invoke :read nil
0 :ok :read 0
6 :invoke :read nil
6 :ok :read 0
3 :invoke :read nil
3 :ok :read 0
3 :invoke :read nil
3 :ok :read 0
4 :invoke :read nil
4 :ok :read 0
8 :invoke :read nil
8 :ok :read 0
7 :ok :read 0
0 :invoke :write 0
0 :ok :write 0
9 :ok :read 0
6 :invoke :read nil
6 :ok :read 0
9 :invoke :read nil
7 :invoke :read nil
7 :ok :read 0
9 :ok :read 0
8 :invoke :read nil
8 :ok :read 0
8 :invoke :read nil
8 :ok :read 0
3 :invoke :write 9
3 :ok :write 9
5 :ok :read 9
4 :invoke :read nil
4 :ok :read 9
7 :invoke :read nil
7 :ok :read 9
8 :invoke :read nil
8 :ok :read 9
1 :ok :write 7
{noformat}
was (Author: aserbin):
Thank you for attaching those files already.
I'm carving out the appropriate parts of the kudu-tserver and kudu-master logs. Will update the script to put those into archive of the kudu-jenkins job artifacts.
From the {{history.txt}}, the following snippet correspond to the operations in the SVG file (spanning between '1 :invoke :write 7' and '1 :ok :write 7'):
{noformat}
1 :invoke :write 7
5 :invoke :read nil
2 :invoke :write 4
4 :invoke :read nil
9 :invoke :read nil
0 :invoke :read nil
:nemesis :info :start {"aserbin-kudu-jepsen-ts-1.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-5.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-2.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-3.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-4.gce.cloudera.com" [:killed :kudu-tserver]}
3 :invoke :write 8
:nemesis :info :stop nil
9 :ok :read 2
8 :ok :read 2
:nemesis :info :stop {"aserbin-kudu-jepsen-ts-1.gce.cloudera.com" [:started :kudu-tserver], "aserbin-kudu-jepsen-ts-5.gce.cloudera.com" [:started :kudu-tserver], "aserbin-kudu-jepsen-ts-2.gce.cloudera.com" [:started :kudu-tserver], "aserbin-kudu-jepsen-ts-3.gce.cloudera.com" [:started :kudu-tserver], "aserbin-kudu-jepsen-ts-4.gce.cloudera.com" [:started :kudu-tserver]}
9 :invoke :read nil
9 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
7 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
6 :ok :read 2
0 :ok :read 2
0 :invoke :write 3
6 :invoke :read nil
6 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
4 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
5 :ok :read 2
4 :invoke :read nil
4 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
4 :invoke :read nil
4 :ok :read 2
2 :ok :write 4
8 :invoke :read nil
8 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
4 :invoke :read nil
4 :ok :read 2
4 :invoke :read nil
4 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
2 :invoke :write 4
2 :ok :write 4
8 :invoke :read nil
8 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
2 :invoke :read nil
2 :ok :read 4
4 :invoke :write 0
9 :invoke :read nil
9 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
2 :invoke :write 5
2 :ok :write 5
6 :invoke :read nil
6 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
9 :invoke :read nil
9 :ok :read 2
2 :invoke :write 7
2 :ok :write 7
6 :invoke :read nil
6 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
2 :invoke :read nil
2 :ok :read 7
5 :invoke :read nil
5 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
8 :invoke :read nil
8 :ok :read 2
2 :invoke :read nil
2 :ok :read 7
9 :invoke :read nil
9 :ok :read 2
5 :invoke :read nil
5 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
7 :invoke :read nil
7 :ok :read 2
6 :invoke :read nil
6 :ok :read 2
9 :invoke :read nil
0 :ok :write 3
9 :ok :read 7
4 :ok :write 0
4 :invoke :write 7
4 :ok :write 7
5 :invoke :read nil
:nemesis :info :start nil
5 :ok :read 7
3 :ok :write 8
0 :invoke :read nil
0 :ok :read 8
2 :invoke :write 2
2 :ok :write 2
8 :invoke :read nil
8 :ok :read 2
6 :invoke :read nil
0 :invoke :write 7
0 :ok :write 7
6 :ok :read 2
0 :invoke :read nil
0 :ok :read 7
0 :invoke :write 9
2 :invoke :read nil
5 :invoke :read nil
2 :ok :read 9
0 :ok :write 9
7 :invoke :read nil
8 :invoke :read nil
5 :ok :read 9
8 :ok :read 9
7 :ok :read 9
4 :invoke :read nil
4 :ok :read 9
5 :invoke :read nil
9 :invoke :read nil
6 :invoke :read nil
7 :invoke :read nil
3 :invoke :write 7
4 :invoke :read nil
2 :invoke :write 7
8 :invoke :read nil
0 :invoke :write 0
:nemesis :info :start {"aserbin-kudu-jepsen-ts-4.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-3.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-2.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-5.gce.cloudera.com" [:killed :kudu-tserver], "aserbin-kudu-jepsen-ts-1.gce.cloudera.com" [:killed :kudu-tserver]}
:nemesis :info :stop nil
8 :ok :read 9
4 :ok :read 9
8 :invoke :read nil
8 :ok :read 9
4 :invoke :write 4
4 :ok :write 4
3 :ok :write 7
3 :invoke :write 3
3 :ok :write 3
4 :invoke :read nil
4 :ok :read 3
6 :ok :read 3
8 :invoke :read nil
8 :ok :read 3
3 :invoke :read nil
3 :ok :read 3
4 :invoke :read nil
4 :ok :read 3
0 :ok :write 0
0 :invoke :read nil
0 :ok :read 0
6 :invoke :read nil
6 :ok :read 0
3 :invoke :read nil
3 :ok :read 0
3 :invoke :read nil
3 :ok :read 0
4 :invoke :read nil
4 :ok :read 0
8 :invoke :read nil
8 :ok :read 0
7 :ok :read 0
0 :invoke :write 0
0 :ok :write 0
9 :ok :read 0
6 :invoke :read nil
6 :ok :read 0
9 :invoke :read nil
7 :invoke :read nil
7 :ok :read 0
9 :ok :read 0
8 :invoke :read nil
8 :ok :read 0
8 :invoke :read nil
8 :ok :read 0
3 :invoke :write 9
3 :ok :write 9
5 :ok :read 9
4 :invoke :read nil
4 :ok :read 9
7 :invoke :read nil
7 :ok :read 9
8 :invoke :read nil
8 :ok :read 9
1 :ok :write 7
{noformat}
> kudu-jepsen reports non-linearizable history for 'kill-all-tservers' scenario
> -----------------------------------------------------------------------------
>
> Key: KUDU-1825
> URL: https://issues.apache.org/jira/browse/KUDU-1825
> Project: Kudu
> Issue Type: Bug
> Reporter: Alexey Serbin
> Attachments: history.txt, linear.svg
>
>
> One of the iterations of the kudu-jepsen test hit non-linearizable history of opertions: http://sandbox.jenkins.cloudera.com/view/Kudu/job/kudu-jepsen/82/
> It's necessary to create a reproducible scenario for that and fix the issue.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)