You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by "Cathy Daw (JIRA)" <ji...@apache.org> on 2011/07/19 02:52:57 UTC
[jira] [Created] (CASSANDRA-2918) After repair, one row missing
from query when two rows expected
After repair, one row missing from query when two rows expected
---------------------------------------------------------------
Key: CASSANDRA-2918
URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
Project: Cassandra
Issue Type: Bug
Components: Core
Affects Versions: 0.8.2
Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
Reporter: Cathy Daw
*Cluster Config*
{code}
cathy1 - 50.57.114.45 - Token: 0
cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
{code}
*1) Create Seed Data*
{code}
create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
{code}
*2) Kill cathy3: 50.57.114.39*
{code}
root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
{code}
*3) Run cassandra-cli*
{code}
use testKS;
create column family metadataCF
with key_validation_class = 'AsciiType'
and comparator = 'AsciiType'
and column_metadata = [
{column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
{column_name: byte_col, validation_class: BytesType, index_type: KEYS},
{column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
{column_name: int_col, validation_class: IntegerType, index_type: KEYS},
{column_name: long_col, validation_class: LongType, index_type: KEYS},
{column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
set metadataCF['key1']['byte_col']=bytes('10101010');
set metadataCF['key1']['uuid_col']=timeuuid();
set metadataCF['key1']['int_col']=integer(1000);
set metadataCF['key1']['long_col']=long(44444444444);
set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
consistencylevel as TWO;
set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
set metadataCF['key2']['byte_col']=bytes('201010102');
set metadataCF['key2']['uuid_col']=timeuuid();
set metadataCF['key2']['int_col']=integer(2000);
set metadataCF['key2']['long_col']=long(222222222);
set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
{code}
{code}
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
*4) Start Cassandra on cathy3*
*5) Run repair on cathy3*
{code}
nodetool -h cathy3 repair testKS metadataCF
{code}
{code}
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,626 Stream context metadata [], 0 sstables.
INFO 00:38:33,643 Streaming to /50.57.114.45
INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
{code}
*6) Kill cathy1 and cathy2*
{code}
root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
{code}
*7) Log into cassandra-cli on node3*
{code}
./cassandra-cli -h cathy3
Connected to: "Test Cluster" on cathy3/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] consistencylevel as ONE;
Consistency level is set to 'ONE'.
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Issue Comment Edited] (CASSANDRA-2918) After repair, one
row missing from query when two rows expected
Posted by "Cathy Daw (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13067882#comment-13067882 ]
Cathy Daw edited comment on CASSANDRA-2918 at 7/19/11 11:24 PM:
----------------------------------------------------------------
Interestingly, I did the same test case using the hector stress tool which writes @ CL=QUORUM.
*Insert 1000 rows. 691 Rows Returned.*
* Bring up 3 nodes. Create a KS with RF=3.
{code}
[default@StressKeyspace] describe keyspace StressKeyspace;
Keyspace: StressKeyspace:
Replication Strategy: org.apache.cassandra.locator.SimpleStrategy
Durable Writes: true
Options: [replication_factor:3]
Column Families:
ColumnFamily: StressStandard
Key Validation Class: org.apache.cassandra.db.marshal.BytesType
Default column value validator: org.apache.cassandra.db.marshal.BytesType
Columns sorted by: org.apache.cassandra.db.marshal.UTF8Type
Row cache size / save period in seconds: 0.0/0
Key cache size / save period in seconds: 10000.0/14400
Memtable thresholds: 0.2859375/1440/32 (millions of ops/MB/minutes)
GC grace seconds: 864000
Compaction min/max thresholds: 4/32
Read repair chance: 1.0
Replicate on write: true
Built indexes: []
{code}
* Kill one node
*TEST #1: first two nodes don't agree*
* Insert 1000 rows with 10 columns each in batches of 100 into standard table.
{code}
Get code from: [https://github.com/zznate/cassandra-stress]
sh bin/stress -o insert -t 2 -b 100 -n 1000 -c 10 node1:9160
{code}
* Verify Data:
** node1: 691 rows
** node2: 1000 rows
** Nate verified the default is to write at QUORUM
* Bring up dead node. Run Repair.
* Kill two other nodes.
* Run in cli: consistencylevel as ONE;
* Run in cli: list StressStandard limit 2000;
** node3: 691 rows
*TEST #2: 1st two nodes agree*
* Insert 1000 rows with 10 columns each in batches of 100 into standard table. Explicitly specify QUORUM.
{code}
sh bin/stress -o insert -t 2 -b 100 -n 1000 -c 10 -L QUORUM:QUORUM node1:9160
{code}
* Verify Data:
** node1: 1000 rows
** node2: 1000 rows
* Bring up dead node. Run Repair.
* Kill two other nodes.
* Run in cli: consistencylevel as ONE.
* Run in cli: list StressStandard limit 2000;
** node3: 332 rows
*TEST #3*
* flush node1 and node2
* Bring up dead node - node3. Run Repair.
* Kill node1 and node2
* Run in cli: consistencylevel as ONE.
* Run in cli: list StressStandard limit 2000;
** node3: 1000 rows
was (Author: cdaw):
Interestingly, I did the same test case using the hector stress tool which writes @ CL=QUORUM.
*Insert 1000 rows. 691 Rows Returned.*
* Bring up 3 nodes. Create a KS with RF=3.
{code}
[default@StressKeyspace] describe keyspace StressKeyspace;
Keyspace: StressKeyspace:
Replication Strategy: org.apache.cassandra.locator.SimpleStrategy
Durable Writes: true
Options: [replication_factor:3]
Column Families:
ColumnFamily: StressStandard
Key Validation Class: org.apache.cassandra.db.marshal.BytesType
Default column value validator: org.apache.cassandra.db.marshal.BytesType
Columns sorted by: org.apache.cassandra.db.marshal.UTF8Type
Row cache size / save period in seconds: 0.0/0
Key cache size / save period in seconds: 10000.0/14400
Memtable thresholds: 0.2859375/1440/32 (millions of ops/MB/minutes)
GC grace seconds: 864000
Compaction min/max thresholds: 4/32
Read repair chance: 1.0
Replicate on write: true
Built indexes: []
{code}
* Kill one node
*TEST #1: first two nodes don't agree*
* Insert 1000 rows with 10 columns each in batches of 100 into standard table.
{code}
sh bin/stress -o insert -t 2 -b 100 -n 1000 -c 10 node1:9160
{code}
* Verify Data:
** node1: 691 rows
** node2: 1000 rows
** Nate verified the default is to write at QUORUM
* Bring up dead node. Run Repair.
* Kill two other nodes.
* Run in cli: consistencylevel as ONE;
* Run in cli: list StressStandard limit 2000;
** node3: 691 rows
*TEST #2: 1st two nodes agree*
* Insert 1000 rows with 10 columns each in batches of 100 into standard table. Explicitly specify QUORUM.
{code}
sh bin/stress -o insert -t 2 -b 100 -n 1000 -c 10 -L QUORUM:QUORUM node1:9160
{code}
* Verify Data:
** node1: 1000 rows
** node2: 1000 rows
* Bring up dead node. Run Repair.
* Kill two other nodes.
* Run in cli: consistencylevel as ONE.
* Run in cli: list StressStandard limit 2000;
** node3: 332 rows
> After repair, one row missing from query when two rows expected
> ---------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run java stress tool+*
> {code}
> ./bin/stress -o insert -n 1000 -c 10 -l 3 -e QUORUM -d 50.57.114.45,50.57.107.176
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair Keyspace1 Standard1
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on cathy3 - expect 1000 rows returned+*
> {code}
> [default@Keyspace1] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@Keyspace1] list Standard1 limit 2000;
> .....
> 323 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Assigned] (CASSANDRA-2918) After repair, one row missing
from query when two rows expected
Posted by "Jonathan Ellis (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Jonathan Ellis reassigned CASSANDRA-2918:
-----------------------------------------
Assignee: Sylvain Lebresne
> After repair, one row missing from query when two rows expected
> ---------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+1) Create Seed Data+*
> {code}
> create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run cassandra-cli+*
> {code}
> use testKS;
> create column family metadataCF
> with key_validation_class = 'AsciiType'
> and comparator = 'AsciiType'
> and column_metadata = [
> {column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
> {column_name: byte_col, validation_class: BytesType, index_type: KEYS},
> {column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
> {column_name: int_col, validation_class: IntegerType, index_type: KEYS},
> {column_name: long_col, validation_class: LongType, index_type: KEYS},
> {column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
> set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
> set metadataCF['key1']['byte_col']=bytes('10101010');
> set metadataCF['key1']['uuid_col']=timeuuid();
> set metadataCF['key1']['int_col']=integer(1000);
> set metadataCF['key1']['long_col']=long(44444444444);
> set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
> //Please note: forget to change CL before inserting 'key1', so that was inserted with CL=ONE by default.
> consistencylevel as TWO;
> set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
> set metadataCF['key2']['byte_col']=bytes('201010102');
> set metadataCF['key2']['uuid_col']=timeuuid();
> set metadataCF['key2']['int_col']=integer(2000);
> set metadataCF['key2']['long_col']=long(222222222);
> set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
> //Assumed that the following read would be done on CL=TWO and will the second replica would be guaranteed to be fix for 'key1'.
> list metadataCF;
> {code}
> {code}
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair testKS metadataCF
> {code}
> {code}
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
> INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
> INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
> INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
> INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
> INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
> INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
> INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
> INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
> INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,626 Stream context metadata [], 0 sstables.
> INFO 00:38:33,643 Streaming to /50.57.114.45
> INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
> INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
> INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
> INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
> INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
> INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
> INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
> INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
> INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
> INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
> INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
> INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
> INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
> INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on node3+*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+8) Bring up nodes 1 and 2 and query+*
> *NODE1 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy1
> Connected to: "Test Cluster" on cathy1/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *NODE2 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy2
> Connected to: "Test Cluster" on cathy2/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *NODE3 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+9) Force Read Repair on node1+*
> {code}
> consistencylevel as THREE;
> Consistency level is set to 'THREE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (CASSANDRA-2918) After repair, one row missing
from query when two rows expected
Posted by "Cathy Daw (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Cathy Daw updated CASSANDRA-2918:
---------------------------------
Description:
*Cluster Config*
{code}
cathy1 - 50.57.114.45 - Token: 0
cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
{code}
*+2) Kill cathy3: 50.57.114.39+*
{code}
root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
{code}
*+3) Run java stress tool+*
{code}
./bin/stress -o insert -n 1000 -c 10 -l 3 -e QUORUM -d 50.57.114.45,50.57.107.176
{code}
*+4) Start Cassandra on cathy3+*
*+5) Run repair on cathy3+*
{code}
nodetool -h cathy3 repair Keyspace1 Standard1
{code}
*+6) Kill cathy1 and cathy2+*
{code}
root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
{code}
*+7) Log into cassandra-cli on cathy3 - expect 1000 rows returned+*
{code}
[default@Keyspace1] consistencylevel as ONE;
Consistency level is set to 'ONE'.
[default@Keyspace1] list Standard1 limit 2000;
.....
323 Rows Returned.
{code}
was:
*Cluster Config*
{code}
cathy1 - 50.57.114.45 - Token: 0
cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
{code}
*+1) Create Seed Data+*
{code}
create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
{code}
*+2) Kill cathy3: 50.57.114.39+*
{code}
root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
{code}
*+3) Run cassandra-cli+*
{code}
use testKS;
create column family metadataCF
with key_validation_class = 'AsciiType'
and comparator = 'AsciiType'
and column_metadata = [
{column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
{column_name: byte_col, validation_class: BytesType, index_type: KEYS},
{column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
{column_name: int_col, validation_class: IntegerType, index_type: KEYS},
{column_name: long_col, validation_class: LongType, index_type: KEYS},
{column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
set metadataCF['key1']['byte_col']=bytes('10101010');
set metadataCF['key1']['uuid_col']=timeuuid();
set metadataCF['key1']['int_col']=integer(1000);
set metadataCF['key1']['long_col']=long(44444444444);
set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
//Please note: forget to change CL before inserting 'key1', so that was inserted with CL=ONE by default.
consistencylevel as TWO;
set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
set metadataCF['key2']['byte_col']=bytes('201010102');
set metadataCF['key2']['uuid_col']=timeuuid();
set metadataCF['key2']['int_col']=integer(2000);
set metadataCF['key2']['long_col']=long(222222222);
set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
//Assumed that the following read would be done on CL=TWO and will the second replica would be guaranteed to be fix for 'key1'.
list metadataCF;
{code}
{code}
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
*+4) Start Cassandra on cathy3+*
*+5) Run repair on cathy3+*
{code}
nodetool -h cathy3 repair testKS metadataCF
{code}
{code}
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,626 Stream context metadata [], 0 sstables.
INFO 00:38:33,643 Streaming to /50.57.114.45
INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
{code}
*+6) Kill cathy1 and cathy2+*
{code}
root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
{code}
*+7) Log into cassandra-cli on node3+*
{code}
./cassandra-cli -h cathy3
Connected to: "Test Cluster" on cathy3/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] consistencylevel as ONE;
Consistency level is set to 'ONE'.
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*+8) Bring up nodes 1 and 2 and query+*
*NODE1 - cassandra-cli*
{code}
./cassandra-cli -h cathy1
Connected to: "Test Cluster" on cathy1/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*NODE2 - cassandra-cli*
{code}
./cassandra-cli -h cathy2
Connected to: "Test Cluster" on cathy2/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
*NODE3 - cassandra-cli*
{code}
./cassandra-cli -h cathy3
Connected to: "Test Cluster" on cathy3/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*+9) Force Read Repair on node1+*
{code}
consistencylevel as THREE;
Consistency level is set to 'THREE'.
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
> After repair, one row missing from query when two rows expected
> ---------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run java stress tool+*
> {code}
> ./bin/stress -o insert -n 1000 -c 10 -l 3 -e QUORUM -d 50.57.114.45,50.57.107.176
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair Keyspace1 Standard1
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on cathy3 - expect 1000 rows returned+*
> {code}
> [default@Keyspace1] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@Keyspace1] list Standard1 limit 2000;
> .....
> 323 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (CASSANDRA-2918) After repair, one row missing
from query when two rows expected
Posted by "Cathy Daw (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Cathy Daw updated CASSANDRA-2918:
---------------------------------
Description:
*Cluster Config*
{code}
cathy1 - 50.57.114.45 - Token: 0
cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
{code}
*+1) Create Seed Data+*
{code}
create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
{code}
*+2) Kill cathy3: 50.57.114.39+*
{code}
root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
{code}
*+3) Run cassandra-cli+*
{code}
use testKS;
create column family metadataCF
with key_validation_class = 'AsciiType'
and comparator = 'AsciiType'
and column_metadata = [
{column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
{column_name: byte_col, validation_class: BytesType, index_type: KEYS},
{column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
{column_name: int_col, validation_class: IntegerType, index_type: KEYS},
{column_name: long_col, validation_class: LongType, index_type: KEYS},
{column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
set metadataCF['key1']['byte_col']=bytes('10101010');
set metadataCF['key1']['uuid_col']=timeuuid();
set metadataCF['key1']['int_col']=integer(1000);
set metadataCF['key1']['long_col']=long(44444444444);
set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
consistencylevel as TWO;
set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
set metadataCF['key2']['byte_col']=bytes('201010102');
set metadataCF['key2']['uuid_col']=timeuuid();
set metadataCF['key2']['int_col']=integer(2000);
set metadataCF['key2']['long_col']=long(222222222);
set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
{code}
{code}
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
*+4) Start Cassandra on cathy3+*
*+5) Run repair on cathy3+*
{code}
nodetool -h cathy3 repair testKS metadataCF
{code}
{code}
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,626 Stream context metadata [], 0 sstables.
INFO 00:38:33,643 Streaming to /50.57.114.45
INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
{code}
*+6) Kill cathy1 and cathy2+*
{code}
root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
{code}
*+7) Log into cassandra-cli on node3+*
{code}
./cassandra-cli -h cathy3
Connected to: "Test Cluster" on cathy3/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] consistencylevel as ONE;
Consistency level is set to 'ONE'.
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*+8) Bring up nodes 1 and 2 and query+*
*NODE1 - cassandra-cli*
{code}
./cassandra-cli -h cathy1
Connected to: "Test Cluster" on cathy1/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*NODE2 - cassandra-cli*
{code}
./cassandra-cli -h cathy2
Connected to: "Test Cluster" on cathy2/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
*NODE3 - cassandra-cli*
{code}
./cassandra-cli -h cathy3
Connected to: "Test Cluster" on cathy3/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*+9) Force Read Repair on node1+*
{code}
consistencylevel as THREE;
Consistency level is set to 'THREE'.
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
was:
*Cluster Config*
{code}
cathy1 - 50.57.114.45 - Token: 0
cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
{code}
*1) Create Seed Data*
{code}
create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
{code}
*2) Kill cathy3: 50.57.114.39*
{code}
root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
{code}
*3) Run cassandra-cli*
{code}
use testKS;
create column family metadataCF
with key_validation_class = 'AsciiType'
and comparator = 'AsciiType'
and column_metadata = [
{column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
{column_name: byte_col, validation_class: BytesType, index_type: KEYS},
{column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
{column_name: int_col, validation_class: IntegerType, index_type: KEYS},
{column_name: long_col, validation_class: LongType, index_type: KEYS},
{column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
set metadataCF['key1']['byte_col']=bytes('10101010');
set metadataCF['key1']['uuid_col']=timeuuid();
set metadataCF['key1']['int_col']=integer(1000);
set metadataCF['key1']['long_col']=long(44444444444);
set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
consistencylevel as TWO;
set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
set metadataCF['key2']['byte_col']=bytes('201010102');
set metadataCF['key2']['uuid_col']=timeuuid();
set metadataCF['key2']['int_col']=integer(2000);
set metadataCF['key2']['long_col']=long(222222222);
set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
{code}
{code}
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
*4) Start Cassandra on cathy3*
*5) Run repair on cathy3*
{code}
nodetool -h cathy3 repair testKS metadataCF
{code}
{code}
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,626 Stream context metadata [], 0 sstables.
INFO 00:38:33,643 Streaming to /50.57.114.45
INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
{code}
*6) Kill cathy1 and cathy2*
{code}
root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
{code}
*7) Log into cassandra-cli on node3*
{code}
./cassandra-cli -h cathy3
Connected to: "Test Cluster" on cathy3/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] consistencylevel as ONE;
Consistency level is set to 'ONE'.
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
> After repair, one row missing from query when two rows expected
> ---------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+1) Create Seed Data+*
> {code}
> create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run cassandra-cli+*
> {code}
> use testKS;
> create column family metadataCF
> with key_validation_class = 'AsciiType'
> and comparator = 'AsciiType'
> and column_metadata = [
> {column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
> {column_name: byte_col, validation_class: BytesType, index_type: KEYS},
> {column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
> {column_name: int_col, validation_class: IntegerType, index_type: KEYS},
> {column_name: long_col, validation_class: LongType, index_type: KEYS},
> {column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
> set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
> set metadataCF['key1']['byte_col']=bytes('10101010');
> set metadataCF['key1']['uuid_col']=timeuuid();
> set metadataCF['key1']['int_col']=integer(1000);
> set metadataCF['key1']['long_col']=long(44444444444);
> set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
> consistencylevel as TWO;
> set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
> set metadataCF['key2']['byte_col']=bytes('201010102');
> set metadataCF['key2']['uuid_col']=timeuuid();
> set metadataCF['key2']['int_col']=integer(2000);
> set metadataCF['key2']['long_col']=long(222222222);
> set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
> {code}
> {code}
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair testKS metadataCF
> {code}
> {code}
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
> INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
> INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
> INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
> INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
> INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
> INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
> INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
> INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
> INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,626 Stream context metadata [], 0 sstables.
> INFO 00:38:33,643 Streaming to /50.57.114.45
> INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
> INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
> INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
> INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
> INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
> INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
> INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
> INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
> INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
> INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
> INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
> INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
> INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
> INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on node3+*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+8) Bring up nodes 1 and 2 and query+*
> *NODE1 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy1
> Connected to: "Test Cluster" on cathy1/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *NODE2 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy2
> Connected to: "Test Cluster" on cathy2/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *NODE3 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+9) Force Read Repair on node1+*
> {code}
> consistencylevel as THREE;
> Consistency level is set to 'THREE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Issue Comment Edited] (CASSANDRA-2918) After repair, one
row missing from query when two rows expected
Posted by "Cathy Daw (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13067896#comment-13067896 ]
Cathy Daw edited comment on CASSANDRA-2918 at 7/19/11 7:30 PM:
---------------------------------------------------------------
This looks more to me like a WRITE with CL.Q bug as the source of the problem. But shouldn't have repair detected that node1 and node2 were out of sync?
* 3 node cluster / KS with RF=3 / 2 nodes up
* Insert 1000 rows on node1 with CL=QUORUM
** node1:691 rows
** node2:1000 rows
** node3: down
* Repair node3
** node1:691 rows
** node2:1000 rows
** node3: 691 rows
was (Author: cdaw):
This looks more to me like a WRITE with CL.Q bug as the source of the problem. But shouldn't have repair detected that node1 and node2 were out of sync?
* 3 node cluster / KS with RF=3 / 2 nodes up
* Insert 1000 rows on node2 with CL=QUORUM
** node1:691 rows
** node2:1000 rows
** node3: down
* Repair node3
** node1:691 rows
** node2:1000 rows
** node3: 691 rows
> After repair, one row missing from query when two rows expected
> ---------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+1) Create Seed Data+*
> {code}
> create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run cassandra-cli+*
> {code}
> use testKS;
> create column family metadataCF
> with key_validation_class = 'AsciiType'
> and comparator = 'AsciiType'
> and column_metadata = [
> {column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
> {column_name: byte_col, validation_class: BytesType, index_type: KEYS},
> {column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
> {column_name: int_col, validation_class: IntegerType, index_type: KEYS},
> {column_name: long_col, validation_class: LongType, index_type: KEYS},
> {column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
> set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
> set metadataCF['key1']['byte_col']=bytes('10101010');
> set metadataCF['key1']['uuid_col']=timeuuid();
> set metadataCF['key1']['int_col']=integer(1000);
> set metadataCF['key1']['long_col']=long(44444444444);
> set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
> //Please note: forget to change CL before inserting 'key1', so that was inserted with CL=ONE by default.
> consistencylevel as TWO;
> set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
> set metadataCF['key2']['byte_col']=bytes('201010102');
> set metadataCF['key2']['uuid_col']=timeuuid();
> set metadataCF['key2']['int_col']=integer(2000);
> set metadataCF['key2']['long_col']=long(222222222);
> set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
> //Assumed that the following read would be done on CL=TWO and will the second replica would be guaranteed to be fix for 'key1'.
> list metadataCF;
> {code}
> {code}
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair testKS metadataCF
> {code}
> {code}
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
> INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
> INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
> INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
> INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
> INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
> INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
> INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
> INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
> INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,626 Stream context metadata [], 0 sstables.
> INFO 00:38:33,643 Streaming to /50.57.114.45
> INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
> INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
> INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
> INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
> INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
> INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
> INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
> INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
> INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
> INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
> INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
> INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
> INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
> INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on node3+*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+8) Bring up nodes 1 and 2 and query+*
> *NODE1 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy1
> Connected to: "Test Cluster" on cathy1/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *NODE2 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy2
> Connected to: "Test Cluster" on cathy2/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *NODE3 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+9) Force Read Repair on node1+*
> {code}
> consistencylevel as THREE;
> Consistency level is set to 'THREE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Issue Comment Edited] (CASSANDRA-2918) After repair, one
row missing from query when two rows expected
Posted by "Cathy Daw (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13067882#comment-13067882 ]
Cathy Daw edited comment on CASSANDRA-2918 at 7/19/11 9:38 PM:
---------------------------------------------------------------
Interestingly, I did the same test case using the hector stress tool which writes @ CL=QUORUM.
*Insert 1000 rows. 691 Rows Returned.*
* Bring up 3 nodes. Create a KS with RF=3.
{code}
[default@StressKeyspace] describe keyspace StressKeyspace;
Keyspace: StressKeyspace:
Replication Strategy: org.apache.cassandra.locator.SimpleStrategy
Durable Writes: true
Options: [replication_factor:3]
Column Families:
ColumnFamily: StressStandard
Key Validation Class: org.apache.cassandra.db.marshal.BytesType
Default column value validator: org.apache.cassandra.db.marshal.BytesType
Columns sorted by: org.apache.cassandra.db.marshal.UTF8Type
Row cache size / save period in seconds: 0.0/0
Key cache size / save period in seconds: 10000.0/14400
Memtable thresholds: 0.2859375/1440/32 (millions of ops/MB/minutes)
GC grace seconds: 864000
Compaction min/max thresholds: 4/32
Read repair chance: 1.0
Replicate on write: true
Built indexes: []
{code}
* Kill one node
*TEST #1: first two nodes don't agree*
* Insert 1000 rows with 10 columns each in batches of 100 into standard table.
{code}
sh bin/stress -o insert -t 2 -b 100 -n 1000 -c 10 node1:9160
{code}
* Verify Data:
** node1: 691 rows
** node2: 1000 rows
** Nate verified the default is to write at QUORUM
* Bring up dead node. Run Repair.
* Kill two other nodes.
* Run in cli: consistencylevel as ONE;
* Run in cli: list StressStandard limit 2000;
** node3: 691 rows
*TEST #2: 1st two nodes agree*
* Insert 1000 rows with 10 columns each in batches of 100 into standard table. Explicitly specify QUORUM.
{code}
sh bin/stress -o insert -t 2 -b 100 -n 1000 -c 10 -L QUORUM:QUORUM node1:9160
{code}
* Verify Data:
** node1: 1000 rows
** node2: 1000 rows
* Bring up dead node. Run Repair.
* Kill two other nodes.
* Run in cli: consistencylevel as ONE.
* Run in cli: list StressStandard limit 2000;
** node3: 332 rows
was (Author: cdaw):
Interestingly, I did the same test case using the hector stress tool which writes @ CL=QUORUM.
*Insert 1000 rows. 691 Rows Returned.*
* Bring up 3 nodes. Create a KS with RF=3.
{code}
[default@StressKeyspace] describe keyspace StressKeyspace;
Keyspace: StressKeyspace:
Replication Strategy: org.apache.cassandra.locator.SimpleStrategy
Durable Writes: true
Options: [replication_factor:3]
Column Families:
ColumnFamily: StressStandard
Key Validation Class: org.apache.cassandra.db.marshal.BytesType
Default column value validator: org.apache.cassandra.db.marshal.BytesType
Columns sorted by: org.apache.cassandra.db.marshal.UTF8Type
Row cache size / save period in seconds: 0.0/0
Key cache size / save period in seconds: 10000.0/14400
Memtable thresholds: 0.2859375/1440/32 (millions of ops/MB/minutes)
GC grace seconds: 864000
Compaction min/max thresholds: 4/32
Read repair chance: 1.0
Replicate on write: true
Built indexes: []
{code}
* Kill one node
* Insert 1000 rows with 10 columns each in batches of 100 into standard table.
* Bring up dead node. Run Repair.
* Kill two other nodes.
* Set: consistencylevel as ONE.
* Run: list StressStandard limit 2000;
* 691 Rows Returned.
> After repair, one row missing from query when two rows expected
> ---------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+1) Create Seed Data+*
> {code}
> create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run cassandra-cli+*
> {code}
> use testKS;
> create column family metadataCF
> with key_validation_class = 'AsciiType'
> and comparator = 'AsciiType'
> and column_metadata = [
> {column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
> {column_name: byte_col, validation_class: BytesType, index_type: KEYS},
> {column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
> {column_name: int_col, validation_class: IntegerType, index_type: KEYS},
> {column_name: long_col, validation_class: LongType, index_type: KEYS},
> {column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
> set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
> set metadataCF['key1']['byte_col']=bytes('10101010');
> set metadataCF['key1']['uuid_col']=timeuuid();
> set metadataCF['key1']['int_col']=integer(1000);
> set metadataCF['key1']['long_col']=long(44444444444);
> set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
> //Please note: forget to change CL before inserting 'key1', so that was inserted with CL=ONE by default.
> consistencylevel as TWO;
> set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
> set metadataCF['key2']['byte_col']=bytes('201010102');
> set metadataCF['key2']['uuid_col']=timeuuid();
> set metadataCF['key2']['int_col']=integer(2000);
> set metadataCF['key2']['long_col']=long(222222222);
> set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
> //Assumed that the following read would be done on CL=TWO and will the second replica would be guaranteed to be fix for 'key1'.
> list metadataCF;
> {code}
> {code}
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair testKS metadataCF
> {code}
> {code}
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
> INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
> INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
> INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
> INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
> INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
> INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
> INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
> INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
> INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,626 Stream context metadata [], 0 sstables.
> INFO 00:38:33,643 Streaming to /50.57.114.45
> INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
> INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
> INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
> INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
> INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
> INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
> INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
> INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
> INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
> INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
> INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
> INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
> INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
> INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on node3+*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+8) Bring up nodes 1 and 2 and query+*
> *NODE1 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy1
> Connected to: "Test Cluster" on cathy1/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *NODE2 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy2
> Connected to: "Test Cluster" on cathy2/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *NODE3 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+9) Force Read Repair on node1+*
> {code}
> consistencylevel as THREE;
> Consistency level is set to 'THREE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Resolved] (CASSANDRA-2918) After repair, missing rows from
query if you don't flush other replicas
Posted by "Jonathan Ellis (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Jonathan Ellis resolved CASSANDRA-2918.
---------------------------------------
Resolution: Duplicate
Assignee: (was: Sylvain Lebresne)
> After repair, missing rows from query if you don't flush other replicas
> -----------------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run java stress tool+*
> {code}
> ./bin/stress -o insert -n 1000 -c 10 -l 3 -e QUORUM -d 50.57.114.45,50.57.107.176
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair Keyspace1 Standard1
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on cathy3 - expect 1000 rows returned+*
> {code}
> [default@Keyspace1] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@Keyspace1] list Standard1 limit 2000;
> .....
> 323 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (CASSANDRA-2918) After repair, missing rows from
query if you don't flush other replicas
Posted by "Cathy Daw (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Cathy Daw updated CASSANDRA-2918:
---------------------------------
Summary: After repair, missing rows from query if you don't flush other replicas (was: After repair, one row missing from query when two rows expected)
> After repair, missing rows from query if you don't flush other replicas
> -----------------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run java stress tool+*
> {code}
> ./bin/stress -o insert -n 1000 -c 10 -l 3 -e QUORUM -d 50.57.114.45,50.57.107.176
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair Keyspace1 Standard1
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on cathy3 - expect 1000 rows returned+*
> {code}
> [default@Keyspace1] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@Keyspace1] list Standard1 limit 2000;
> .....
> 323 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (CASSANDRA-2918) After repair, one row missing
from query when two rows expected
Posted by "Cathy Daw (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Cathy Daw updated CASSANDRA-2918:
---------------------------------
Description:
*Cluster Config*
{code}
cathy1 - 50.57.114.45 - Token: 0
cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
{code}
*+1) Create Seed Data+*
{code}
create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
{code}
*+2) Kill cathy3: 50.57.114.39+*
{code}
root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
{code}
*+3) Run cassandra-cli+*
{code}
use testKS;
create column family metadataCF
with key_validation_class = 'AsciiType'
and comparator = 'AsciiType'
and column_metadata = [
{column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
{column_name: byte_col, validation_class: BytesType, index_type: KEYS},
{column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
{column_name: int_col, validation_class: IntegerType, index_type: KEYS},
{column_name: long_col, validation_class: LongType, index_type: KEYS},
{column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
set metadataCF['key1']['byte_col']=bytes('10101010');
set metadataCF['key1']['uuid_col']=timeuuid();
set metadataCF['key1']['int_col']=integer(1000);
set metadataCF['key1']['long_col']=long(44444444444);
set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
//Please note: forget to change CL before inserting 'key1', so that was inserted with CL=ONE by default.
consistencylevel as TWO;
set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
set metadataCF['key2']['byte_col']=bytes('201010102');
set metadataCF['key2']['uuid_col']=timeuuid();
set metadataCF['key2']['int_col']=integer(2000);
set metadataCF['key2']['long_col']=long(222222222);
set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
//Assumed that the following read would be done on CL=TWO and will the second replica would be guaranteed to be fix for 'key1'.
list metadataCF;
{code}
{code}
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
*+4) Start Cassandra on cathy3+*
*+5) Run repair on cathy3+*
{code}
nodetool -h cathy3 repair testKS metadataCF
{code}
{code}
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,626 Stream context metadata [], 0 sstables.
INFO 00:38:33,643 Streaming to /50.57.114.45
INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
{code}
*+6) Kill cathy1 and cathy2+*
{code}
root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
{code}
*+7) Log into cassandra-cli on node3+*
{code}
./cassandra-cli -h cathy3
Connected to: "Test Cluster" on cathy3/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] consistencylevel as ONE;
Consistency level is set to 'ONE'.
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*+8) Bring up nodes 1 and 2 and query+*
*NODE1 - cassandra-cli*
{code}
./cassandra-cli -h cathy1
Connected to: "Test Cluster" on cathy1/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*NODE2 - cassandra-cli*
{code}
./cassandra-cli -h cathy2
Connected to: "Test Cluster" on cathy2/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
*NODE3 - cassandra-cli*
{code}
./cassandra-cli -h cathy3
Connected to: "Test Cluster" on cathy3/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*+9) Force Read Repair on node1+*
{code}
consistencylevel as THREE;
Consistency level is set to 'THREE'.
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
was:
*Cluster Config*
{code}
cathy1 - 50.57.114.45 - Token: 0
cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
{code}
*+1) Create Seed Data+*
{code}
create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
{code}
*+2) Kill cathy3: 50.57.114.39+*
{code}
root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
{code}
*+3) Run cassandra-cli+*
{code}
use testKS;
create column family metadataCF
with key_validation_class = 'AsciiType'
and comparator = 'AsciiType'
and column_metadata = [
{column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
{column_name: byte_col, validation_class: BytesType, index_type: KEYS},
{column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
{column_name: int_col, validation_class: IntegerType, index_type: KEYS},
{column_name: long_col, validation_class: LongType, index_type: KEYS},
{column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
set metadataCF['key1']['byte_col']=bytes('10101010');
set metadataCF['key1']['uuid_col']=timeuuid();
set metadataCF['key1']['int_col']=integer(1000);
set metadataCF['key1']['long_col']=long(44444444444);
set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
consistencylevel as TWO;
set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
set metadataCF['key2']['byte_col']=bytes('201010102');
set metadataCF['key2']['uuid_col']=timeuuid();
set metadataCF['key2']['int_col']=integer(2000);
set metadataCF['key2']['long_col']=long(222222222);
set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
{code}
{code}
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
*+4) Start Cassandra on cathy3+*
*+5) Run repair on cathy3+*
{code}
nodetool -h cathy3 repair testKS metadataCF
{code}
{code}
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,626 Stream context metadata [], 0 sstables.
INFO 00:38:33,643 Streaming to /50.57.114.45
INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
{code}
*+6) Kill cathy1 and cathy2+*
{code}
root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
Address DC Rack Status State Load Owns Token
113427455640312821154458202477256070484
50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
{code}
*+7) Log into cassandra-cli on node3+*
{code}
./cassandra-cli -h cathy3
Connected to: "Test Cluster" on cathy3/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] consistencylevel as ONE;
Consistency level is set to 'ONE'.
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*+8) Bring up nodes 1 and 2 and query+*
*NODE1 - cassandra-cli*
{code}
./cassandra-cli -h cathy1
Connected to: "Test Cluster" on cathy1/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*NODE2 - cassandra-cli*
{code}
./cassandra-cli -h cathy2
Connected to: "Test Cluster" on cathy2/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
*NODE3 - cassandra-cli*
{code}
./cassandra-cli -h cathy3
Connected to: "Test Cluster" on cathy3/9160
Welcome to the Cassandra CLI.
Type 'help;' or '?' for help.
Type 'quit;' or 'exit;' to quit.
[default@unknown] use testKS;
Authenticated to keyspace: testKS
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
1 Row Returned.
{code}
*+9) Force Read Repair on node1+*
{code}
consistencylevel as THREE;
Consistency level is set to 'THREE'.
[default@testKS] list metadataCF;
Using default limit of 100
-------------------
RowKey: key1
=> (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
=> (column=byte_col, value=10101010, timestamp=1311035260662000)
=> (column=int_col, value=1000, timestamp=1311035260669000)
=> (column=long_col, value=44444444444, timestamp=1311035260674000)
=> (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
=> (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
-------------------
RowKey: key2
=> (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
=> (column=byte_col, value=0201010102, timestamp=1311035260685000)
=> (column=int_col, value=2000, timestamp=1311035260692000)
=> (column=long_col, value=222222222, timestamp=1311035260695000)
=> (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
=> (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
2 Rows Returned.
{code}
> After repair, one row missing from query when two rows expected
> ---------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+1) Create Seed Data+*
> {code}
> create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run cassandra-cli+*
> {code}
> use testKS;
> create column family metadataCF
> with key_validation_class = 'AsciiType'
> and comparator = 'AsciiType'
> and column_metadata = [
> {column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
> {column_name: byte_col, validation_class: BytesType, index_type: KEYS},
> {column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
> {column_name: int_col, validation_class: IntegerType, index_type: KEYS},
> {column_name: long_col, validation_class: LongType, index_type: KEYS},
> {column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
> set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
> set metadataCF['key1']['byte_col']=bytes('10101010');
> set metadataCF['key1']['uuid_col']=timeuuid();
> set metadataCF['key1']['int_col']=integer(1000);
> set metadataCF['key1']['long_col']=long(44444444444);
> set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
> //Please note: forget to change CL before inserting 'key1', so that was inserted with CL=ONE by default.
> consistencylevel as TWO;
> set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
> set metadataCF['key2']['byte_col']=bytes('201010102');
> set metadataCF['key2']['uuid_col']=timeuuid();
> set metadataCF['key2']['int_col']=integer(2000);
> set metadataCF['key2']['long_col']=long(222222222);
> set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
> //Assumed that the following read would be done on CL=TWO and will the second replica would be guaranteed to be fix for 'key1'.
> list metadataCF;
> {code}
> {code}
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair testKS metadataCF
> {code}
> {code}
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
> INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
> INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
> INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
> INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
> INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
> INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
> INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
> INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
> INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,626 Stream context metadata [], 0 sstables.
> INFO 00:38:33,643 Streaming to /50.57.114.45
> INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
> INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
> INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
> INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
> INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
> INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
> INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
> INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
> INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
> INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
> INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
> INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
> INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
> INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on node3+*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+8) Bring up nodes 1 and 2 and query+*
> *NODE1 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy1
> Connected to: "Test Cluster" on cathy1/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *NODE2 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy2
> Connected to: "Test Cluster" on cathy2/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *NODE3 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+9) Force Read Repair on node1+*
> {code}
> consistencylevel as THREE;
> Consistency level is set to 'THREE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Updated] (CASSANDRA-2918) After repair, one row missing
from query when two rows expected
Posted by "Cathy Daw (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Cathy Daw updated CASSANDRA-2918:
---------------------------------
Comment: was deleted
(was: This looks more to me like a WRITE with CL.Q bug as the source of the problem. But shouldn't have repair detected that node1 and node2 were out of sync?
* 3 node cluster / KS with RF=3 / 2 nodes up
* Insert 1000 rows on node1 with CL=QUORUM
** node1:691 rows
** node2:1000 rows
** node3: down
* Repair node3
** node1:691 rows
** node2:1000 rows
** node3: 691 rows
)
> After repair, one row missing from query when two rows expected
> ---------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run java stress tool+*
> {code}
> ./bin/stress -o insert -n 1000 -c 10 -l 3 -e QUORUM -d 50.57.114.45,50.57.107.176
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair Keyspace1 Standard1
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on cathy3 - expect 1000 rows returned+*
> {code}
> [default@Keyspace1] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@Keyspace1] list Standard1 limit 2000;
> .....
> 323 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-2918) After repair, missing rows from
query if you don't flush other replicas
Posted by "Peter Schuller (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13068327#comment-13068327 ]
Peter Schuller commented on CASSANDRA-2918:
-------------------------------------------
@jbellis Note that if repair didn't flush, it would also be broken for another reason: You could no longer rely on the start time of a successfully completed repair session when comparing against gc grace seconds.
> After repair, missing rows from query if you don't flush other replicas
> -----------------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run java stress tool+*
> {code}
> ./bin/stress -o insert -n 1000 -c 10 -l 3 -e QUORUM -d 50.57.114.45,50.57.107.176
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair Keyspace1 Standard1
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on cathy3 - expect 1000 rows returned+*
> {code}
> [default@Keyspace1] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@Keyspace1] list Standard1 limit 2000;
> .....
> 323 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-2918) After repair, one row missing
from query when two rows expected
Posted by "Cathy Daw (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13067882#comment-13067882 ]
Cathy Daw commented on CASSANDRA-2918:
--------------------------------------
Interestingly, I did the same test case using the hector stress tool which writes @ CL=QUORUM.
*Insert 1000 rows. 691 Rows Returned.*
* Bring up 3 nodes. Create a KS with RF=3.
{code}
[default@StressKeyspace] describe keyspace StressKeyspace;
Keyspace: StressKeyspace:
Replication Strategy: org.apache.cassandra.locator.SimpleStrategy
Durable Writes: true
Options: [replication_factor:3]
Column Families:
ColumnFamily: StressStandard
Key Validation Class: org.apache.cassandra.db.marshal.BytesType
Default column value validator: org.apache.cassandra.db.marshal.BytesType
Columns sorted by: org.apache.cassandra.db.marshal.UTF8Type
Row cache size / save period in seconds: 0.0/0
Key cache size / save period in seconds: 10000.0/14400
Memtable thresholds: 0.2859375/1440/32 (millions of ops/MB/minutes)
GC grace seconds: 864000
Compaction min/max thresholds: 4/32
Read repair chance: 1.0
Replicate on write: true
Built indexes: []
{code}
* Kill one node
* Insert 1000 rows with 10 columns each in batches of 100 into standard table.
* Bring up dead node. Run Repair.
* Kill two other nodes.
* Set: consistencylevel as ONE.
* Run: list StressStandard limit 2000;
* 691 Rows Returned.
> After repair, one row missing from query when two rows expected
> ---------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+1) Create Seed Data+*
> {code}
> create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run cassandra-cli+*
> {code}
> use testKS;
> create column family metadataCF
> with key_validation_class = 'AsciiType'
> and comparator = 'AsciiType'
> and column_metadata = [
> {column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
> {column_name: byte_col, validation_class: BytesType, index_type: KEYS},
> {column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
> {column_name: int_col, validation_class: IntegerType, index_type: KEYS},
> {column_name: long_col, validation_class: LongType, index_type: KEYS},
> {column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
> set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
> set metadataCF['key1']['byte_col']=bytes('10101010');
> set metadataCF['key1']['uuid_col']=timeuuid();
> set metadataCF['key1']['int_col']=integer(1000);
> set metadataCF['key1']['long_col']=long(44444444444);
> set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
> //Please note: forget to change CL before inserting 'key1', so that was inserted with CL=ONE by default.
> consistencylevel as TWO;
> set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
> set metadataCF['key2']['byte_col']=bytes('201010102');
> set metadataCF['key2']['uuid_col']=timeuuid();
> set metadataCF['key2']['int_col']=integer(2000);
> set metadataCF['key2']['long_col']=long(222222222);
> set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
> //Assumed that the following read would be done on CL=TWO and will the second replica would be guaranteed to be fix for 'key1'.
> list metadataCF;
> {code}
> {code}
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair testKS metadataCF
> {code}
> {code}
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
> INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
> INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
> INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
> INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
> INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
> INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
> INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
> INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
> INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,626 Stream context metadata [], 0 sstables.
> INFO 00:38:33,643 Streaming to /50.57.114.45
> INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
> INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
> INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
> INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
> INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
> INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
> INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
> INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
> INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
> INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
> INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
> INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
> INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
> INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on node3+*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+8) Bring up nodes 1 and 2 and query+*
> *NODE1 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy1
> Connected to: "Test Cluster" on cathy1/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *NODE2 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy2
> Connected to: "Test Cluster" on cathy2/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *NODE3 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+9) Force Read Repair on node1+*
> {code}
> consistencylevel as THREE;
> Consistency level is set to 'THREE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-2918) After repair, missing rows from
query if you don't flush other replicas
Posted by "Jonathan Ellis (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13068125#comment-13068125 ]
Jonathan Ellis commented on CASSANDRA-2918:
-------------------------------------------
So the tl;dr is, repair doesn't flush before computing the trees.
I think the reasoning is, on any reasonably busy system a single memtable's worth of out-of-syncness is fine to ignore, until next repair when it will presumably have been flushed.
But it seems to me this could actually contribute to the kinds of problem we have in CASSANDRA-2816: if a memtable's updates are randomly distributed throughout the token range, it could pollute a relatively high number of hash blocks.
Maybe we should revisit that decision?
> After repair, missing rows from query if you don't flush other replicas
> -----------------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run java stress tool+*
> {code}
> ./bin/stress -o insert -n 1000 -c 10 -l 3 -e QUORUM -d 50.57.114.45,50.57.107.176
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair Keyspace1 Standard1
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on cathy3 - expect 1000 rows returned+*
> {code}
> [default@Keyspace1] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@Keyspace1] list Standard1 limit 2000;
> .....
> 323 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-2918) After repair, missing rows from
query if you don't flush other replicas
Posted by "Sylvain Lebresne (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13068188#comment-13068188 ]
Sylvain Lebresne commented on CASSANDRA-2918:
---------------------------------------------
We do flush before computing the trees:
{noformat}
private void doValidationCompaction(ColumnFamilyStore cfs, AntiEntropyService.Validator validator) throws IOException
{
// flush first so everyone is validating data that is as similar as possible
try
{
StorageService.instance.forceTableFlush(cfs.table.name, cfs.getColumnFamilyName());
}
{noformat}
However this is done badly, or more precisely, the problem here is due to CASSANDRA-2811. More precisely the part about:
{quote}
It turns out to also have a more subtle problem for repair itself. If two validation compaction for the same column family (but different range) are started in a very short time interval, the first validation will block on the flush, but the second one may not block at all if the memtable is clean when it request it's own flush. In which case that second validation will be executed on data older than it should.
{quote}
And because RF=3, we do start 3 validations for the same CF but different ranges at the same time. Which mean that 2 of them don't correctly wait the end of the flush and validate old data. This is fixed by CASSANDRA-2816 (I'll fix the tests asap so that we can commit it for 0.8.2).
> After repair, missing rows from query if you don't flush other replicas
> -----------------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run java stress tool+*
> {code}
> ./bin/stress -o insert -n 1000 -c 10 -l 3 -e QUORUM -d 50.57.114.45,50.57.107.176
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair Keyspace1 Standard1
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on cathy3 - expect 1000 rows returned+*
> {code}
> [default@Keyspace1] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@Keyspace1] list Standard1 limit 2000;
> .....
> 323 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira
[jira] [Commented] (CASSANDRA-2918) After repair, one row missing
from query when two rows expected
Posted by "Cathy Daw (JIRA)" <ji...@apache.org>.
[ https://issues.apache.org/jira/browse/CASSANDRA-2918?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13067896#comment-13067896 ]
Cathy Daw commented on CASSANDRA-2918:
--------------------------------------
This looks more to me like a WRITE with CL.Q bug as the source of the problem. But shouldn't have repair detected that node1 and node2 were out of sync?
* 3 node cluster / KS with RF=3 / 2 nodes up
* Insert 1000 rows on node2 with CL=QUORUM
** node1:691 rows
** node2:1000 rows
** node3: down
* Repair node3
** node1:691 rows
** node2:1000 rows
** node3: 691 rows
> After repair, one row missing from query when two rows expected
> ---------------------------------------------------------------
>
> Key: CASSANDRA-2918
> URL: https://issues.apache.org/jira/browse/CASSANDRA-2918
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Affects Versions: 0.8.2
> Environment: Cassandra-0.8 branch @ 07/18 around 1pm PST.
> Reporter: Cathy Daw
> Assignee: Sylvain Lebresne
>
> *Cluster Config*
> {code}
> cathy1 - 50.57.114.45 - Token: 0
> cathy2 - 50.57.107.176 - Token: 56713727820156410577229101238628035242
> cathy3 - 50.57.114.39 - Token: 113427455640312821154458202477256070484
> {code}
> *+1) Create Seed Data+*
> {code}
> create keyspace testKS with placement_strategy = 'SimpleStrategy' and strategy_options = [{replication_factor : 3}];
> {code}
> *+2) Kill cathy3: 50.57.114.39+*
> {code}
> root@cathy2:~/cass-0.8/bin# ./nodetool -h localhost ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Up Normal 59.84 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Up Normal 59.85 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Down Normal 59.85 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+3) Run cassandra-cli+*
> {code}
> use testKS;
> create column family metadataCF
> with key_validation_class = 'AsciiType'
> and comparator = 'AsciiType'
> and column_metadata = [
> {column_name: ascii_col, validation_class:AsciiType, index_type: KEYS},
> {column_name: byte_col, validation_class: BytesType, index_type: KEYS},
> {column_name: uuid_col, validation_class: LexicalUUIDType, index_type: KEYS},
> {column_name: int_col, validation_class: IntegerType, index_type: KEYS},
> {column_name: long_col, validation_class: LongType, index_type: KEYS},
> {column_name: utf8_col, validation_class: UTF8Type, index_type: KEYS}];
> set metadataCF['key1']['ascii_col']=ascii('this is data inserted into ascii column');
> set metadataCF['key1']['byte_col']=bytes('10101010');
> set metadataCF['key1']['uuid_col']=timeuuid();
> set metadataCF['key1']['int_col']=integer(1000);
> set metadataCF['key1']['long_col']=long(44444444444);
> set metadataCF['key1']['utf8_col']=utf8('this is data inserted into UTF8 column');
> //Please note: forget to change CL before inserting 'key1', so that was inserted with CL=ONE by default.
> consistencylevel as TWO;
> set metadataCF['key2']['ascii_col']=ascii('key2: this is data inserted into ascii column');
> set metadataCF['key2']['byte_col']=bytes('201010102');
> set metadataCF['key2']['uuid_col']=timeuuid();
> set metadataCF['key2']['int_col']=integer(2000);
> set metadataCF['key2']['long_col']=long(222222222);
> set metadataCF['key2']['utf8_col']=utf8('key2-this is data inserted into UTF8 column');
> //Assumed that the following read would be done on CL=TWO and will the second replica would be guaranteed to be fix for 'key1'.
> list metadataCF;
> {code}
> {code}
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *+4) Start Cassandra on cathy3+*
> *+5) Run repair on cathy3+*
> {code}
> nodetool -h cathy3 repair testKS metadataCF
> {code}
> {code}
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>, #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>]
> INFO 00:38:32,696 Waiting for repair requests: [#<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>, #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>]
> INFO 00:38:32,697 Waiting for repair requests: [#<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>, #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>]
> INFO 00:38:33,052 Sending AEService tree for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.39, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,100 Sending AEService tree for #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.39, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>
> INFO 00:38:33,203 Sending AEService tree for #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.39, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>
> INFO 00:38:33,242 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,270 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:33,363 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,364 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.107.176, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 1 outstanding.
> INFO 00:38:33,428 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:33,429 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.114.45, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 1 outstanding.
> INFO 00:38:33,441 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,453 Queueing comparison #<Differencer #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]>>
> INFO 00:38:33,519 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,519 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.107.176, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 1 outstanding.
> INFO 00:38:33,554 Endpoints /50.57.114.39 and /50.57.114.45 are consistent for (testKS,metadataCF) on (56713727820156410577229101238628035242,113427455640312821154458202477256070484]
> INFO 00:38:33,555 #<TreeRequest manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc, /50.57.114.45, (testKS,metadataCF), (56713727820156410577229101238628035242,113427455640312821154458202477256070484]> completed successfully: 0 outstanding.
> INFO 00:38:33,555 Repair session manual-repair-20f203a7-53b1-4eea-a9bd-4cbaa7b434dc completed successfully.
> INFO 00:38:33,567 Queueing comparison #<Differencer #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>>
> INFO 00:38:33,598 Endpoints /50.57.114.39 and /50.57.114.45 have 1 range(s) out of sync for (testKS,metadataCF) on (113427455640312821154458202477256070484,0]
> INFO 00:38:33,599 Performing streaming repair of 1 ranges for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,626 Stream context metadata [], 0 sstables.
> INFO 00:38:33,643 Streaming to /50.57.114.45
> INFO 00:38:33,732 Opening /var/lib/cassandra/data/testKS/metadataCF-g-1
> INFO 00:38:33,736 Submitting index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, for data in SSTableReader(path='/var/lib/cassandra/data/testKS/metadataCF-g-1-Data.db')
> INFO 00:38:33,741 Enqueuing flush of Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,742 Writing Memtable-metadataCF.ascii_col_idx@1146840108(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,748 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.ascii_col_idx-g-1-Data.db (114 bytes)
> INFO 00:38:33,750 Enqueuing flush of Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,750 Writing Memtable-metadataCF.byte_col_idx@1562533825(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,756 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.byte_col_idx-g-1-Data.db (74 bytes)
> INFO 00:38:33,758 Enqueuing flush of Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,758 Writing Memtable-metadataCF.int_col_idx@1515686296(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,820 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.int_col_idx-g-1-Data.db (71 bytes)
> INFO 00:38:33,821 Enqueuing flush of Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,821 Writing Memtable-metadataCF.long_col_idx@507274563(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,828 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.long_col_idx-g-1-Data.db (77 bytes)
> INFO 00:38:33,829 Enqueuing flush of Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,830 Writing Memtable-metadataCF.utf8_col_idx@1201340161(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,836 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.utf8_col_idx-g-1-Data.db (112 bytes)
> INFO 00:38:33,837 Enqueuing flush of Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,838 Writing Memtable-metadataCF.uuid_col_idx@703489457(19/23 serialized/live bytes, 1 ops)
> INFO 00:38:33,857 Completed flushing /var/lib/cassandra/data/testKS/metadataCF.uuid_col_idx-g-1-Data.db (85 bytes)
> INFO 00:38:33,858 Index build of ascii_col,byte_col,int_col,long_col,utf8_col,uuid_col, complete
> INFO 00:38:33,858 Finished streaming session 4937677751163 from /50.57.114.45
> INFO 00:38:33,859 Finished streaming repair for #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]>
> INFO 00:38:33,859 #<TreeRequest manual-repair-9a957823-9969-4b91-900a-96a00659a85e, /50.57.114.45, (testKS,metadataCF), (113427455640312821154458202477256070484,0]> completed successfully: 0 outstanding.
> INFO 00:38:33,859 Repair session manual-repair-9a957823-9969-4b91-900a-96a00659a85e completed successfully.
> INFO 00:38:36,026 Queueing comparison #<Differencer #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]>>
> INFO 00:38:36,057 Endpoints /50.57.114.39 and /50.57.107.176 are consistent for (testKS,metadataCF) on (0,56713727820156410577229101238628035242]
> INFO 00:38:36,058 #<TreeRequest manual-repair-9b989c5e-66a1-459d-93ed-793acc401794, /50.57.107.176, (testKS,metadataCF), (0,56713727820156410577229101238628035242]> completed successfully: 0 outstanding.
> INFO 00:38:36,058 Repair session manual-repair-9b989c5e-66a1-459d-93ed-793acc401794 completed successfully.
> {code}
> *+6) Kill cathy1 and cathy2+*
> {code}
> root@cathy3:~/cass-0.8/bin# ./nodetool -h cathy3 ring
> Address DC Rack Status State Load Owns Token
> 113427455640312821154458202477256070484
> 50.57.114.45 datacenter1 rack1 Down Normal 105.46 KB 33.33% 0
> 50.57.107.176 datacenter1 rack1 Down Normal 106 KB 33.33% 56713727820156410577229101238628035242
> 50.57.114.39 datacenter1 rack1 Up Normal 331.33 KB 33.33% 113427455640312821154458202477256070484
> {code}
> *+7) Log into cassandra-cli on node3+*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] consistencylevel as ONE;
> Consistency level is set to 'ONE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+8) Bring up nodes 1 and 2 and query+*
> *NODE1 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy1
> Connected to: "Test Cluster" on cathy1/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *NODE2 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy2
> Connected to: "Test Cluster" on cathy2/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
> *NODE3 - cassandra-cli*
> {code}
> ./cassandra-cli -h cathy3
> Connected to: "Test Cluster" on cathy3/9160
> Welcome to the Cassandra CLI.
> Type 'help;' or '?' for help.
> Type 'quit;' or 'exit;' to quit.
> [default@unknown] use testKS;
> Authenticated to keyspace: testKS
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 1 Row Returned.
> {code}
> *+9) Force Read Repair on node1+*
> {code}
> consistencylevel as THREE;
> Consistency level is set to 'THREE'.
> [default@testKS] list metadataCF;
> Using default limit of 100
> -------------------
> RowKey: key1
> => (column=ascii_col, value=this is data inserted into ascii column, timestamp=1311035260656000)
> => (column=byte_col, value=10101010, timestamp=1311035260662000)
> => (column=int_col, value=1000, timestamp=1311035260669000)
> => (column=long_col, value=44444444444, timestamp=1311035260674000)
> => (column=utf8_col, value=this is data inserted into UTF8 column, timestamp=1311035260678000)
> => (column=uuid_col, value=e9811e90-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260666000)
> -------------------
> RowKey: key2
> => (column=ascii_col, value=key2: this is data inserted into ascii column, timestamp=1311035260682000)
> => (column=byte_col, value=0201010102, timestamp=1311035260685000)
> => (column=int_col, value=2000, timestamp=1311035260692000)
> => (column=long_col, value=222222222, timestamp=1311035260695000)
> => (column=utf8_col, value=key2-this is data inserted into UTF8 column, timestamp=1311035260699000)
> => (column=uuid_col, value=e984a100-b19d-11e0-0000-2069cd105fbf, timestamp=1311035260689000)
> 2 Rows Returned.
> {code}
--
This message is automatically generated by JIRA.
For more information on JIRA, see: http://www.atlassian.com/software/jira