You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2014/03/14 21:40:17 UTC
[01/12] git commit: Fix failing pig test
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.0 7ce3950a9 -> 57c5969e1
refs/heads/cassandra-2.1 3595131df -> e0718bcdf
refs/heads/trunk c54c618b3 -> 5011437b1
Fix failing pig test
Patch by Alex Liu, reviewed by brandonwilliams for CASSANDRA-6376
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8c52cc0e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8c52cc0e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8c52cc0e
Branch: refs/heads/cassandra-2.1
Commit: 8c52cc0e7f67378d66093c958558e765b682888f
Parents: 972d3da
Author: Brandon Williams <br...@apache.org>
Authored: Fri Mar 14 14:46:53 2014 -0500
Committer: Brandon Williams <br...@apache.org>
Committed: Fri Mar 14 14:46:53 2014 -0500
----------------------------------------------------------------------
.../unit/org/apache/cassandra/pig/CqlTableTest.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8c52cc0e/test/unit/org/apache/cassandra/pig/CqlTableTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/pig/CqlTableTest.java b/test/unit/org/apache/cassandra/pig/CqlTableTest.java
index 785d819..55586c7 100644
--- a/test/unit/org/apache/cassandra/pig/CqlTableTest.java
+++ b/test/unit/org/apache/cassandra/pig/CqlTableTest.java
@@ -174,18 +174,18 @@ public class CqlTableTest extends PigTestBase
pig.registerQuery("STORE recs INTO 'cql://cql3ks/collectiontable?" + defaultParameters + "&output_query=update+cql3ks.collectiontable+set+n+%3D+%3F' USING CqlStorage();");
pig.executeBatch();
- //(book2,((m,mm),(n,nn)))
- //(book3,((m,mm),(n,nn)))
- //(book4,((m,mm),(n,nn)))
- //(book1,((m,mm),(n,nn)))
+ //(book2,((key2, value2),(m,mm),(n,nn)))
+ //(book3,((key3, value3),(m,mm),(n,nn)))
+ //(book4,((key4, value4),(m,mm),(n,nn)))
+ //(book1,((key1, value1),(m,mm),(n,nn)))
pig.registerQuery("result= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + "' USING CqlStorage();");
Iterator<Tuple> it = pig.openIterator("result");
- if (it.hasNext()) {
+ while (it.hasNext()) {
Tuple t = it.next();
Tuple t1 = (Tuple) t.get(1);
- Assert.assertEquals(t1.size(), 2);
- Tuple element1 = (Tuple) t1.get(0);
- Tuple element2 = (Tuple) t1.get(1);
+ Assert.assertEquals(t1.size(), 3);
+ Tuple element1 = (Tuple) t1.get(1);
+ Tuple element2 = (Tuple) t1.get(2);
Assert.assertEquals(element1.get(0), "m");
Assert.assertEquals(element1.get(1), "mm");
Assert.assertEquals(element2.get(0), "n");
[06/12] git commit: Merge remote-tracking branch
'origin/cassandra-2.0' into cassandra-2.0
Posted by jb...@apache.org.
Merge remote-tracking branch 'origin/cassandra-2.0' into cassandra-2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7ce3950a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7ce3950a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7ce3950a
Branch: refs/heads/cassandra-2.1
Commit: 7ce3950a97a576b00c9936380cc3c5dc750ef4e7
Parents: 3050134 f122f99
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Mar 14 15:28:10 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Mar 14 15:28:10 2014 -0500
----------------------------------------------------------------------
test/pig/org/apache/cassandra/pig/CqlTableTest.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[07/12] git commit: Log more information when exceeding
tombstone_warn_threshold patch by Oleg Anastasyev;
reviewed by jbellis for CASSANDRA-6865
Posted by jb...@apache.org.
Log more information when exceeding tombstone_warn_threshold
patch by Oleg Anastasyev; reviewed by jbellis for CASSANDRA-6865
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/57c5969e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/57c5969e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/57c5969e
Branch: refs/heads/cassandra-2.1
Commit: 57c5969e1dac82c6a676668edf1dc5ba64bcbf1f
Parents: 7ce3950
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Mar 14 15:39:19 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Mar 14 15:39:19 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/filter/SliceQueryFilter.java | 20 ++++++++++++++++++--
2 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/57c5969e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 045e4f8..61b1ee1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.7
+ * Log more information when exceeding tombstone_warn_threshold (CASSANDRA-6865)
* Fix truncate to not abort due to unreachable fat clients (CASSANDRA-6864)
* Fix schema concurrency exceptions (CASSANDRA-6841)
* Fix leaking validator FH in StreamWriter (CASSANDRA-6832)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/57c5969e/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index b43308f..356418e 100644
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@ -207,8 +207,24 @@ public class SliceQueryFilter implements IDiskAtomFilter
Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
if (respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneWarnThreshold())
- logger.warn("Read {} live and {} tombstoned cells in {}.{} (see tombstone_warn_threshold)",
- columnCounter.live(), columnCounter.ignored(), container.metadata().ksName, container.metadata().cfName);
+ {
+ StringBuilder sb = new StringBuilder();
+ AbstractType<?> type = container.metadata().comparator;
+ for (ColumnSlice sl : slices)
+ {
+ if (sl == null)
+ continue;
+
+ sb.append('[');
+ sb.append(type.getString(sl.start));
+ sb.append('-');
+ sb.append(type.getString(sl.finish));
+ sb.append(']');
+ }
+
+ logger.warn("Read {} live and {} tombstoned cells in {}.{} (see tombstone_warn_threshold). {} columns was requested, slices={}, delInfo={}",
+ columnCounter.live(), columnCounter.ignored(), container.metadata().ksName, container.metadata().cfName, count, sb, container.deletionInfo());
+ }
}
protected boolean respectTombstoneThresholds()
[04/12] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0
Posted by jb...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f122f999
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f122f999
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f122f999
Branch: refs/heads/trunk
Commit: f122f9994e92e4a1d7de32ff9f6bc116a4a65d2a
Parents: 22b9453 8c52cc0
Author: Brandon Williams <br...@apache.org>
Authored: Fri Mar 14 14:47:31 2014 -0500
Committer: Brandon Williams <br...@apache.org>
Committed: Fri Mar 14 14:47:31 2014 -0500
----------------------------------------------------------------------
test/pig/org/apache/cassandra/pig/CqlTableTest.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f122f999/test/pig/org/apache/cassandra/pig/CqlTableTest.java
----------------------------------------------------------------------
diff --cc test/pig/org/apache/cassandra/pig/CqlTableTest.java
index e33364f,0000000..3bbc3d1
mode 100644,000000..100644
--- a/test/pig/org/apache/cassandra/pig/CqlTableTest.java
+++ b/test/pig/org/apache/cassandra/pig/CqlTableTest.java
@@@ -1,257 -1,0 +1,257 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.pig;
+
+import java.io.IOException;
+import java.nio.charset.CharacterCodingException;
+import java.util.Iterator;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.thrift.AuthenticationException;
+import org.apache.cassandra.thrift.AuthorizationException;
+import org.apache.cassandra.thrift.InvalidRequestException;
+import org.apache.cassandra.thrift.NotFoundException;
+import org.apache.cassandra.thrift.SchemaDisagreementException;
+import org.apache.cassandra.thrift.TimedOutException;
+import org.apache.cassandra.thrift.UnavailableException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class CqlTableTest extends PigTestBase
+{
+ private static String[] statements = {
+ "CREATE KEYSPACE cql3ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1}",
+ "USE cql3ks;",
+
+ "CREATE TABLE cqltable (key1 text, key2 int, column1 int, column2 float, primary key(key1, key2))",
+ "INSERT INTO cqltable (key1, key2, column1, column2) values ('key1', 111, 100, 10.1)",
+ "CREATE TABLE compactcqltable (key1 text, column1 int, column2 float, primary key(key1)) WITH COMPACT STORAGE",
+ "INSERT INTO compactcqltable (key1, column1, column2) values ('key1', 100, 10.1)",
+
+ "CREATE TABLE test (a int PRIMARY KEY, b int);",
+ "CREATE INDEX test_b on test (b);",
+
+ "CREATE TABLE moredata (x int PRIMARY KEY, y int);",
+ "INSERT INTO test (a,b) VALUES (1,1);",
+ "INSERT INTO test (a,b) VALUES (2,2);",
+ "INSERT INTO test (a,b) VALUES (3,3);",
+ "INSERT INTO moredata (x, y) VALUES (4,4);",
+ "INSERT INTO moredata (x, y) VALUES (5,5);",
+ "INSERT INTO moredata (x, y) VALUES (6,6);",
+
+ "CREATE TABLE compotable (a int, b int, c text, d text, PRIMARY KEY (a,b,c));",
+ "INSERT INTO compotable (a, b , c , d ) VALUES ( 1,1,'One','match');",
+ "INSERT INTO compotable (a, b , c , d ) VALUES ( 2,2,'Two','match');",
+ "INSERT INTO compotable (a, b , c , d ) VALUES ( 3,3,'Three','match');",
+ "INSERT INTO compotable (a, b , c , d ) VALUES ( 4,4,'Four','match');",
+
+ "create table compmore (id int PRIMARY KEY, x int, y int, z text, data text);",
+ "INSERT INTO compmore (id, x, y, z,data) VALUES (1,5,6,'Fix','nomatch');",
+ "INSERT INTO compmore (id, x, y, z,data) VALUES (2,6,5,'Sive','nomatch');",
+ "INSERT INTO compmore (id, x, y, z,data) VALUES (3,7,7,'Seven','match');",
+ "INSERT INTO compmore (id, x, y, z,data) VALUES (4,8,8,'Eight','match');",
+ "INSERT INTO compmore (id, x, y, z,data) VALUES (5,9,10,'Ninen','nomatch');",
+
+ "CREATE TABLE collectiontable(m text PRIMARY KEY, n map<text, text>);",
+ "UPDATE collectiontable SET n['key1'] = 'value1' WHERE m = 'book1';",
+ "UPDATE collectiontable SET n['key2'] = 'value2' WHERE m = 'book2';",
+ "UPDATE collectiontable SET n['key3'] = 'value3' WHERE m = 'book3';",
+ "UPDATE collectiontable SET n['key4'] = 'value4' WHERE m = 'book4';",
+ };
+
+ @BeforeClass
+ public static void setup() throws TTransportException, IOException, InterruptedException, ConfigurationException,
+ AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, CharacterCodingException, ClassNotFoundException, NoSuchFieldException, IllegalAccessException, InstantiationException
+ {
+ startCassandra();
+ setupDataByCql(statements);
+ startHadoopCluster();
+ }
+
+ @Test
+ public void testCqlStorageSchema()
+ throws AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, SchemaDisagreementException, IOException
+ {
+ pig.registerQuery("rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + "' USING CqlStorage();");
+ Iterator<Tuple> it = pig.openIterator("rows");
+ if (it.hasNext()) {
+ Tuple t = it.next();
+ Assert.assertEquals(t.get(0).toString(), "key1");
+ Assert.assertEquals(t.get(1), 111);
+ Assert.assertEquals(t.get(2), 100);
+ Assert.assertEquals(t.get(3), 10.1f);
+ Assert.assertEquals(4, t.size());
+ }
+
+ pig.registerQuery("rows = LOAD 'cql://cql3ks/compactcqltable?" + defaultParameters + "' USING CqlStorage();");
+ it = pig.openIterator("rows");
+ if (it.hasNext()) {
+ Tuple t = it.next();
+ Assert.assertEquals(t.get(0).toString(), "key1");
+ Assert.assertEquals(t.get(1), 100);
+ Assert.assertEquals(t.get(2), 10.1f);
+ Assert.assertEquals(3, t.size());
+ }
+ }
+
+ @Test
+ public void testCqlStorageSingleKeyTable()
+ throws AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, SchemaDisagreementException, IOException
+ {
+ pig.setBatchOn();
+ pig.registerQuery("moretestvalues= LOAD 'cql://cql3ks/moredata?" + defaultParameters + "' USING CqlStorage();");
+ pig.registerQuery("insertformat= FOREACH moretestvalues GENERATE TOTUPLE(TOTUPLE('a',x)),TOTUPLE(y);");
+ pig.registerQuery("STORE insertformat INTO 'cql://cql3ks/test?" + defaultParameters + "&output_query=UPDATE+cql3ks.test+set+b+%3D+%3F' USING CqlStorage();");
+ pig.executeBatch();
+ //(5,5)
+ //(6,6)
+ //(4,4)
+ //(2,2)
+ //(3,3)
+ //(1,1)
+ pig.registerQuery("result= LOAD 'cql://cql3ks/test?" + defaultParameters + "' USING CqlStorage();");
+ Iterator<Tuple> it = pig.openIterator("result");
+ if (it.hasNext()) {
+ Tuple t = it.next();
+ Assert.assertEquals(t.get(0), t.get(1));
+ }
+ }
+
+ @Test
+ public void testCqlStorageCompositeKeyTable()
+ throws AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, SchemaDisagreementException, IOException
+ {
+ pig.setBatchOn();
+ pig.registerQuery("moredata= LOAD 'cql://cql3ks/compmore?" + defaultParameters + "' USING CqlStorage();");
+ pig.registerQuery("insertformat = FOREACH moredata GENERATE TOTUPLE (TOTUPLE('a',x),TOTUPLE('b',y), TOTUPLE('c',z)),TOTUPLE(data);");
+ pig.registerQuery("STORE insertformat INTO 'cql://cql3ks/compotable?" + defaultParameters + "&output_query=UPDATE%20cql3ks.compotable%20SET%20d%20%3D%20%3F' USING CqlStorage();");
+ pig.executeBatch();
+
+ //(5,6,Fix,nomatch)
+ //(3,3,Three,match)
+ //(1,1,One,match)
+ //(2,2,Two,match)
+ //(7,7,Seven,match)
+ //(8,8,Eight,match)
+ //(6,5,Sive,nomatch)
+ //(4,4,Four,match)
+ //(9,10,Ninen,nomatch)
+ pig.registerQuery("result= LOAD 'cql://cql3ks/compotable?" + defaultParameters + "' USING CqlStorage();");
+ Iterator<Tuple> it = pig.openIterator("result");
+ int count = 0;
+ while (it.hasNext()) {
+ it.next();
+ count ++;
+ }
+ Assert.assertEquals(count, 9);
+ }
+
+ @Test
+ public void testCqlStorageCollectionColumnTable()
+ throws AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, SchemaDisagreementException, IOException
+ {
+ pig.setBatchOn();
+ pig.registerQuery("collectiontable= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + "' USING CqlStorage();");
+ pig.registerQuery("recs= FOREACH collectiontable GENERATE TOTUPLE(TOTUPLE('m', m) ), TOTUPLE(TOTUPLE('map', TOTUPLE('m', 'mm'), TOTUPLE('n', 'nn')));");
+ pig.registerQuery("STORE recs INTO 'cql://cql3ks/collectiontable?" + defaultParameters + "&output_query=update+cql3ks.collectiontable+set+n+%3D+%3F' USING CqlStorage();");
+ pig.executeBatch();
+
- //(book2,((m,mm),(n,nn)))
- //(book3,((m,mm),(n,nn)))
- //(book4,((m,mm),(n,nn)))
- //(book1,((m,mm),(n,nn)))
++ //(book2,((key2, value2),(m,mm),(n,nn)))
++ //(book3,((key3, value3),(m,mm),(n,nn)))
++ //(book4,((key4, value4),(m,mm),(n,nn)))
++ //(book1,((key1, value1),(m,mm),(n,nn)))
+ pig.registerQuery("result= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + "' USING CqlStorage();");
+ Iterator<Tuple> it = pig.openIterator("result");
- if (it.hasNext()) {
++ while (it.hasNext()) {
+ Tuple t = it.next();
+ Tuple t1 = (Tuple) t.get(1);
- Assert.assertEquals(t1.size(), 2);
- Tuple element1 = (Tuple) t1.get(0);
- Tuple element2 = (Tuple) t1.get(1);
++ Assert.assertEquals(t1.size(), 3);
++ Tuple element1 = (Tuple) t1.get(1);
++ Tuple element2 = (Tuple) t1.get(2);
+ Assert.assertEquals(element1.get(0), "m");
+ Assert.assertEquals(element1.get(1), "mm");
+ Assert.assertEquals(element2.get(0), "n");
+ Assert.assertEquals(element2.get(1), "nn");
+ }
+ }
+
+ @Test
+ public void testCassandraStorageSchema() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException
+ {
+ //results: (key1,{((111,),),((111,column1),100),((111,column2),10.1)})
+ pig.registerQuery("rows = LOAD 'cassandra://cql3ks/cqltable?" + defaultParameters + "' USING CassandraStorage();");
+
+ //schema: {key: chararray,columns: {(name: (),value: bytearray)}}
+ Iterator<Tuple> it = pig.openIterator("rows");
+ if (it.hasNext()) {
+ Tuple t = it.next();
+ String rowKey = t.get(0).toString();
+ Assert.assertEquals(rowKey, "key1");
+ DataBag columns = (DataBag) t.get(1);
+ Iterator<Tuple> iter = columns.iterator();
+ int i = 0;
+ while(iter.hasNext())
+ {
+ i++;
+ Tuple column = (Tuple) iter.next();
+ if (i==1)
+ {
+ Assert.assertEquals(((Tuple) column.get(0)).get(0), 111);
+ Assert.assertEquals(((Tuple) column.get(0)).get(1), "");
+ Assert.assertEquals(column.get(1).toString(), "");
+ }
+ if (i==2)
+ {
+ Assert.assertEquals(((Tuple) column.get(0)).get(0), 111);
+ Assert.assertEquals(((Tuple) column.get(0)).get(1), "column1");
+ Assert.assertEquals(column.get(1), 100);
+ }
+ if (i==3)
+ {
+ Assert.assertEquals(((Tuple) column.get(0)).get(0), 111);
+ Assert.assertEquals(((Tuple) column.get(0)).get(1), "column2");
+ Assert.assertEquals(column.get(1), 10.1f);
+ }
+ }
+ Assert.assertEquals(3, columns.size());
+ }
+
+ //results: (key1,(column1,100),(column2,10.1))
+ pig.registerQuery("compact_rows = LOAD 'cassandra://cql3ks/compactcqltable?" + defaultParameters + "' USING CassandraStorage();");
+
+ //schema: {key: chararray,column1: (name: chararray,value: int),column2: (name: chararray,value: float)}
+ it = pig.openIterator("compact_rows");
+ if (it.hasNext()) {
+ Tuple t = it.next();
+ String rowKey = t.get(0).toString();
+ Assert.assertEquals(rowKey, "key1");
+ Tuple column = (Tuple) t.get(1);
+ Assert.assertEquals(column.get(0), "column1");
+ Assert.assertEquals(column.get(1), 100);
+ column = (Tuple) t.get(2);
+ Assert.assertEquals(column.get(0), "column2");
+ Assert.assertEquals(column.get(1), 10.1f);
+ }
+ }
+}
[02/12] git commit: Fix failing pig test
Posted by jb...@apache.org.
Fix failing pig test
Patch by Alex Liu, reviewed by brandonwilliams for CASSANDRA-6376
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8c52cc0e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8c52cc0e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8c52cc0e
Branch: refs/heads/trunk
Commit: 8c52cc0e7f67378d66093c958558e765b682888f
Parents: 972d3da
Author: Brandon Williams <br...@apache.org>
Authored: Fri Mar 14 14:46:53 2014 -0500
Committer: Brandon Williams <br...@apache.org>
Committed: Fri Mar 14 14:46:53 2014 -0500
----------------------------------------------------------------------
.../unit/org/apache/cassandra/pig/CqlTableTest.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/8c52cc0e/test/unit/org/apache/cassandra/pig/CqlTableTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/pig/CqlTableTest.java b/test/unit/org/apache/cassandra/pig/CqlTableTest.java
index 785d819..55586c7 100644
--- a/test/unit/org/apache/cassandra/pig/CqlTableTest.java
+++ b/test/unit/org/apache/cassandra/pig/CqlTableTest.java
@@ -174,18 +174,18 @@ public class CqlTableTest extends PigTestBase
pig.registerQuery("STORE recs INTO 'cql://cql3ks/collectiontable?" + defaultParameters + "&output_query=update+cql3ks.collectiontable+set+n+%3D+%3F' USING CqlStorage();");
pig.executeBatch();
- //(book2,((m,mm),(n,nn)))
- //(book3,((m,mm),(n,nn)))
- //(book4,((m,mm),(n,nn)))
- //(book1,((m,mm),(n,nn)))
+ //(book2,((key2, value2),(m,mm),(n,nn)))
+ //(book3,((key3, value3),(m,mm),(n,nn)))
+ //(book4,((key4, value4),(m,mm),(n,nn)))
+ //(book1,((key1, value1),(m,mm),(n,nn)))
pig.registerQuery("result= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + "' USING CqlStorage();");
Iterator<Tuple> it = pig.openIterator("result");
- if (it.hasNext()) {
+ while (it.hasNext()) {
Tuple t = it.next();
Tuple t1 = (Tuple) t.get(1);
- Assert.assertEquals(t1.size(), 2);
- Tuple element1 = (Tuple) t1.get(0);
- Tuple element2 = (Tuple) t1.get(1);
+ Assert.assertEquals(t1.size(), 3);
+ Tuple element1 = (Tuple) t1.get(1);
+ Tuple element2 = (Tuple) t1.get(2);
Assert.assertEquals(element1.get(0), "m");
Assert.assertEquals(element1.get(1), "mm");
Assert.assertEquals(element2.get(0), "n");
[11/12] git commit: merge from 2.0
Posted by jb...@apache.org.
merge from 2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e0718bcd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e0718bcd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e0718bcd
Branch: refs/heads/trunk
Commit: e0718bcdfbd3b52033adf7d3dd23094060b6990a
Parents: 3595131 57c5969
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Mar 14 15:40:04 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Mar 14 15:40:04 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/filter/SliceQueryFilter.java | 20 ++++++++++++++++++--
.../org/apache/cassandra/pig/CqlTableTest.java | 16 ++++++++--------
3 files changed, 27 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e0718bcd/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 9575c8b,61b1ee1..ea436f0
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,28 -1,5 +1,29 @@@
-2.0.7
+2.1.0-beta2
+ * Eliminate possibility of CL segment appearing twice in active list
+ (CASSANDRA-6557)
+ * Apply DONTNEED fadvise to commitlog segments (CASSANDRA-6759)
+ * Switch CRC component to Adler and include it for compressed sstables
+ (CASSANDRA-4165)
+ * Allow cassandra-stress to set compaction strategy options (CASSANDRA-6451)
+ * Add broadcast_rpc_address option to cassandra.yaml (CASSANDRA-5899)
+ * Auto reload GossipingPropertyFileSnitch config (CASSANDRA-5897)
+ * Fix overflow of memtable_total_space_in_mb (CASSANDRA-6573)
+ * Fix ABTC NPE and apply update function correctly (CASSANDRA-6692)
+ * Allow nodetool to use a file or prompt for password (CASSANDRA-6660)
+ * Fix AIOOBE when concurrently accessing ABSC (CASSANDRA-6742)
+ * Fix assertion error in ALTER TYPE RENAME (CASSANDRA-6705)
+ * Scrub should not always clear out repaired status (CASSANDRA-5351)
+ * Improve handling of range tombstone for wide partitions (CASSANDRA-6446)
+ * Fix ClassCastException for compact table with composites (CASSANDRA-6738)
+ * Fix potentially repairing with wrong nodes (CASSANDRA-6808)
+ * Change caching option syntax (CASSANDRA-6745)
+ * Fix stress to do proper counter reads (CASSANDRA-6835)
+ * Fix help message for stress counter_write (CASSANDRA-6824)
+ * Fix stress smart Thrift client to pick servers correctly (CASSANDRA-6848)
+ * Add logging levels (minimal, normal or verbose) to stress tool (CASSANDRA-6849)
+ * Fix race condition in Batch CLE (CASSANDRA-6860)
+Merged from 2.0:
+ * Log more information when exceeding tombstone_warn_threshold (CASSANDRA-6865)
* Fix truncate to not abort due to unreachable fat clients (CASSANDRA-6864)
* Fix schema concurrency exceptions (CASSANDRA-6841)
* Fix leaking validator FH in StreamWriter (CASSANDRA-6832)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e0718bcd/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
[09/12] git commit: Log more information when exceeding
tombstone_warn_threshold patch by Oleg Anastasyev;
reviewed by jbellis for CASSANDRA-6865
Posted by jb...@apache.org.
Log more information when exceeding tombstone_warn_threshold
patch by Oleg Anastasyev; reviewed by jbellis for CASSANDRA-6865
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/57c5969e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/57c5969e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/57c5969e
Branch: refs/heads/trunk
Commit: 57c5969e1dac82c6a676668edf1dc5ba64bcbf1f
Parents: 7ce3950
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Mar 14 15:39:19 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Mar 14 15:39:19 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/filter/SliceQueryFilter.java | 20 ++++++++++++++++++--
2 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/57c5969e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 045e4f8..61b1ee1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.7
+ * Log more information when exceeding tombstone_warn_threshold (CASSANDRA-6865)
* Fix truncate to not abort due to unreachable fat clients (CASSANDRA-6864)
* Fix schema concurrency exceptions (CASSANDRA-6841)
* Fix leaking validator FH in StreamWriter (CASSANDRA-6832)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/57c5969e/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index b43308f..356418e 100644
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@ -207,8 +207,24 @@ public class SliceQueryFilter implements IDiskAtomFilter
Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
if (respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneWarnThreshold())
- logger.warn("Read {} live and {} tombstoned cells in {}.{} (see tombstone_warn_threshold)",
- columnCounter.live(), columnCounter.ignored(), container.metadata().ksName, container.metadata().cfName);
+ {
+ StringBuilder sb = new StringBuilder();
+ AbstractType<?> type = container.metadata().comparator;
+ for (ColumnSlice sl : slices)
+ {
+ if (sl == null)
+ continue;
+
+ sb.append('[');
+ sb.append(type.getString(sl.start));
+ sb.append('-');
+ sb.append(type.getString(sl.finish));
+ sb.append(']');
+ }
+
+ logger.warn("Read {} live and {} tombstoned cells in {}.{} (see tombstone_warn_threshold). {} columns was requested, slices={}, delInfo={}",
+ columnCounter.live(), columnCounter.ignored(), container.metadata().ksName, container.metadata().cfName, count, sb, container.deletionInfo());
+ }
}
protected boolean respectTombstoneThresholds()
[10/12] git commit: merge from 2.0
Posted by jb...@apache.org.
merge from 2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e0718bcd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e0718bcd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e0718bcd
Branch: refs/heads/cassandra-2.1
Commit: e0718bcdfbd3b52033adf7d3dd23094060b6990a
Parents: 3595131 57c5969
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Mar 14 15:40:04 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Mar 14 15:40:04 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/filter/SliceQueryFilter.java | 20 ++++++++++++++++++--
.../org/apache/cassandra/pig/CqlTableTest.java | 16 ++++++++--------
3 files changed, 27 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e0718bcd/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 9575c8b,61b1ee1..ea436f0
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,28 -1,5 +1,29 @@@
-2.0.7
+2.1.0-beta2
+ * Eliminate possibility of CL segment appearing twice in active list
+ (CASSANDRA-6557)
+ * Apply DONTNEED fadvise to commitlog segments (CASSANDRA-6759)
+ * Switch CRC component to Adler and include it for compressed sstables
+ (CASSANDRA-4165)
+ * Allow cassandra-stress to set compaction strategy options (CASSANDRA-6451)
+ * Add broadcast_rpc_address option to cassandra.yaml (CASSANDRA-5899)
+ * Auto reload GossipingPropertyFileSnitch config (CASSANDRA-5897)
+ * Fix overflow of memtable_total_space_in_mb (CASSANDRA-6573)
+ * Fix ABTC NPE and apply update function correctly (CASSANDRA-6692)
+ * Allow nodetool to use a file or prompt for password (CASSANDRA-6660)
+ * Fix AIOOBE when concurrently accessing ABSC (CASSANDRA-6742)
+ * Fix assertion error in ALTER TYPE RENAME (CASSANDRA-6705)
+ * Scrub should not always clear out repaired status (CASSANDRA-5351)
+ * Improve handling of range tombstone for wide partitions (CASSANDRA-6446)
+ * Fix ClassCastException for compact table with composites (CASSANDRA-6738)
+ * Fix potentially repairing with wrong nodes (CASSANDRA-6808)
+ * Change caching option syntax (CASSANDRA-6745)
+ * Fix stress to do proper counter reads (CASSANDRA-6835)
+ * Fix help message for stress counter_write (CASSANDRA-6824)
+ * Fix stress smart Thrift client to pick servers correctly (CASSANDRA-6848)
+ * Add logging levels (minimal, normal or verbose) to stress tool (CASSANDRA-6849)
+ * Fix race condition in Batch CLE (CASSANDRA-6860)
+Merged from 2.0:
+ * Log more information when exceeding tombstone_warn_threshold (CASSANDRA-6865)
* Fix truncate to not abort due to unreachable fat clients (CASSANDRA-6864)
* Fix schema concurrency exceptions (CASSANDRA-6841)
* Fix leaking validator FH in StreamWriter (CASSANDRA-6832)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/e0718bcd/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
[03/12] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0
Posted by jb...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f122f999
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f122f999
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f122f999
Branch: refs/heads/cassandra-2.1
Commit: f122f9994e92e4a1d7de32ff9f6bc116a4a65d2a
Parents: 22b9453 8c52cc0
Author: Brandon Williams <br...@apache.org>
Authored: Fri Mar 14 14:47:31 2014 -0500
Committer: Brandon Williams <br...@apache.org>
Committed: Fri Mar 14 14:47:31 2014 -0500
----------------------------------------------------------------------
test/pig/org/apache/cassandra/pig/CqlTableTest.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f122f999/test/pig/org/apache/cassandra/pig/CqlTableTest.java
----------------------------------------------------------------------
diff --cc test/pig/org/apache/cassandra/pig/CqlTableTest.java
index e33364f,0000000..3bbc3d1
mode 100644,000000..100644
--- a/test/pig/org/apache/cassandra/pig/CqlTableTest.java
+++ b/test/pig/org/apache/cassandra/pig/CqlTableTest.java
@@@ -1,257 -1,0 +1,257 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.pig;
+
+import java.io.IOException;
+import java.nio.charset.CharacterCodingException;
+import java.util.Iterator;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.thrift.AuthenticationException;
+import org.apache.cassandra.thrift.AuthorizationException;
+import org.apache.cassandra.thrift.InvalidRequestException;
+import org.apache.cassandra.thrift.NotFoundException;
+import org.apache.cassandra.thrift.SchemaDisagreementException;
+import org.apache.cassandra.thrift.TimedOutException;
+import org.apache.cassandra.thrift.UnavailableException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.thrift.TException;
+import org.apache.thrift.transport.TTransportException;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class CqlTableTest extends PigTestBase
+{
+ private static String[] statements = {
+ "CREATE KEYSPACE cql3ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1}",
+ "USE cql3ks;",
+
+ "CREATE TABLE cqltable (key1 text, key2 int, column1 int, column2 float, primary key(key1, key2))",
+ "INSERT INTO cqltable (key1, key2, column1, column2) values ('key1', 111, 100, 10.1)",
+ "CREATE TABLE compactcqltable (key1 text, column1 int, column2 float, primary key(key1)) WITH COMPACT STORAGE",
+ "INSERT INTO compactcqltable (key1, column1, column2) values ('key1', 100, 10.1)",
+
+ "CREATE TABLE test (a int PRIMARY KEY, b int);",
+ "CREATE INDEX test_b on test (b);",
+
+ "CREATE TABLE moredata (x int PRIMARY KEY, y int);",
+ "INSERT INTO test (a,b) VALUES (1,1);",
+ "INSERT INTO test (a,b) VALUES (2,2);",
+ "INSERT INTO test (a,b) VALUES (3,3);",
+ "INSERT INTO moredata (x, y) VALUES (4,4);",
+ "INSERT INTO moredata (x, y) VALUES (5,5);",
+ "INSERT INTO moredata (x, y) VALUES (6,6);",
+
+ "CREATE TABLE compotable (a int, b int, c text, d text, PRIMARY KEY (a,b,c));",
+ "INSERT INTO compotable (a, b , c , d ) VALUES ( 1,1,'One','match');",
+ "INSERT INTO compotable (a, b , c , d ) VALUES ( 2,2,'Two','match');",
+ "INSERT INTO compotable (a, b , c , d ) VALUES ( 3,3,'Three','match');",
+ "INSERT INTO compotable (a, b , c , d ) VALUES ( 4,4,'Four','match');",
+
+ "create table compmore (id int PRIMARY KEY, x int, y int, z text, data text);",
+ "INSERT INTO compmore (id, x, y, z,data) VALUES (1,5,6,'Fix','nomatch');",
+ "INSERT INTO compmore (id, x, y, z,data) VALUES (2,6,5,'Sive','nomatch');",
+ "INSERT INTO compmore (id, x, y, z,data) VALUES (3,7,7,'Seven','match');",
+ "INSERT INTO compmore (id, x, y, z,data) VALUES (4,8,8,'Eight','match');",
+ "INSERT INTO compmore (id, x, y, z,data) VALUES (5,9,10,'Ninen','nomatch');",
+
+ "CREATE TABLE collectiontable(m text PRIMARY KEY, n map<text, text>);",
+ "UPDATE collectiontable SET n['key1'] = 'value1' WHERE m = 'book1';",
+ "UPDATE collectiontable SET n['key2'] = 'value2' WHERE m = 'book2';",
+ "UPDATE collectiontable SET n['key3'] = 'value3' WHERE m = 'book3';",
+ "UPDATE collectiontable SET n['key4'] = 'value4' WHERE m = 'book4';",
+ };
+
+ @BeforeClass
+ public static void setup() throws TTransportException, IOException, InterruptedException, ConfigurationException,
+ AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, CharacterCodingException, ClassNotFoundException, NoSuchFieldException, IllegalAccessException, InstantiationException
+ {
+ startCassandra();
+ setupDataByCql(statements);
+ startHadoopCluster();
+ }
+
+ @Test
+ public void testCqlStorageSchema()
+ throws AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, SchemaDisagreementException, IOException
+ {
+ pig.registerQuery("rows = LOAD 'cql://cql3ks/cqltable?" + defaultParameters + "' USING CqlStorage();");
+ Iterator<Tuple> it = pig.openIterator("rows");
+ if (it.hasNext()) {
+ Tuple t = it.next();
+ Assert.assertEquals(t.get(0).toString(), "key1");
+ Assert.assertEquals(t.get(1), 111);
+ Assert.assertEquals(t.get(2), 100);
+ Assert.assertEquals(t.get(3), 10.1f);
+ Assert.assertEquals(4, t.size());
+ }
+
+ pig.registerQuery("rows = LOAD 'cql://cql3ks/compactcqltable?" + defaultParameters + "' USING CqlStorage();");
+ it = pig.openIterator("rows");
+ if (it.hasNext()) {
+ Tuple t = it.next();
+ Assert.assertEquals(t.get(0).toString(), "key1");
+ Assert.assertEquals(t.get(1), 100);
+ Assert.assertEquals(t.get(2), 10.1f);
+ Assert.assertEquals(3, t.size());
+ }
+ }
+
+ @Test
+ public void testCqlStorageSingleKeyTable()
+ throws AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, SchemaDisagreementException, IOException
+ {
+ pig.setBatchOn();
+ pig.registerQuery("moretestvalues= LOAD 'cql://cql3ks/moredata?" + defaultParameters + "' USING CqlStorage();");
+ pig.registerQuery("insertformat= FOREACH moretestvalues GENERATE TOTUPLE(TOTUPLE('a',x)),TOTUPLE(y);");
+ pig.registerQuery("STORE insertformat INTO 'cql://cql3ks/test?" + defaultParameters + "&output_query=UPDATE+cql3ks.test+set+b+%3D+%3F' USING CqlStorage();");
+ pig.executeBatch();
+ //(5,5)
+ //(6,6)
+ //(4,4)
+ //(2,2)
+ //(3,3)
+ //(1,1)
+ pig.registerQuery("result= LOAD 'cql://cql3ks/test?" + defaultParameters + "' USING CqlStorage();");
+ Iterator<Tuple> it = pig.openIterator("result");
+ if (it.hasNext()) {
+ Tuple t = it.next();
+ Assert.assertEquals(t.get(0), t.get(1));
+ }
+ }
+
+ @Test
+ public void testCqlStorageCompositeKeyTable()
+ throws AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, SchemaDisagreementException, IOException
+ {
+ pig.setBatchOn();
+ pig.registerQuery("moredata= LOAD 'cql://cql3ks/compmore?" + defaultParameters + "' USING CqlStorage();");
+ pig.registerQuery("insertformat = FOREACH moredata GENERATE TOTUPLE (TOTUPLE('a',x),TOTUPLE('b',y), TOTUPLE('c',z)),TOTUPLE(data);");
+ pig.registerQuery("STORE insertformat INTO 'cql://cql3ks/compotable?" + defaultParameters + "&output_query=UPDATE%20cql3ks.compotable%20SET%20d%20%3D%20%3F' USING CqlStorage();");
+ pig.executeBatch();
+
+ //(5,6,Fix,nomatch)
+ //(3,3,Three,match)
+ //(1,1,One,match)
+ //(2,2,Two,match)
+ //(7,7,Seven,match)
+ //(8,8,Eight,match)
+ //(6,5,Sive,nomatch)
+ //(4,4,Four,match)
+ //(9,10,Ninen,nomatch)
+ pig.registerQuery("result= LOAD 'cql://cql3ks/compotable?" + defaultParameters + "' USING CqlStorage();");
+ Iterator<Tuple> it = pig.openIterator("result");
+ int count = 0;
+ while (it.hasNext()) {
+ it.next();
+ count ++;
+ }
+ Assert.assertEquals(count, 9);
+ }
+
+ @Test
+ public void testCqlStorageCollectionColumnTable()
+ throws AuthenticationException, AuthorizationException, InvalidRequestException, UnavailableException, TimedOutException, TException, NotFoundException, SchemaDisagreementException, IOException
+ {
+ pig.setBatchOn();
+ pig.registerQuery("collectiontable= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + "' USING CqlStorage();");
+ pig.registerQuery("recs= FOREACH collectiontable GENERATE TOTUPLE(TOTUPLE('m', m) ), TOTUPLE(TOTUPLE('map', TOTUPLE('m', 'mm'), TOTUPLE('n', 'nn')));");
+ pig.registerQuery("STORE recs INTO 'cql://cql3ks/collectiontable?" + defaultParameters + "&output_query=update+cql3ks.collectiontable+set+n+%3D+%3F' USING CqlStorage();");
+ pig.executeBatch();
+
- //(book2,((m,mm),(n,nn)))
- //(book3,((m,mm),(n,nn)))
- //(book4,((m,mm),(n,nn)))
- //(book1,((m,mm),(n,nn)))
++ //(book2,((key2, value2),(m,mm),(n,nn)))
++ //(book3,((key3, value3),(m,mm),(n,nn)))
++ //(book4,((key4, value4),(m,mm),(n,nn)))
++ //(book1,((key1, value1),(m,mm),(n,nn)))
+ pig.registerQuery("result= LOAD 'cql://cql3ks/collectiontable?" + defaultParameters + "' USING CqlStorage();");
+ Iterator<Tuple> it = pig.openIterator("result");
- if (it.hasNext()) {
++ while (it.hasNext()) {
+ Tuple t = it.next();
+ Tuple t1 = (Tuple) t.get(1);
- Assert.assertEquals(t1.size(), 2);
- Tuple element1 = (Tuple) t1.get(0);
- Tuple element2 = (Tuple) t1.get(1);
++ Assert.assertEquals(t1.size(), 3);
++ Tuple element1 = (Tuple) t1.get(1);
++ Tuple element2 = (Tuple) t1.get(2);
+ Assert.assertEquals(element1.get(0), "m");
+ Assert.assertEquals(element1.get(1), "mm");
+ Assert.assertEquals(element2.get(0), "n");
+ Assert.assertEquals(element2.get(1), "nn");
+ }
+ }
+
+ @Test
+ public void testCassandraStorageSchema() throws IOException, ClassNotFoundException, TException, TimedOutException, NotFoundException, InvalidRequestException, NoSuchFieldException, UnavailableException, IllegalAccessException, InstantiationException
+ {
+ //results: (key1,{((111,),),((111,column1),100),((111,column2),10.1)})
+ pig.registerQuery("rows = LOAD 'cassandra://cql3ks/cqltable?" + defaultParameters + "' USING CassandraStorage();");
+
+ //schema: {key: chararray,columns: {(name: (),value: bytearray)}}
+ Iterator<Tuple> it = pig.openIterator("rows");
+ if (it.hasNext()) {
+ Tuple t = it.next();
+ String rowKey = t.get(0).toString();
+ Assert.assertEquals(rowKey, "key1");
+ DataBag columns = (DataBag) t.get(1);
+ Iterator<Tuple> iter = columns.iterator();
+ int i = 0;
+ while(iter.hasNext())
+ {
+ i++;
+ Tuple column = (Tuple) iter.next();
+ if (i==1)
+ {
+ Assert.assertEquals(((Tuple) column.get(0)).get(0), 111);
+ Assert.assertEquals(((Tuple) column.get(0)).get(1), "");
+ Assert.assertEquals(column.get(1).toString(), "");
+ }
+ if (i==2)
+ {
+ Assert.assertEquals(((Tuple) column.get(0)).get(0), 111);
+ Assert.assertEquals(((Tuple) column.get(0)).get(1), "column1");
+ Assert.assertEquals(column.get(1), 100);
+ }
+ if (i==3)
+ {
+ Assert.assertEquals(((Tuple) column.get(0)).get(0), 111);
+ Assert.assertEquals(((Tuple) column.get(0)).get(1), "column2");
+ Assert.assertEquals(column.get(1), 10.1f);
+ }
+ }
+ Assert.assertEquals(3, columns.size());
+ }
+
+ //results: (key1,(column1,100),(column2,10.1))
+ pig.registerQuery("compact_rows = LOAD 'cassandra://cql3ks/compactcqltable?" + defaultParameters + "' USING CassandraStorage();");
+
+ //schema: {key: chararray,column1: (name: chararray,value: int),column2: (name: chararray,value: float)}
+ it = pig.openIterator("compact_rows");
+ if (it.hasNext()) {
+ Tuple t = it.next();
+ String rowKey = t.get(0).toString();
+ Assert.assertEquals(rowKey, "key1");
+ Tuple column = (Tuple) t.get(1);
+ Assert.assertEquals(column.get(0), "column1");
+ Assert.assertEquals(column.get(1), 100);
+ column = (Tuple) t.get(2);
+ Assert.assertEquals(column.get(0), "column2");
+ Assert.assertEquals(column.get(1), 10.1f);
+ }
+ }
+}
[12/12] git commit: Merge branch 'cassandra-2.1' into trunk
Posted by jb...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5011437b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5011437b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5011437b
Branch: refs/heads/trunk
Commit: 5011437b1e8e82a42ea532e454c8df242e12a620
Parents: c54c618 e0718bc
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Mar 14 15:40:09 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Mar 14 15:40:09 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/filter/SliceQueryFilter.java | 20 ++++++++++++++++++--
.../org/apache/cassandra/pig/CqlTableTest.java | 16 ++++++++--------
3 files changed, 27 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5011437b/CHANGES.txt
----------------------------------------------------------------------
[05/12] git commit: Merge remote-tracking branch
'origin/cassandra-2.0' into cassandra-2.0
Posted by jb...@apache.org.
Merge remote-tracking branch 'origin/cassandra-2.0' into cassandra-2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7ce3950a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7ce3950a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7ce3950a
Branch: refs/heads/trunk
Commit: 7ce3950a97a576b00c9936380cc3c5dc750ef4e7
Parents: 3050134 f122f99
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Mar 14 15:28:10 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Mar 14 15:28:10 2014 -0500
----------------------------------------------------------------------
test/pig/org/apache/cassandra/pig/CqlTableTest.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[08/12] git commit: Log more information when exceeding
tombstone_warn_threshold patch by Oleg Anastasyev;
reviewed by jbellis for CASSANDRA-6865
Posted by jb...@apache.org.
Log more information when exceeding tombstone_warn_threshold
patch by Oleg Anastasyev; reviewed by jbellis for CASSANDRA-6865
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/57c5969e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/57c5969e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/57c5969e
Branch: refs/heads/cassandra-2.0
Commit: 57c5969e1dac82c6a676668edf1dc5ba64bcbf1f
Parents: 7ce3950
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Mar 14 15:39:19 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Mar 14 15:39:19 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/db/filter/SliceQueryFilter.java | 20 ++++++++++++++++++--
2 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/57c5969e/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 045e4f8..61b1ee1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.7
+ * Log more information when exceeding tombstone_warn_threshold (CASSANDRA-6865)
* Fix truncate to not abort due to unreachable fat clients (CASSANDRA-6864)
* Fix schema concurrency exceptions (CASSANDRA-6841)
* Fix leaking validator FH in StreamWriter (CASSANDRA-6832)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/57c5969e/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index b43308f..356418e 100644
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@ -207,8 +207,24 @@ public class SliceQueryFilter implements IDiskAtomFilter
Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
if (respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneWarnThreshold())
- logger.warn("Read {} live and {} tombstoned cells in {}.{} (see tombstone_warn_threshold)",
- columnCounter.live(), columnCounter.ignored(), container.metadata().ksName, container.metadata().cfName);
+ {
+ StringBuilder sb = new StringBuilder();
+ AbstractType<?> type = container.metadata().comparator;
+ for (ColumnSlice sl : slices)
+ {
+ if (sl == null)
+ continue;
+
+ sb.append('[');
+ sb.append(type.getString(sl.start));
+ sb.append('-');
+ sb.append(type.getString(sl.finish));
+ sb.append(']');
+ }
+
+ logger.warn("Read {} live and {} tombstoned cells in {}.{} (see tombstone_warn_threshold). {} columns was requested, slices={}, delInfo={}",
+ columnCounter.live(), columnCounter.ignored(), container.metadata().ksName, container.metadata().cfName, count, sb, container.deletionInfo());
+ }
}
protected boolean respectTombstoneThresholds()