You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yu...@apache.org on 2013/07/24 18:13:51 UTC

[1/4] git commit: now() is being rejected in INSERTs when inside collections

Updated Branches:
  refs/heads/cassandra-1.2 ce4e4b9b5 -> 422d2236b
  refs/heads/trunk 371e7bf69 -> 813577e32


now() is being rejected in INSERTs when inside collections

patch by slebresne; reviewed by iamaleksey for CASSANDRA-5795


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ce4e4b9b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ce4e4b9b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ce4e4b9b

Branch: refs/heads/trunk
Commit: ce4e4b9b5b29b0e0518aafe02544de3765ea9dd7
Parents: d38446a
Author: Sylvain Lebresne <sy...@datastax.com>
Authored: Wed Jul 24 17:46:52 2013 +0200
Committer: Sylvain Lebresne <sy...@datastax.com>
Committed: Wed Jul 24 17:46:52 2013 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../apache/cassandra/cql3/AbstractMarker.java   |  5 ++
 src/java/org/apache/cassandra/cql3/Lists.java   | 76 ++++++++++++++----
 src/java/org/apache/cassandra/cql3/Maps.java    | 84 +++++++++++++++-----
 src/java/org/apache/cassandra/cql3/Sets.java    | 73 +++++++++++++----
 src/java/org/apache/cassandra/cql3/Term.java    | 28 ++++++-
 .../cassandra/cql3/functions/FunctionCall.java  | 10 +++
 7 files changed, 219 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce4e4b9b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index aaeb10b..71a956b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -27,6 +27,7 @@
    after TTL expires (CASSANDRA-5762)
  * Sort nodetool help output (CASSANDRA-5776)
  * Fix column expiring during 2 phases compaction (CASSANDRA-5799)
+ * now() is being rejected in INSERTs when inside collections (CASSANDRA-5795)
 
 
 1.2.6

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce4e4b9b/src/java/org/apache/cassandra/cql3/AbstractMarker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/AbstractMarker.java b/src/java/org/apache/cassandra/cql3/AbstractMarker.java
index d8353a1..b4a4143 100644
--- a/src/java/org/apache/cassandra/cql3/AbstractMarker.java
+++ b/src/java/org/apache/cassandra/cql3/AbstractMarker.java
@@ -40,6 +40,11 @@ public abstract class AbstractMarker extends Term.NonTerminal
         boundNames[bindIndex] = receiver;
     }
 
+    public boolean containsBindMarker()
+    {
+        return true;
+    }
+
     /**
      * A parsed, but non prepared, bind marker.
      */

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce4e4b9b/src/java/org/apache/cassandra/cql3/Lists.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/Lists.java b/src/java/org/apache/cassandra/cql3/Lists.java
index b579b8c..ae95dca 100644
--- a/src/java/org/apache/cassandra/cql3/Lists.java
+++ b/src/java/org/apache/cassandra/cql3/Lists.java
@@ -19,6 +19,7 @@ package org.apache.cassandra.cql3;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -60,34 +61,27 @@ public abstract class Lists
             this.elements = elements;
         }
 
-        public Value prepare(ColumnSpecification receiver) throws InvalidRequestException
+        public Term prepare(ColumnSpecification receiver) throws InvalidRequestException
         {
             validateAssignableTo(receiver);
 
             ColumnSpecification valueSpec = Lists.valueSpecOf(receiver);
-            List<ByteBuffer> values = new ArrayList<ByteBuffer>(elements.size());
+            List<Term> values = new ArrayList<Term>(elements.size());
+            boolean allTerminal = true;
             for (Term.Raw rt : elements)
             {
                 Term t = rt.prepare(valueSpec);
 
-                if (t instanceof Term.NonTerminal)
+                if (t.containsBindMarker())
                     throw new InvalidRequestException(String.format("Invalid list literal for %s: bind variables are not supported inside collection literals", receiver));
 
-                // We don't allow prepared marker in collections, nor nested collections (for the later, prepare will throw an exception)
-                assert t instanceof Constants.Value;
-                ByteBuffer bytes = ((Constants.Value)t).bytes;
-                if (bytes == null)
-                    throw new InvalidRequestException("null is not supported inside collections");
-
-                // We don't support value > 64K because the serialization format encode the length as an unsigned short.
-                if (bytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
-                    throw new InvalidRequestException(String.format("List value is too long. List values are limited to %d bytes but %d bytes value provided",
-                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
-                                                                    bytes.remaining()));
+                if (t instanceof Term.NonTerminal)
+                    allTerminal = false;
 
-                values.add(bytes);
+                values.add(t);
             }
-            return new Value(values);
+            DelayedValue value = new DelayedValue(values);
+            return allTerminal ? value.bind(Collections.<ByteBuffer>emptyList()) : value;
         }
 
         private void validateAssignableTo(ColumnSpecification receiver) throws InvalidRequestException
@@ -156,6 +150,56 @@ public abstract class Lists
         }
     }
 
+    /*
+     * Basically similar to a Value, but with some non-pure function (that need
+     * to be evaluated at execution time) in it.
+     *
+     * Note: this would also work for a list with bind markers, but we don't support
+     * that because 1) it's not excessively useful and 2) we wouldn't have a good
+     * column name to return in the ColumnSpecification for those markers (not a
+     * blocker per-se but we don't bother due to 1)).
+     */
+    public static class DelayedValue extends Term.NonTerminal
+    {
+        private final List<Term> elements;
+
+        public DelayedValue(List<Term> elements)
+        {
+            this.elements = elements;
+        }
+
+        public boolean containsBindMarker()
+        {
+            // False since we don't support them in collection
+            return false;
+        }
+
+        public void collectMarkerSpecification(ColumnSpecification[] boundNames)
+        {
+        }
+
+        public Value bind(List<ByteBuffer> values) throws InvalidRequestException
+        {
+            List<ByteBuffer> buffers = new ArrayList<ByteBuffer>(elements.size());
+            for (Term t : elements)
+            {
+                ByteBuffer bytes = t.bindAndGet(values);
+
+                if (bytes == null)
+                    throw new InvalidRequestException("null is not supported inside collections");
+
+                // We don't support value > 64K because the serialization format encode the length as an unsigned short.
+                if (bytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
+                    throw new InvalidRequestException(String.format("List value is too long. List values are limited to %d bytes but %d bytes value provided",
+                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
+                                                                    bytes.remaining()));
+
+                buffers.add(bytes);
+            }
+            return new Value(buffers);
+        }
+    }
+
     public static class Marker extends AbstractMarker
     {
         protected Marker(int bindIndex, ColumnSpecification receiver)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce4e4b9b/src/java/org/apache/cassandra/cql3/Maps.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/Maps.java b/src/java/org/apache/cassandra/cql3/Maps.java
index e6d7c7e..e34e076 100644
--- a/src/java/org/apache/cassandra/cql3/Maps.java
+++ b/src/java/org/apache/cassandra/cql3/Maps.java
@@ -19,6 +19,9 @@ package org.apache.cassandra.cql3;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -58,42 +61,29 @@ public abstract class Maps
             this.entries = entries;
         }
 
-        public Value prepare(ColumnSpecification receiver) throws InvalidRequestException
+        public Term prepare(ColumnSpecification receiver) throws InvalidRequestException
         {
             validateAssignableTo(receiver);
 
             ColumnSpecification keySpec = Maps.keySpecOf(receiver);
             ColumnSpecification valueSpec = Maps.valueSpecOf(receiver);
-            Map<ByteBuffer, ByteBuffer> values = new TreeMap<ByteBuffer, ByteBuffer>(((MapType)receiver.type).keys);
+            Map<Term, Term> values = new HashMap<Term, Term>(entries.size());
+            boolean allTerminal = true;
             for (Pair<Term.Raw, Term.Raw> entry : entries)
             {
                 Term k = entry.left.prepare(keySpec);
                 Term v = entry.right.prepare(valueSpec);
 
-                if (k instanceof Term.NonTerminal || v instanceof Term.NonTerminal)
+                if (k.containsBindMarker() || v.containsBindMarker())
                     throw new InvalidRequestException(String.format("Invalid map literal for %s: bind variables are not supported inside collection literals", receiver));
 
-                // We don't support values > 64K because the serialization format encode the length as an unsigned short.
-                ByteBuffer keyBytes = ((Constants.Value)k).bytes;
-                if (keyBytes == null)
-                    throw new InvalidRequestException("null is not supported inside collections");
-                if (keyBytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
-                    throw new InvalidRequestException(String.format("Map key is too long. Map keys are limited to %d bytes but %d bytes keys provided",
-                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
-                                                                    keyBytes.remaining()));
-
-                ByteBuffer valueBytes = ((Constants.Value)v).bytes;
-                if (valueBytes == null)
-                    throw new InvalidRequestException("null is not supported inside collections");
-                if (valueBytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
-                    throw new InvalidRequestException(String.format("Map value is too long. Map values are limited to %d bytes but %d bytes value provided",
-                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
-                                                                    valueBytes.remaining()));
+                if (k instanceof Term.NonTerminal || v instanceof Term.NonTerminal)
+                    allTerminal = false;
 
-                if (values.put(keyBytes, valueBytes) != null)
-                    throw new InvalidRequestException(String.format("Invalid map literal: duplicate entry for key %s", entry.left));
+                values.put(k, v);
             }
-            return new Value(values);
+            DelayedValue value = new DelayedValue(((MapType)receiver.type).keys, values);
+            return allTerminal ? value.bind(Collections.<ByteBuffer>emptyList()) : value;
         }
 
         private void validateAssignableTo(ColumnSpecification receiver) throws InvalidRequestException
@@ -179,6 +169,56 @@ public abstract class Maps
         }
     }
 
+    // See Lists.DelayedValue
+    public static class DelayedValue extends Term.NonTerminal
+    {
+        private final Comparator<ByteBuffer> comparator;
+        private final Map<Term, Term> elements;
+
+        public DelayedValue(Comparator<ByteBuffer> comparator, Map<Term, Term> elements)
+        {
+            this.comparator = comparator;
+            this.elements = elements;
+        }
+
+        public boolean containsBindMarker()
+        {
+            // False since we don't support them in collection
+            return false;
+        }
+
+        public void collectMarkerSpecification(ColumnSpecification[] boundNames)
+        {
+        }
+
+        public Value bind(List<ByteBuffer> values) throws InvalidRequestException
+        {
+            Map<ByteBuffer, ByteBuffer> buffers = new TreeMap<ByteBuffer, ByteBuffer>(comparator);
+            for (Map.Entry<Term, Term> entry : elements.entrySet())
+            {
+                // We don't support values > 64K because the serialization format encode the length as an unsigned short.
+                ByteBuffer keyBytes = entry.getKey().bindAndGet(values);
+                if (keyBytes == null)
+                    throw new InvalidRequestException("null is not supported inside collections");
+                if (keyBytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
+                    throw new InvalidRequestException(String.format("Map key is too long. Map keys are limited to %d bytes but %d bytes keys provided",
+                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
+                                                                    keyBytes.remaining()));
+
+                ByteBuffer valueBytes = entry.getValue().bindAndGet(values);
+                if (valueBytes == null)
+                    throw new InvalidRequestException("null is not supported inside collections");
+                if (valueBytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
+                    throw new InvalidRequestException(String.format("Map value is too long. Map values are limited to %d bytes but %d bytes value provided",
+                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
+                                                                    valueBytes.remaining()));
+
+                buffers.put(keyBytes, valueBytes);
+            }
+            return new Value(buffers);
+        }
+    }
+
     public static class Marker extends AbstractMarker
     {
         protected Marker(int bindIndex, ColumnSpecification receiver)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce4e4b9b/src/java/org/apache/cassandra/cql3/Sets.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/Sets.java b/src/java/org/apache/cassandra/cql3/Sets.java
index fed6625..748f269 100644
--- a/src/java/org/apache/cassandra/cql3/Sets.java
+++ b/src/java/org/apache/cassandra/cql3/Sets.java
@@ -20,6 +20,8 @@ package org.apache.cassandra.cql3;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
@@ -57,7 +59,7 @@ public abstract class Sets
             this.elements = elements;
         }
 
-        public Term.Terminal prepare(ColumnSpecification receiver) throws InvalidRequestException
+        public Term prepare(ColumnSpecification receiver) throws InvalidRequestException
         {
             validateAssignableTo(receiver);
 
@@ -66,31 +68,24 @@ public abstract class Sets
             if (receiver.type instanceof MapType && elements.isEmpty())
                 return new Maps.Value(Collections.<ByteBuffer, ByteBuffer>emptyMap());
 
+
             ColumnSpecification valueSpec = Sets.valueSpecOf(receiver);
-            Set<ByteBuffer> values = new TreeSet<ByteBuffer>(((SetType)receiver.type).elements);
+            Set<Term> values = new HashSet<Term>(elements.size());
+            boolean allTerminal = true;
             for (Term.Raw rt : elements)
             {
                 Term t = rt.prepare(valueSpec);
 
-                if (t instanceof Term.NonTerminal)
+                if (t.containsBindMarker())
                     throw new InvalidRequestException(String.format("Invalid set literal for %s: bind variables are not supported inside collection literals", receiver));
 
-                // We don't allow prepared marker in collections, nor nested collections (for the later, prepare will throw an exception)
-                assert t instanceof Constants.Value;
-                ByteBuffer bytes = ((Constants.Value)t).bytes;
-                if (bytes == null)
-                    throw new InvalidRequestException("null is not supported inside collections");
-
-                // We don't support value > 64K because the serialization format encode the length as an unsigned short.
-                if (bytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
-                    throw new InvalidRequestException(String.format("Set value is too long. Set values are limited to %d bytes but %d bytes value provided",
-                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
-                                                                    bytes.remaining()));
+                if (t instanceof Term.NonTerminal)
+                    allTerminal = false;
 
-                if (!values.add(bytes))
-                    throw new InvalidRequestException(String.format("Invalid set literal: duplicate value %s", rt));
+                values.add(t);
             }
-            return new Value(values);
+            DelayedValue value = new DelayedValue(((SetType)receiver.type).elements, values);
+            return allTerminal ? value.bind(Collections.<ByteBuffer>emptyList()) : value;
         }
 
         private void validateAssignableTo(ColumnSpecification receiver) throws InvalidRequestException
@@ -166,6 +161,50 @@ public abstract class Sets
         }
     }
 
+    // See Lists.DelayedValue
+    public static class DelayedValue extends Term.NonTerminal
+    {
+        private final Comparator<ByteBuffer> comparator;
+        private final Set<Term> elements;
+
+        public DelayedValue(Comparator<ByteBuffer> comparator, Set<Term> elements)
+        {
+            this.comparator = comparator;
+            this.elements = elements;
+        }
+
+        public boolean containsBindMarker()
+        {
+            // False since we don't support them in collection
+            return false;
+        }
+
+        public void collectMarkerSpecification(ColumnSpecification[] boundNames)
+        {
+        }
+
+        public Value bind(List<ByteBuffer> values) throws InvalidRequestException
+        {
+            Set<ByteBuffer> buffers = new TreeSet<ByteBuffer>(comparator);
+            for (Term t : elements)
+            {
+                ByteBuffer bytes = t.bindAndGet(values);
+
+                if (bytes == null)
+                    throw new InvalidRequestException("null is not supported inside collections");
+
+                // We don't support value > 64K because the serialization format encode the length as an unsigned short.
+                if (bytes.remaining() > FBUtilities.MAX_UNSIGNED_SHORT)
+                    throw new InvalidRequestException(String.format("Set value is too long. Set values are limited to %d bytes but %d bytes value provided",
+                                                                    FBUtilities.MAX_UNSIGNED_SHORT,
+                                                                    bytes.remaining()));
+
+                buffers.add(bytes);
+            }
+            return new Value(buffers);
+        }
+    }
+
     public static class Marker extends AbstractMarker
     {
         protected Marker(int bindIndex, ColumnSpecification receiver)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce4e4b9b/src/java/org/apache/cassandra/cql3/Term.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/Term.java b/src/java/org/apache/cassandra/cql3/Term.java
index 347a459..f2fd74e 100644
--- a/src/java/org/apache/cassandra/cql3/Term.java
+++ b/src/java/org/apache/cassandra/cql3/Term.java
@@ -59,6 +59,15 @@ public interface Term
     public ByteBuffer bindAndGet(List<ByteBuffer> values) throws InvalidRequestException;
 
     /**
+     * Whether or not that term contains at least one bind marker.
+     *
+     * Note that this is slightly different from being or not a NonTerminal,
+     * because calls to non pure functions will be NonTerminal (see #5616)
+     * even if they don't have bind markers.
+     */
+    public abstract boolean containsBindMarker();
+
+    /**
      * A parsed, non prepared (thus untyped) term.
      *
      * This can be one of:
@@ -83,7 +92,11 @@ public interface Term
     }
 
     /**
-     * A terminal term, i.e. one without any bind marker.
+     * A terminal term, one that can be reduced to a byte buffer directly.
+     *
+     * This includes most terms that don't have a bind marker (an exception
+     * being delayed call for non pure function that are NonTerminal even
+     * if they don't have bind markers).
      *
      * This can be only one of:
      *   - a constant value
@@ -97,6 +110,13 @@ public interface Term
         public void collectMarkerSpecification(ColumnSpecification[] boundNames) {}
         public Terminal bind(List<ByteBuffer> values) { return this; }
 
+        // While some NonTerminal may not have bind markers, no Term can be Terminal
+        // with a bind marker
+        public boolean containsBindMarker()
+        {
+            return false;
+        }
+
         /**
          * @return the serialized value of this terminal.
          */
@@ -109,12 +129,14 @@ public interface Term
     }
 
     /**
-     * A non terminal term, i.e. one that contains at least one bind marker.
+     * A non terminal term, i.e. a term that can only be reduce to a byte buffer
+     * at execution time.
      *
-     * We distinguish between the following type of NonTerminal:
+     * We have the following type of NonTerminal:
      *   - marker for a constant value
      *   - marker for a collection value (list, set, map)
      *   - a function having bind marker
+     *   - a non pure function (even if it doesn't have bind marker - see #5616)
      */
     public abstract class NonTerminal implements Term
     {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ce4e4b9b/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java b/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
index dcc3d8b..a6b86a2 100644
--- a/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
+++ b/src/java/org/apache/cassandra/cql3/functions/FunctionCall.java
@@ -72,6 +72,16 @@ public class FunctionCall extends Term.NonTerminal
         return fun.execute(buffers);
     }
 
+    public boolean containsBindMarker()
+    {
+        for (Term t : terms)
+        {
+            if (t.containsBindMarker())
+                return true;
+        }
+        return false;
+    }
+
     private static Term.Terminal makeTerminal(Function fun, ByteBuffer result) throws InvalidRequestException
     {
         if (!(fun.returnType() instanceof CollectionType))


[3/4] git commit: fix KeyCacheTest to work with globalized key cache

Posted by yu...@apache.org.
fix KeyCacheTest to work with globalized key cache


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/422d2236
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/422d2236
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/422d2236

Branch: refs/heads/cassandra-1.2
Commit: 422d2236b59b80e3cafef9fb8cad3235280e8626
Parents: ce4e4b9
Author: Yuki Morishita <yu...@apache.org>
Authored: Wed Jul 24 10:58:07 2013 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Wed Jul 24 11:00:05 2013 -0500

----------------------------------------------------------------------
 .../org/apache/cassandra/db/KeyCacheTest.java   | 75 +++++++++++++-------
 1 file changed, 49 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/422d2236/test/unit/org/apache/cassandra/db/KeyCacheTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
index 93f1fea..3458961 100644
--- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
@@ -1,6 +1,4 @@
-package org.apache.cassandra.db;
 /*
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -9,35 +7,34 @@ package org.apache.cassandra.db;
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
+package org.apache.cassandra.db;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
-import org.apache.cassandra.cache.KeyCacheKey;
-import org.apache.cassandra.db.filter.QueryFilter;
-import org.apache.cassandra.service.CacheService;
-import org.apache.cassandra.thrift.ColumnParent;
-
 import org.junit.AfterClass;
 import org.junit.Test;
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
-import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.cache.KeyCacheKey;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.filter.QueryFilter;
+import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.thrift.ColumnParent;
 import org.apache.cassandra.utils.ByteBufferUtil;
+
 import static junit.framework.Assert.assertEquals;
 
 public class KeyCacheTest extends SchemaLoader
@@ -61,7 +58,7 @@ public class KeyCacheTest extends SchemaLoader
 
         // empty the cache
         CacheService.instance.invalidateKeyCache();
-        assert CacheService.instance.keyCache.size() == 0;
+        assertKeyCacheSize(0, TABLE1, COLUMN_FAMILY2);
 
         // insert data and force to disk
         insertData(TABLE1, COLUMN_FAMILY2, 0, 100);
@@ -69,20 +66,37 @@ public class KeyCacheTest extends SchemaLoader
 
         // populate the cache
         readData(TABLE1, COLUMN_FAMILY2, 0, 100);
-        assertEquals(100, CacheService.instance.keyCache.size());
+        assertKeyCacheSize(100, TABLE1, COLUMN_FAMILY2);
 
         // really? our caches don't implement the map interface? (hence no .addAll)
         Map<KeyCacheKey, RowIndexEntry> savedMap = new HashMap<KeyCacheKey, RowIndexEntry>();
         for (KeyCacheKey k : CacheService.instance.keyCache.getKeySet())
         {
-            savedMap.put(k, CacheService.instance.keyCache.get(k));
+            if (k.desc.ksname.equals(TABLE1) && k.desc.cfname.equals(COLUMN_FAMILY2))
+                savedMap.put(k, CacheService.instance.keyCache.get(k));
         }
 
         // force the cache to disk
         CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();
 
         CacheService.instance.invalidateKeyCache();
-        assert CacheService.instance.keyCache.size() == 0;
+        assertKeyCacheSize(0, TABLE1, COLUMN_FAMILY2);
+
+        CacheService.instance.keyCache.loadSaved(store);
+        assertKeyCacheSize(savedMap.size(), TABLE1, COLUMN_FAMILY2);
+
+        // probably it's better to add equals/hashCode to RowIndexEntry...
+        for (Map.Entry<KeyCacheKey, RowIndexEntry> entry : savedMap.entrySet())
+        {
+            RowIndexEntry expected = entry.getValue();
+            RowIndexEntry actual = CacheService.instance.keyCache.get(entry.getKey());
+            assertEquals(expected.position, actual.position);
+            assertEquals(expected.columnsIndex(), actual.columnsIndex());
+            if (expected.isIndexed())
+            {
+                assertEquals(expected.deletionTime(), actual.deletionTime());
+            }
+        }
     }
 
     @Test
@@ -97,8 +111,7 @@ public class KeyCacheTest extends SchemaLoader
         CacheService.instance.invalidateKeyCache();
 
         // KeyCache should start at size 0 if we're caching X% of zero data.
-        int keyCacheSize = CacheService.instance.keyCache.size();
-        assert keyCacheSize == 0 : keyCacheSize;
+        assertKeyCacheSize(0, TABLE1, COLUMN_FAMILY1);
 
         DecoratedKey key1 = Util.dk("key1");
         DecoratedKey key2 = Util.dk("key2");
@@ -130,13 +143,12 @@ public class KeyCacheTest extends SchemaLoader
                                                        false,
                                                        10));
 
-        assert CacheService.instance.keyCache.size() == 2;
+        assertKeyCacheSize(2, TABLE1, COLUMN_FAMILY1);
 
         Util.compactAll(cfs).get();
-        keyCacheSize = CacheService.instance.keyCache.size();
         // after compaction cache should have entries for
         // new SSTables, if we had 2 keys in cache previously it should become 4
-        assert keyCacheSize == 4 : keyCacheSize;
+        assertKeyCacheSize(4, TABLE1, COLUMN_FAMILY1);
 
         // re-read same keys to verify that key cache didn't grow further
         cfs.getColumnFamily(QueryFilter.getSliceFilter(key1,
@@ -153,6 +165,17 @@ public class KeyCacheTest extends SchemaLoader
                                                        false,
                                                        10));
 
-        assert CacheService.instance.keyCache.size() == 4;
+        assertKeyCacheSize(4, TABLE1, COLUMN_FAMILY1);
+    }
+
+    private void assertKeyCacheSize(int expected, String keyspace, String columnFamily)
+    {
+        int size = 0;
+        for (KeyCacheKey k : CacheService.instance.keyCache.getKeySet())
+        {
+            if (k.desc.ksname.equals(keyspace) && k.desc.cfname.equals(columnFamily))
+                size++;
+        }
+        assertEquals(expected, size);
     }
 }


[4/4] git commit: Merge branch 'cassandra-1.2' into trunk

Posted by yu...@apache.org.
Merge branch 'cassandra-1.2' into trunk

Conflicts:
	test/unit/org/apache/cassandra/db/KeyCacheTest.java


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/813577e3
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/813577e3
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/813577e3

Branch: refs/heads/trunk
Commit: 813577e32b7e9a40ff1e71875d019eda44c84586
Parents: 371e7bf 422d223
Author: Yuki Morishita <yu...@apache.org>
Authored: Wed Jul 24 11:07:31 2013 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Wed Jul 24 11:07:31 2013 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../apache/cassandra/cql3/AbstractMarker.java   |  5 ++
 src/java/org/apache/cassandra/cql3/Lists.java   | 76 ++++++++++++++----
 src/java/org/apache/cassandra/cql3/Maps.java    | 84 +++++++++++++++-----
 src/java/org/apache/cassandra/cql3/Sets.java    | 73 +++++++++++++----
 src/java/org/apache/cassandra/cql3/Term.java    | 28 ++++++-
 .../cassandra/cql3/functions/FunctionCall.java  | 10 +++
 .../org/apache/cassandra/db/KeyCacheTest.java   | 70 ++++++++++------
 8 files changed, 266 insertions(+), 81 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/813577e3/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/813577e3/src/java/org/apache/cassandra/cql3/Lists.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/813577e3/src/java/org/apache/cassandra/cql3/Maps.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/813577e3/src/java/org/apache/cassandra/cql3/Sets.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/813577e3/test/unit/org/apache/cassandra/db/KeyCacheTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/KeyCacheTest.java
index e8e524d,3458961..10bcedd
--- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
@@@ -35,8 -27,14 +27,12 @@@ import org.junit.Test
  
  import org.apache.cassandra.SchemaLoader;
  import org.apache.cassandra.Util;
+ import org.apache.cassandra.cache.KeyCacheKey;
  import org.apache.cassandra.db.compaction.CompactionManager;
+ import org.apache.cassandra.db.filter.QueryFilter;
 -import org.apache.cassandra.db.filter.QueryPath;
+ import org.apache.cassandra.service.CacheService;
 -import org.apache.cassandra.thrift.ColumnParent;
  import org.apache.cassandra.utils.ByteBufferUtil;
+ 
  import static junit.framework.Assert.assertEquals;
  
  public class KeyCacheTest extends SchemaLoader
@@@ -60,28 -58,45 +56,45 @@@
  
          // empty the cache
          CacheService.instance.invalidateKeyCache();
-         assert CacheService.instance.keyCache.size() == 0;
 -        assertKeyCacheSize(0, TABLE1, COLUMN_FAMILY2);
++        assertKeyCacheSize(0, KEYSPACE1, COLUMN_FAMILY2);
  
          // insert data and force to disk
 -        insertData(TABLE1, COLUMN_FAMILY2, 0, 100);
 +        insertData(KEYSPACE1, COLUMN_FAMILY2, 0, 100);
          store.forceBlockingFlush();
  
          // populate the cache
 -        readData(TABLE1, COLUMN_FAMILY2, 0, 100);
 -        assertKeyCacheSize(100, TABLE1, COLUMN_FAMILY2);
 +        readData(KEYSPACE1, COLUMN_FAMILY2, 0, 100);
-         assertEquals(100, CacheService.instance.keyCache.size());
++        assertKeyCacheSize(100, KEYSPACE1, COLUMN_FAMILY2);
  
          // really? our caches don't implement the map interface? (hence no .addAll)
          Map<KeyCacheKey, RowIndexEntry> savedMap = new HashMap<KeyCacheKey, RowIndexEntry>();
          for (KeyCacheKey k : CacheService.instance.keyCache.getKeySet())
          {
-             savedMap.put(k, CacheService.instance.keyCache.get(k));
 -            if (k.desc.ksname.equals(TABLE1) && k.desc.cfname.equals(COLUMN_FAMILY2))
++            if (k.desc.ksname.equals(KEYSPACE1) && k.desc.cfname.equals(COLUMN_FAMILY2))
+                 savedMap.put(k, CacheService.instance.keyCache.get(k));
          }
  
          // force the cache to disk
          CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();
  
          CacheService.instance.invalidateKeyCache();
-         assert CacheService.instance.keyCache.size() == 0;
 -        assertKeyCacheSize(0, TABLE1, COLUMN_FAMILY2);
++        assertKeyCacheSize(0, KEYSPACE1, COLUMN_FAMILY2);
+ 
+         CacheService.instance.keyCache.loadSaved(store);
 -        assertKeyCacheSize(savedMap.size(), TABLE1, COLUMN_FAMILY2);
++        assertKeyCacheSize(savedMap.size(), KEYSPACE1, COLUMN_FAMILY2);
+ 
+         // probably it's better to add equals/hashCode to RowIndexEntry...
+         for (Map.Entry<KeyCacheKey, RowIndexEntry> entry : savedMap.entrySet())
+         {
+             RowIndexEntry expected = entry.getValue();
+             RowIndexEntry actual = CacheService.instance.keyCache.get(entry.getKey());
+             assertEquals(expected.position, actual.position);
+             assertEquals(expected.columnsIndex(), actual.columnsIndex());
+             if (expected.isIndexed())
+             {
+                 assertEquals(expected.deletionTime(), actual.deletionTime());
+             }
+         }
      }
  
      @Test
@@@ -96,7 -111,7 +109,7 @@@
          CacheService.instance.invalidateKeyCache();
  
          // KeyCache should start at size 0 if we're caching X% of zero data.
-         assertEquals(0, CacheService.instance.keyCache.size());
 -        assertKeyCacheSize(0, TABLE1, COLUMN_FAMILY1);
++        assertKeyCacheSize(0, KEYSPACE1, COLUMN_FAMILY1);
  
          DecoratedKey key1 = Util.dk("key1");
          DecoratedKey key2 = Util.dk("key2");
@@@ -127,15 -141,14 +140,15 @@@
                                                         ByteBufferUtil.EMPTY_BYTE_BUFFER,
                                                         ByteBufferUtil.EMPTY_BYTE_BUFFER,
                                                         false,
 -                                                       10));
 +                                                       10,
 +                                                       System.currentTimeMillis()));
  
-         assertEquals(2, CacheService.instance.keyCache.size());
 -        assertKeyCacheSize(2, TABLE1, COLUMN_FAMILY1);
++        assertKeyCacheSize(2, KEYSPACE1, COLUMN_FAMILY1);
  
          Util.compactAll(cfs).get();
          // after compaction cache should have entries for
          // new SSTables, if we had 2 keys in cache previously it should become 4
-         assertEquals(4, CacheService.instance.keyCache.size());
 -        assertKeyCacheSize(4, TABLE1, COLUMN_FAMILY1);
++        assertKeyCacheSize(4, KEYSPACE1, COLUMN_FAMILY1);
  
          // re-read same keys to verify that key cache didn't grow further
          cfs.getColumnFamily(QueryFilter.getSliceFilter(key1,
@@@ -151,9 -163,19 +164,20 @@@
                                                         ByteBufferUtil.EMPTY_BYTE_BUFFER,
                                                         ByteBufferUtil.EMPTY_BYTE_BUFFER,
                                                         false,
 -                                                       10));
 +                                                       10,
 +                                                       System.currentTimeMillis()));
  
-         assert CacheService.instance.keyCache.size() == 4;
 -        assertKeyCacheSize(4, TABLE1, COLUMN_FAMILY1);
++        assertKeyCacheSize(4, KEYSPACE1, COLUMN_FAMILY1);
+     }
+ 
+     private void assertKeyCacheSize(int expected, String keyspace, String columnFamily)
+     {
+         int size = 0;
+         for (KeyCacheKey k : CacheService.instance.keyCache.getKeySet())
+         {
+             if (k.desc.ksname.equals(keyspace) && k.desc.cfname.equals(columnFamily))
+                 size++;
+         }
+         assertEquals(expected, size);
      }
  }


[2/4] git commit: fix KeyCacheTest to work with globalized key cache

Posted by yu...@apache.org.
fix KeyCacheTest to work with globalized key cache


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/422d2236
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/422d2236
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/422d2236

Branch: refs/heads/trunk
Commit: 422d2236b59b80e3cafef9fb8cad3235280e8626
Parents: ce4e4b9
Author: Yuki Morishita <yu...@apache.org>
Authored: Wed Jul 24 10:58:07 2013 -0500
Committer: Yuki Morishita <yu...@apache.org>
Committed: Wed Jul 24 11:00:05 2013 -0500

----------------------------------------------------------------------
 .../org/apache/cassandra/db/KeyCacheTest.java   | 75 +++++++++++++-------
 1 file changed, 49 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/422d2236/test/unit/org/apache/cassandra/db/KeyCacheTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
index 93f1fea..3458961 100644
--- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
@@ -1,6 +1,4 @@
-package org.apache.cassandra.db;
 /*
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -9,35 +7,34 @@ package org.apache.cassandra.db;
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
+package org.apache.cassandra.db;
 
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
-import org.apache.cassandra.cache.KeyCacheKey;
-import org.apache.cassandra.db.filter.QueryFilter;
-import org.apache.cassandra.service.CacheService;
-import org.apache.cassandra.thrift.ColumnParent;
-
 import org.junit.AfterClass;
 import org.junit.Test;
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
-import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.cache.KeyCacheKey;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.filter.QueryFilter;
+import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.thrift.ColumnParent;
 import org.apache.cassandra.utils.ByteBufferUtil;
+
 import static junit.framework.Assert.assertEquals;
 
 public class KeyCacheTest extends SchemaLoader
@@ -61,7 +58,7 @@ public class KeyCacheTest extends SchemaLoader
 
         // empty the cache
         CacheService.instance.invalidateKeyCache();
-        assert CacheService.instance.keyCache.size() == 0;
+        assertKeyCacheSize(0, TABLE1, COLUMN_FAMILY2);
 
         // insert data and force to disk
         insertData(TABLE1, COLUMN_FAMILY2, 0, 100);
@@ -69,20 +66,37 @@ public class KeyCacheTest extends SchemaLoader
 
         // populate the cache
         readData(TABLE1, COLUMN_FAMILY2, 0, 100);
-        assertEquals(100, CacheService.instance.keyCache.size());
+        assertKeyCacheSize(100, TABLE1, COLUMN_FAMILY2);
 
         // really? our caches don't implement the map interface? (hence no .addAll)
         Map<KeyCacheKey, RowIndexEntry> savedMap = new HashMap<KeyCacheKey, RowIndexEntry>();
         for (KeyCacheKey k : CacheService.instance.keyCache.getKeySet())
         {
-            savedMap.put(k, CacheService.instance.keyCache.get(k));
+            if (k.desc.ksname.equals(TABLE1) && k.desc.cfname.equals(COLUMN_FAMILY2))
+                savedMap.put(k, CacheService.instance.keyCache.get(k));
         }
 
         // force the cache to disk
         CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();
 
         CacheService.instance.invalidateKeyCache();
-        assert CacheService.instance.keyCache.size() == 0;
+        assertKeyCacheSize(0, TABLE1, COLUMN_FAMILY2);
+
+        CacheService.instance.keyCache.loadSaved(store);
+        assertKeyCacheSize(savedMap.size(), TABLE1, COLUMN_FAMILY2);
+
+        // probably it's better to add equals/hashCode to RowIndexEntry...
+        for (Map.Entry<KeyCacheKey, RowIndexEntry> entry : savedMap.entrySet())
+        {
+            RowIndexEntry expected = entry.getValue();
+            RowIndexEntry actual = CacheService.instance.keyCache.get(entry.getKey());
+            assertEquals(expected.position, actual.position);
+            assertEquals(expected.columnsIndex(), actual.columnsIndex());
+            if (expected.isIndexed())
+            {
+                assertEquals(expected.deletionTime(), actual.deletionTime());
+            }
+        }
     }
 
     @Test
@@ -97,8 +111,7 @@ public class KeyCacheTest extends SchemaLoader
         CacheService.instance.invalidateKeyCache();
 
         // KeyCache should start at size 0 if we're caching X% of zero data.
-        int keyCacheSize = CacheService.instance.keyCache.size();
-        assert keyCacheSize == 0 : keyCacheSize;
+        assertKeyCacheSize(0, TABLE1, COLUMN_FAMILY1);
 
         DecoratedKey key1 = Util.dk("key1");
         DecoratedKey key2 = Util.dk("key2");
@@ -130,13 +143,12 @@ public class KeyCacheTest extends SchemaLoader
                                                        false,
                                                        10));
 
-        assert CacheService.instance.keyCache.size() == 2;
+        assertKeyCacheSize(2, TABLE1, COLUMN_FAMILY1);
 
         Util.compactAll(cfs).get();
-        keyCacheSize = CacheService.instance.keyCache.size();
         // after compaction cache should have entries for
         // new SSTables, if we had 2 keys in cache previously it should become 4
-        assert keyCacheSize == 4 : keyCacheSize;
+        assertKeyCacheSize(4, TABLE1, COLUMN_FAMILY1);
 
         // re-read same keys to verify that key cache didn't grow further
         cfs.getColumnFamily(QueryFilter.getSliceFilter(key1,
@@ -153,6 +165,17 @@ public class KeyCacheTest extends SchemaLoader
                                                        false,
                                                        10));
 
-        assert CacheService.instance.keyCache.size() == 4;
+        assertKeyCacheSize(4, TABLE1, COLUMN_FAMILY1);
+    }
+
+    private void assertKeyCacheSize(int expected, String keyspace, String columnFamily)
+    {
+        int size = 0;
+        for (KeyCacheKey k : CacheService.instance.keyCache.getKeySet())
+        {
+            if (k.desc.ksname.equals(keyspace) && k.desc.cfname.equals(columnFamily))
+                size++;
+        }
+        assertEquals(expected, size);
     }
 }