You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sn...@apache.org on 2017/01/05 21:30:37 UTC
[01/10] cassandra git commit: Remove support for non-JavaScript UDFs
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.2 0cf0f67df -> 6f360b6d2
refs/heads/cassandra-3.0 c0765ed82 -> 6e716c6da
refs/heads/cassandra-3.X 5f3b35981 -> eca51a034
refs/heads/trunk d384e781d -> 2cabf9c49
Remove support for non-JavaScript UDFs
patch by Robert Stupp; reviewed by Tyler Hobbs for CASSANDRA-12883
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6f360b6d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6f360b6d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6f360b6d
Branch: refs/heads/cassandra-2.2
Commit: 6f360b6d239a7b3d3b625e1bcefcaeb2f6b2be1d
Parents: 0cf0f67
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu Jan 5 22:18:37 2017 +0100
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu Jan 5 22:19:50 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 16 ++++++++++++++++
.../cassandra/cql3/functions/ScriptBasedUDF.java | 5 +++++
3 files changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d31ffc8..b41313d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.9
+ * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
* Fix DynamicEndpointSnitch noop in multi-datacenter situations (CASSANDRA-13074)
* cqlsh copy-from: encode column names to avoid primary key parsing errors (CASSANDRA-12909)
* Temporarily fix bug that creates commit log when running offline tools (CASSANDRA-8616)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index 0a3ab36..37949a1 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -13,6 +13,19 @@ restore snapshots created with the previous major version using the
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
+2.2.9
+=====
+
+Deprecation
+-----------
+
+(See note about the new feature User-Defined-Functions in 2.2.0.)
+
+Since the security manager added in 3.0 only allows Java and JavaScript
+UDFs to be run, UDFs for other languages are deprecated and support for
+non-Java and non-JavaScript UDFs is deprecated in 2.2 and has been removed
+in version 3.0.11.
+
2.2.8
=====
@@ -230,6 +243,9 @@ New features
3.0. This will inherently be backwards-incompatible with any 2.2
UDF that perform insecure operations such as opening a socket or
writing to the filesystem.
+
+ Per the previous note about adding a security manager in 3.0, this security manager
+ means that non JavaScipt UDF's won't run, there for their use is deprecated.
************************************************************************
- Row-cache is now fully off-heap.
- jemalloc is now automatically preloaded and used on Linux and OS-X if
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
index e55d450..2d46934 100644
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
@@ -73,6 +73,11 @@ public class ScriptBasedUDF extends UDFunction
{
super(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
+ if (!"JavaScript".equalsIgnoreCase(language))
+ logger.warn("Support for UDFs using '" + language + "' has been deprecated and removed in 3.0. If '" +
+ language + "' actually is JavaScript, change the language used in CREATE/ALTER FUNCTION to " +
+ "'javascript'.");
+
Compilable scriptEngine = scriptEngines.get(language);
if (scriptEngine == null)
throw new InvalidRequestException(String.format("Invalid language '%s' for function '%s'", language, name));
[03/10] cassandra git commit: Remove support for non-JavaScript UDFs
Posted by sn...@apache.org.
Remove support for non-JavaScript UDFs
patch by Robert Stupp; reviewed by Tyler Hobbs for CASSANDRA-12883
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6f360b6d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6f360b6d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6f360b6d
Branch: refs/heads/cassandra-3.X
Commit: 6f360b6d239a7b3d3b625e1bcefcaeb2f6b2be1d
Parents: 0cf0f67
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu Jan 5 22:18:37 2017 +0100
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu Jan 5 22:19:50 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 16 ++++++++++++++++
.../cassandra/cql3/functions/ScriptBasedUDF.java | 5 +++++
3 files changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d31ffc8..b41313d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.9
+ * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
* Fix DynamicEndpointSnitch noop in multi-datacenter situations (CASSANDRA-13074)
* cqlsh copy-from: encode column names to avoid primary key parsing errors (CASSANDRA-12909)
* Temporarily fix bug that creates commit log when running offline tools (CASSANDRA-8616)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index 0a3ab36..37949a1 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -13,6 +13,19 @@ restore snapshots created with the previous major version using the
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
+2.2.9
+=====
+
+Deprecation
+-----------
+
+(See note about the new feature User-Defined-Functions in 2.2.0.)
+
+Since the security manager added in 3.0 only allows Java and JavaScript
+UDFs to be run, UDFs for other languages are deprecated and support for
+non-Java and non-JavaScript UDFs is deprecated in 2.2 and has been removed
+in version 3.0.11.
+
2.2.8
=====
@@ -230,6 +243,9 @@ New features
3.0. This will inherently be backwards-incompatible with any 2.2
UDF that perform insecure operations such as opening a socket or
writing to the filesystem.
+
+ Per the previous note about adding a security manager in 3.0, this security manager
+ means that non JavaScipt UDF's won't run, there for their use is deprecated.
************************************************************************
- Row-cache is now fully off-heap.
- jemalloc is now automatically preloaded and used on Linux and OS-X if
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
index e55d450..2d46934 100644
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
@@ -73,6 +73,11 @@ public class ScriptBasedUDF extends UDFunction
{
super(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
+ if (!"JavaScript".equalsIgnoreCase(language))
+ logger.warn("Support for UDFs using '" + language + "' has been deprecated and removed in 3.0. If '" +
+ language + "' actually is JavaScript, change the language used in CREATE/ALTER FUNCTION to " +
+ "'javascript'.");
+
Compilable scriptEngine = scriptEngines.get(language);
if (scriptEngine == null)
throw new InvalidRequestException(String.format("Invalid language '%s' for function '%s'", language, name));
[06/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by sn...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6e716c6d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6e716c6d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6e716c6d
Branch: refs/heads/cassandra-3.0
Commit: 6e716c6da41900950e32a5549b3bb1e858ecad18
Parents: c0765ed 6f360b6
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu Jan 5 22:20:31 2017 +0100
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu Jan 5 22:20:31 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 3 ++
doc/cql3/CQL.textile | 2 +-
lib/jsr223/clojure/README.txt | 8 ---
lib/jsr223/groovy/README.txt | 35 -------------
lib/jsr223/jaskell/README.txt | 5 --
lib/jsr223/jruby/README.txt | 54 --------------------
lib/jsr223/jython/README.txt | 33 ------------
lib/jsr223/scala/README.txt | 37 --------------
.../cql3/functions/ScriptBasedUDFunction.java | 30 +++++------
.../cql3/validation/entities/UFScriptTest.java | 3 +-
.../validation/entities/UFSecurityTest.java | 12 ++++-
12 files changed, 29 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 666a771,b41313d..77a310d
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,23 -1,5 +1,24 @@@
-2.2.9
+3.0.11
+ * Fixed flacky SSTableRewriterTest: check file counts before calling validateCFS (CASSANDRA-12348)
+ * Fix deserialization of 2.x DeletedCells (CASSANDRA-12620)
+ * Add parent repair session id to anticompaction log message (CASSANDRA-12186)
+ * Improve contention handling on failure to acquire MV lock for streaming and hints (CASSANDRA-12905)
+ * Fix DELETE and UPDATE queries with empty IN restrictions (CASSANDRA-12829)
+ * Mark MVs as built after successful bootstrap (CASSANDRA-12984)
+ * Estimated TS drop-time histogram updated with Cell.NO_DELETION_TIME (CASSANDRA-13040)
+ * Nodetool compactionstats fails with NullPointerException (CASSANDRA-13021)
+ * Thread local pools never cleaned up (CASSANDRA-13033)
+ * Set RPC_READY to false when draining or if a node is marked as shutdown (CASSANDRA-12781)
+ * CQL often queries static columns unnecessarily (CASSANDRA-12768)
+ * Make sure sstables only get committed when it's safe to discard commit log records (CASSANDRA-12956)
+ * Reject default_time_to_live option when creating or altering MVs (CASSANDRA-12868)
+ * Nodetool should use a more sane max heap size (CASSANDRA-12739)
+ * LocalToken ensures token values are cloned on heap (CASSANDRA-12651)
+ * AnticompactionRequestSerializer serializedSize is incorrect (CASSANDRA-12934)
+ * Prevent reloading of logback.xml from UDF sandbox (CASSANDRA-12535)
+ * Reenable HeapPool (CASSANDRA-12900)
+Merged from 2.2:
+ * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
* Fix DynamicEndpointSnitch noop in multi-datacenter situations (CASSANDRA-13074)
* cqlsh copy-from: encode column names to avoid primary key parsing errors (CASSANDRA-12909)
* Temporarily fix bug that creates commit log when running offline tools (CASSANDRA-8616)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/NEWS.txt
----------------------------------------------------------------------
diff --cc NEWS.txt
index 32b5084,37949a1..b4e6551
--- a/NEWS.txt
+++ b/NEWS.txt
@@@ -13,77 -13,20 +13,80 @@@ restore snapshots created with the prev
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
-2.2.9
+3.0.11
=====
+Upgrading
+---------
+ - Nothing specific to this release, but please see previous versions upgrading section,
+ especially if you are upgrading from 2.2.
+ - Specifying the default_time_to_live option when creating or altering a
+ materialized view was erroneously accepted (and ignored). It is now
+ properly rejected.
++ - Only Java and JavaScript are now supported UDF languages.
++ The sandbox in 3.0 already prevented the use of script languages except Java
++ and JavaScript.
+
+3.0.10
+=====
+
+Upgrading
+---------
+ - memtable_allocation_type: offheap_buffers is no longer allowed to be specified in the 3.0 series.
+ This was an oversight that can cause segfaults. Offheap was re-introduced in 3.4 see CASSANDRA-11039
+ and CASSANDRA-9472 for details.
+
+3.0.9
+=====
+
+Upgrading
+---------
+ - The ReversedType behaviour has been corrected for clustering columns of
+ BYTES type containing empty value. Scrub should be run on the existing
+ SSTables containing a descending clustering column of BYTES type to correct
+ their ordering. See CASSANDRA-12127 for more details.
+
+3.0.8
+=====
+
+Upgrading
+---------
+ - Ec2MultiRegionSnitch will no longer automatically set broadcast_rpc_address
+ to the public instance IP if this property is defined on cassandra.yaml.
+
+3.0.7
+=====
+
+Upgrading
+---------
+ - A maximum size for SSTables values has been introduced, to prevent out of memory
+ exceptions when reading corrupt SSTables. This maximum size can be set via
+ max_value_size_in_mb in cassandra.yaml. The default is 256MB, which matches the default
+ value of native_transport_max_frame_size_in_mb. SSTables will be considered corrupt if
+ they contain values whose size exceeds this limit. See CASSANDRA-9530 for more details.
+
Deprecation
-----------
+ - DateTieredCompactionStrategy has been deprecated - new tables should use
+ TimeWindowCompactionStrategy. Note that migrating an existing DTCS-table to TWCS might
+ cause increased compaction load for a while after the migration so make sure you run
+ tests before migrating. Read CASSANDRA-9666 for background on this.
-(See note about the new feature User-Defined-Functions in 2.2.0.)
+New features
+------------
+ - TimeWindowCompactionStrategy has been added. This has proven to be a better approach
+ to time series compaction and new tables should use this instead of DTCS. See
+ CASSANDRA-9666 for details.
-Since the security manager added in 3.0 only allows Java and JavaScript
-UDFs to be run, UDFs for other languages are deprecated and support for
-non-Java and non-JavaScript UDFs is deprecated in 2.2 and has been removed
-in version 3.0.11.
+3.0.6
+=====
+
+New features
+------------
+ - JSON timestamps are now in UTC and contain the timezone information, see
+ CASSANDRA-11137 for more details.
-2.2.8
+3.0.5
=====
Upgrading
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/doc/cql3/CQL.textile
----------------------------------------------------------------------
diff --cc doc/cql3/CQL.textile
index 2a37452,af584d0..2544878
--- a/doc/cql3/CQL.textile
+++ b/doc/cql3/CQL.textile
@@@ -1977,7 -1910,7 +1977,7 @@@ SELECT AVG(players) FROM plays
h2(#udfs). User-Defined Functions
--User-defined functions allow execution of user-provided code in Cassandra. By default, Cassandra supports defining functions in _Java_ and _JavaScript_. Support for other JSR 223 compliant scripting languages (such as Python, Ruby, and Scala) can be added by adding a JAR to the classpath.
++User-defined functions allow execution of user-provided code in Cassandra. By default, Cassandra supports defining functions in _Java_ and _JavaScript_. Support for other JSR 223 compliant scripting languages (such as Python, Ruby, and Scala) has been removed in 3.0.11.
UDFs are part of the Cassandra schema. As such, they are automatically propagated to all nodes in the cluster.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/clojure/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/clojure/README.txt
index 7ed7551,7ed7551..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/clojure/README.txt
+++ /dev/null
@@@ -1,8 -1,8 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Unfortunately the JSR-223 support provided by the project https://github.com/ato/clojure-jsr223
--and the related ones do not provide compileable script support.
--
--The JSR-223 javax.script.Compilable implementation takes source file names or readers but not script sources
--as all other JSR-223 implementations do.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/groovy/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/groovy/README.txt
index 09fef93,09fef93..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/groovy/README.txt
+++ /dev/null
@@@ -1,35 -1,35 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable Groovy
--
--Tested with version 2.3.6
--
--Installation
--------------
--
--1. Download Groovy binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy the jar groovy-all-2.3.6-indy.jar from the Groovy embeddable directory to $CASSANDRA_HOME/lib/jsr223/groovy
-- "indy" means "invokedynamic" and is a JVM instruction for scripting languages new to Java 7.
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 10:49:45 Found scripting engine Groovy Scripting Engine 2.0 - Groovy 2.3.6 - language names: [groovy, Groovy]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--Smoke Test
------------
--
--To test Groovy functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE groovy AS 'return "foo";' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language groovy for 'foobar'"
--Groovy for Apache Cassandra has not been installed correctly.
--
--Notes / Java7 invokedynamic
-----------------------------
--
--Groovy provides jars that support invokedynamic bytecode instruction. These jars are whose ending with
--"-indy.jar".
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jaskell/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/jaskell/README.txt
index 53e942e,53e942e..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/jaskell/README.txt
+++ /dev/null
@@@ -1,5 -1,5 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Unfortunately Jaskell JSR-223 support is quite old and the Jaskell engine seems to be quite
--unsupported. If you find a solution, please open a ticket at Apache Cassandra JIRA.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jruby/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/jruby/README.txt
index cbc12dc,cbc12dc..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/jruby/README.txt
+++ /dev/null
@@@ -1,54 -1,54 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable JRuby
--
--Tested with version 1.7.15
--
--Installation
--------------
--
--1. Download JRuby binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy everything from the JRuby lib directory to $CASSANDRA_HOME/lib/jsr223/jruby
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 10:29:03 Found scripting engine JSR 223 JRuby Engine 1.7.15 - ruby jruby 1.7.15 - language names: [ruby, jruby]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--
--Smoke Test
------------
--
--To test JRuby functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE ruby AS 'return "foo";' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language ruby for 'foobar'"
--JRuby for Apache Cassandra has not been installed correctly.
--
--
--Ruby require/include
----------------------
--
--You can use Ruby require and include in your scripts as in the following example:
--
--
--CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE ruby AS '
--require "bigdecimal"
--require "bigdecimal/math"
--
--include BigMath
--
--a = BigDecimal((PI(100)/2).to_s)
--
--return "foo " + a.to_s;
--' ;
--
--
--Notes / Java7 invokedynamic
-----------------------------
--
--See JRuby wiki pages https://github.com/jruby/jruby/wiki/ConfiguringJRuby and
--https://github.com/jruby/jruby/wiki/PerformanceTuning for more information and optimization tips.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jython/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/jython/README.txt
index bef3c83,bef3c83..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/jython/README.txt
+++ /dev/null
@@@ -1,33 -1,33 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable Jython
--
--Tested with version 2.3.5
--
--Installation
--------------
--
--1. Download Jython binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy the jar jython.jar from the Jython directory to $CASSANDRA_HOME/lib/jsr223/jython
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 10:58:18 Found scripting engine jython 2.5.3 - python 2.5 - language names: [python, jython]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--Smoke Test
------------
--
--To test Jython functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE python AS '''foo''' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language python for 'foobar'"
--Jython for Apache Cassandra has not been installed correctly.
--
--Notes / Java7 invokedynamic
-----------------------------
--
--Jython currently targets Java6 only. They want to switch to Java7 + invokedynamic in Jython 3.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/scala/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/scala/README.txt
index 7f5d6fe,7f5d6fe..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/scala/README.txt
+++ /dev/null
@@@ -1,37 -1,37 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable Scala
--
--Tested with version 2.11.2
--
--Installation
--------------
--
--1. Download Scala binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy the following jars from the Scala lib directory to $CASSANDRA_HOME/lib/jsr223/scala
-- scala-compiler.jar
-- scala-library.jar
-- scala-reflect.jar
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 11:42:35 Found scripting engine Scala Interpreter 1.0 - Scala version 2.11.2 - language names: [scala]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--Smoke Test
------------
--
--To test Scala functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE scala AS 'return "foo";' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language scala for 'foobar'"
--Scala for Apache Cassandra has not been installed correctly.
--
--Notes / Java7 invokedynamic
-----------------------------
--
--Scala 2.10 has Java6 support only. 2.11 has experimental invokedynamic support (use at your own risk!).
--2.12 introduces an upgrade directly to Java8 - see https://stackoverflow.com/questions/14285894/advantages-of-scala-emitting-bytecode-for-the-jvm-1-7
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
index 8743a20,0000000..47deafa
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
@@@ -1,246 -1,0 +1,240 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.*;
+import java.nio.ByteBuffer;
+import java.security.*;
+import java.security.cert.Certificate;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import javax.script.*;
+
++import jdk.nashorn.api.scripting.ClassFilter;
++import jdk.nashorn.api.scripting.NashornScriptEngine;
++import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+final class ScriptBasedUDFunction extends UDFunction
+{
- static final Map<String, Compilable> scriptEngines = new HashMap<>();
-
+ private static final ProtectionDomain protectionDomain;
+ private static final AccessControlContext accessControlContext;
+
+ //
+ // For scripted UDFs we have to rely on the security mechanisms of the scripting engine and
+ // SecurityManager - especially SecurityManager.checkPackageAccess(). Unlike Java-UDFs, strict checking
+ // of class access via the UDF class loader is not possible, since e.g. Nashorn builds its own class loader
+ // (jdk.nashorn.internal.runtime.ScriptLoader / jdk.nashorn.internal.runtime.NashornLoader) configured with
+ // a system class loader.
+ //
+ private static final String[] allowedPackagesArray =
+ {
+ // following required by jdk.nashorn.internal.objects.Global.initJavaAccess()
+ "",
+ "com",
+ "edu",
+ "java",
+ "javax",
+ "javafx",
+ "org",
+ // following required by Nashorn runtime
+ "java.lang",
+ "java.lang.invoke",
+ "java.lang.reflect",
+ "java.nio.charset",
+ "java.util",
+ "java.util.concurrent",
+ "javax.script",
+ "sun.reflect",
+ "jdk.internal.org.objectweb.asm.commons",
+ "jdk.nashorn.internal.runtime",
+ "jdk.nashorn.internal.runtime.linker",
+ // following required by Java Driver
+ "java.math",
+ "java.nio",
+ "java.text",
+ "com.google.common.base",
+ "com.google.common.collect",
+ "com.google.common.reflect",
+ // following required by UDF
+ "com.datastax.driver.core",
+ "com.datastax.driver.core.utils"
+ };
+
+ // use a JVM standard ExecutorService as DebuggableThreadPoolExecutor references internal
+ // classes, which triggers AccessControlException from the UDF sandbox
+ private static final UDFExecutorService executor =
+ new UDFExecutorService(new NamedThreadFactory("UserDefinedScriptFunctions",
+ Thread.MIN_PRIORITY,
+ udfClassLoader,
+ new SecurityThreadGroup("UserDefinedScriptFunctions",
+ Collections.unmodifiableSet(new HashSet<>(Arrays.asList(allowedPackagesArray))),
+ UDFunction::initializeThread)),
+ "userscripts");
+
++ private static final ClassFilter classFilter = clsName -> secureResource(clsName.replace('.', '/') + ".class");
++
++ private static final NashornScriptEngine scriptEngine;
++
++
+ static
+ {
+ ScriptEngineManager scriptEngineManager = new ScriptEngineManager();
- for (ScriptEngineFactory scriptEngineFactory : scriptEngineManager.getEngineFactories())
- {
- ScriptEngine scriptEngine = scriptEngineFactory.getScriptEngine();
- boolean compilable = scriptEngine instanceof Compilable;
- if (compilable)
- {
- logger.info("Found scripting engine {} {} - {} {} - language names: {}",
- scriptEngineFactory.getEngineName(), scriptEngineFactory.getEngineVersion(),
- scriptEngineFactory.getLanguageName(), scriptEngineFactory.getLanguageVersion(),
- scriptEngineFactory.getNames());
- for (String name : scriptEngineFactory.getNames())
- scriptEngines.put(name, (Compilable) scriptEngine);
- }
- }
++ ScriptEngine engine = scriptEngineManager.getEngineByName("nashorn");
++ NashornScriptEngineFactory factory = engine != null ? (NashornScriptEngineFactory) engine.getFactory() : null;
++ scriptEngine = factory != null ? (NashornScriptEngine) factory.getScriptEngine(new String[]{}, udfClassLoader, classFilter) : null;
+
+ try
+ {
+ protectionDomain = new ProtectionDomain(new CodeSource(new URL("udf", "localhost", 0, "/script", new URLStreamHandler()
+ {
+ protected URLConnection openConnection(URL u)
+ {
+ return null;
+ }
+ }), (Certificate[]) null), ThreadAwareSecurityManager.noPermissions);
+ }
+ catch (MalformedURLException e)
+ {
+ throw new RuntimeException(e);
+ }
+ accessControlContext = new AccessControlContext(new ProtectionDomain[]{ protectionDomain });
+ }
+
+ private final CompiledScript script;
+
+ ScriptBasedUDFunction(FunctionName name,
+ List<ColumnIdentifier> argNames,
+ List<AbstractType<?>> argTypes,
+ AbstractType<?> returnType,
+ boolean calledOnNullInput,
+ String language,
+ String body)
+ {
+ super(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
+
- Compilable scriptEngine = scriptEngines.get(language);
- if (scriptEngine == null)
++ if (!"JavaScript".equalsIgnoreCase(language) || scriptEngine == null)
+ throw new InvalidRequestException(String.format("Invalid language '%s' for function '%s'", language, name));
+
+ // execute compilation with no-permissions to prevent evil code e.g. via "static code blocks" / "class initialization"
+ try
+ {
+ this.script = AccessController.doPrivileged((PrivilegedExceptionAction<CompiledScript>) () -> scriptEngine.compile(body),
+ accessControlContext);
+ }
+ catch (PrivilegedActionException x)
+ {
+ Throwable e = x.getCause();
+ logger.info("Failed to compile function '{}' for language {}: ", name, language, e);
+ throw new InvalidRequestException(
+ String.format("Failed to compile function '%s' for language %s: %s", name, language, e));
+ }
+ }
+
+ protected ExecutorService executor()
+ {
+ return executor;
+ }
+
+ public ByteBuffer executeUserDefined(int protocolVersion, List<ByteBuffer> parameters)
+ {
+ Object[] params = new Object[argTypes.size()];
+ for (int i = 0; i < params.length; i++)
+ params[i] = compose(protocolVersion, i, parameters.get(i));
+
+ ScriptContext scriptContext = new SimpleScriptContext();
+ scriptContext.setAttribute("javax.script.filename", this.name.toString(), ScriptContext.ENGINE_SCOPE);
+ Bindings bindings = scriptContext.getBindings(ScriptContext.ENGINE_SCOPE);
+ for (int i = 0; i < params.length; i++)
+ bindings.put(argNames.get(i).toString(), params[i]);
+
+ Object result;
+ try
+ {
+ // How to prevent Class.forName() _without_ "help" from the script engine ?
+ // NOTE: Nashorn enforces a special permission to allow class-loading, which is not granted - so it's fine.
+
+ result = script.eval(scriptContext);
+ }
+ catch (ScriptException e)
+ {
+ throw new RuntimeException(e);
+ }
+ if (result == null)
+ return null;
+
+ Class<?> javaReturnType = UDHelper.asJavaClass(returnCodec);
+ Class<?> resultType = result.getClass();
+ if (!javaReturnType.isAssignableFrom(resultType))
+ {
+ if (result instanceof Number)
+ {
+ Number rNumber = (Number) result;
+ if (javaReturnType == Integer.class)
+ result = rNumber.intValue();
+ else if (javaReturnType == Long.class)
+ result = rNumber.longValue();
+ else if (javaReturnType == Short.class)
+ result = rNumber.shortValue();
+ else if (javaReturnType == Byte.class)
+ result = rNumber.byteValue();
+ else if (javaReturnType == Float.class)
+ result = rNumber.floatValue();
+ else if (javaReturnType == Double.class)
+ result = rNumber.doubleValue();
+ else if (javaReturnType == BigInteger.class)
+ {
+ if (javaReturnType == Integer.class)
+ result = rNumber.intValue();
+ else if (javaReturnType == Short.class)
+ result = rNumber.shortValue();
+ else if (javaReturnType == Byte.class)
+ result = rNumber.byteValue();
+ else if (javaReturnType == Long.class)
+ result = rNumber.longValue();
+ else if (javaReturnType == Float.class)
+ result = rNumber.floatValue();
+ else if (javaReturnType == Double.class)
+ result = rNumber.doubleValue();
+ else if (javaReturnType == BigInteger.class)
+ {
+ if (rNumber instanceof BigDecimal)
+ result = ((BigDecimal) rNumber).toBigInteger();
+ else if (rNumber instanceof Double || rNumber instanceof Float)
+ result = new BigDecimal(rNumber.toString()).toBigInteger();
+ else
+ result = BigInteger.valueOf(rNumber.longValue());
+ }
+ else if (javaReturnType == BigDecimal.class)
+ // String c'tor of BigDecimal is more accurate than valueOf(double)
+ result = new BigDecimal(rNumber.toString());
+ }
+ else if (javaReturnType == BigDecimal.class)
+ // String c'tor of BigDecimal is more accurate than valueOf(double)
+ result = new BigDecimal(rNumber.toString());
+ }
+ }
+
+ return decompose(protocolVersion, result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
index af3c894,d3050a5..9c931e8
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
@@@ -382,47 -501,4 +382,46 @@@ public class UFScriptTest extends CQLTe
row(1, expected1, expected2));
}
}
+
+ @Test
+ public void testJavascriptDisabled() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key int primary key, val double)");
+
+ DatabaseDescriptor.enableScriptedUserDefinedFunctions(false);
+ try
+ {
- assertInvalid("double",
- "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".assertNotEnabled(val double) " +
++ assertInvalid("CREATE OR REPLACE FUNCTION " + KEYSPACE + ".assertNotEnabled(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS 'Math.sin(val);';");
+ }
+ finally
+ {
+ DatabaseDescriptor.enableScriptedUserDefinedFunctions(true);
+ }
+ }
+
+ @Test
+ public void testJavascriptCompileFailure() throws Throwable
+ {
+ assertInvalidMessage("Failed to compile function 'cql_test_keyspace.scrinv'",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS 'foo bar';");
+ }
+
+ @Test
+ public void testScriptInvalidLanguage() throws Throwable
+ {
+ assertInvalidMessage("Invalid language 'artificial_intelligence' for function 'cql_test_keyspace.scrinv'",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE artificial_intelligence\n" +
+ "AS 'question for 42?';");
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
index 5c7ca2b,0000000..4e45a8a
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
@@@ -1,258 -1,0 +1,268 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.validation.entities;
+
+import java.security.AccessControlException;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.functions.UDHelper;
+import org.apache.cassandra.exceptions.FunctionExecutionException;
+import org.apache.cassandra.service.ClientWarn;
+
+public class UFSecurityTest extends CQLTester
+{
+ @Test
+ public void testSecurityPermissions() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key int primary key, dval double)");
+ execute("INSERT INTO %s (key, dval) VALUES (?, ?)", 1, 1d);
+
+ // Java UDFs
+
+ try
+ {
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS 'System.getProperty(\"foo.bar.baz\"); return 0d;';");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ Assert.fail();
+ }
+ catch (FunctionExecutionException e)
+ {
+ assertAccessControlException("System.getProperty(\"foo.bar.baz\"); return 0d;", e);
+ }
+
+ String[][] typesAndSources =
+ {
+ {"", "try { Class.forName(\"" + UDHelper.class.getName() + "\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;"},
+ {"sun.misc.Unsafe", "sun.misc.Unsafe.getUnsafe(); return 0d;"},
+ {"", "try { Class.forName(\"sun.misc.Unsafe\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;"},
+ {"java.nio.file.FileSystems", "try {" +
+ " java.nio.file.FileSystems.getDefault(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.nio.channels.FileChannel", "try {" +
+ " java.nio.channels.FileChannel.open(java.nio.file.FileSystems.getDefault().getPath(\"/etc/passwd\")).close(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.nio.channels.SocketChannel", "try {" +
+ " java.nio.channels.SocketChannel.open().close(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.io.FileInputStream", "try {" +
+ " new java.io.FileInputStream(\"./foobar\").close(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.lang.Runtime", "try {" +
+ " java.lang.Runtime.getRuntime(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"org.apache.cassandra.service.StorageService",
+ "try {" +
+ " org.apache.cassandra.service.StorageService v = org.apache.cassandra.service.StorageService.instance; v.isShutdown(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.net.ServerSocket", "try {" +
+ " new java.net.ServerSocket().bind(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.io.FileOutputStream","try {" +
+ " new java.io.FileOutputStream(\".foo\"); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.lang.Runtime", "try {" +
+ " java.lang.Runtime.getRuntime().exec(\"/tmp/foo\"); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'}
+ };
+
+ for (String[] typeAndSource : typesAndSources)
+ {
+ assertInvalidMessage(typeAndSource[0] + " cannot be resolved",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".invalid_class_access(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS '" + typeAndSource[1] + "';");
+ }
+
+ // JavaScript UDFs
+
+ try
+ {
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS 'org.apache.cassandra.service.StorageService.instance.isShutdown(); 0;';");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ Assert.fail("Javascript security check failed");
+ }
+ catch (FunctionExecutionException e)
+ {
+ assertAccessControlException("", e);
+ }
+
+ String[] javascript =
+ {
+ "java.lang.management.ManagmentFactory.getThreadMXBean(); 0;",
+ "new java.io.FileInputStream(\"/tmp/foo\"); 0;",
+ "new java.io.FileOutputStream(\"/tmp/foo\"); 0;",
+ "java.nio.file.FileSystems.getDefault().createFileExclusively(\"./foo_bar_baz\"); 0;",
+ "java.nio.channels.FileChannel.open(java.nio.file.FileSystems.getDefault().getPath(\"/etc/passwd\")); 0;",
+ "java.nio.channels.SocketChannel.open(); 0;",
+ "new java.net.ServerSocket().bind(null); 0;",
+ "var thread = new java.lang.Thread(); thread.start(); 0;",
+ "java.lang.System.getProperty(\"foo.bar.baz\"); 0;",
- "java.lang.Class.forName(\"java.lang.System\"); 0;",
+ "java.lang.Runtime.getRuntime().exec(\"/tmp/foo\"); 0;",
+ "java.lang.Runtime.getRuntime().loadLibrary(\"foobar\"); 0;",
+ "java.lang.Runtime.getRuntime().loadLibrary(\"foobar\"); 0;",
+ // TODO these (ugly) calls are still possible - these can consume CPU (as one could do with an evil loop, too)
+// "java.lang.Runtime.getRuntime().traceMethodCalls(true); 0;",
+// "java.lang.Runtime.getRuntime().gc(); 0;",
+// "java.lang.Runtime.getRuntime(); 0;",
+ };
+
+ for (String script : javascript)
+ {
+ try
+ {
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS '" + script + "';");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ Assert.fail("Javascript security check failed: " + script);
+ }
+ catch (FunctionExecutionException e)
+ {
+ assertAccessControlException(script, e);
+ }
+ }
++
++ String script = "java.lang.Class.forName(\"java.lang.System\"); 0;";
++ String fName = createFunction(KEYSPACE_PER_TEST, "double",
++ "CREATE OR REPLACE FUNCTION %s(val double) " +
++ "RETURNS NULL ON NULL INPUT " +
++ "RETURNS double " +
++ "LANGUAGE javascript\n" +
++ "AS '" + script + "';");
++ assertInvalidThrowMessage("Java reflection not supported when class filter is present",
++ FunctionExecutionException.class,
++ "SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ }
+
+ private static void assertAccessControlException(String script, FunctionExecutionException e)
+ {
+ for (Throwable t = e; t != null && t != t.getCause(); t = t.getCause())
+ if (t instanceof AccessControlException)
+ return;
+ Assert.fail("no AccessControlException for " + script + " (got " + e + ')');
+ }
+
+ @Test
+ public void testAmokUDF() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key int primary key, dval double)");
+ execute("INSERT INTO %s (key, dval) VALUES (?, ?)", 1, 1d);
+
+ long udfWarnTimeout = DatabaseDescriptor.getUserDefinedFunctionWarnTimeout();
+ long udfFailTimeout = DatabaseDescriptor.getUserDefinedFunctionFailTimeout();
+ int maxTries = 5;
+ for (int i = 1; i <= maxTries; i++)
+ {
+ try
+ {
+ // short timeout
+ DatabaseDescriptor.setUserDefinedFunctionWarnTimeout(10);
+ DatabaseDescriptor.setUserDefinedFunctionFailTimeout(250);
+ // don't kill the unit test... - default policy is "die"
+ DatabaseDescriptor.setUserFunctionTimeoutPolicy(Config.UserFunctionTimeoutPolicy.ignore);
+
+ ClientWarn.instance.captureWarnings();
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS 'long t=System.currentTimeMillis()+110; while (t>System.currentTimeMillis()) { }; return 0d;'");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ List<String> warnings = ClientWarn.instance.getWarnings();
+ Assert.assertNotNull(warnings);
+ Assert.assertFalse(warnings.isEmpty());
+ ClientWarn.instance.resetWarnings();
+
+ // Java UDF
+
+ fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS 'long t=System.currentTimeMillis()+500; while (t>System.currentTimeMillis()) { }; return 0d;';");
+ assertInvalidMessage("ran longer than 250ms", "SELECT " + fName + "(dval) FROM %s WHERE key=1");
+
+ // Javascript UDF
+
+ fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVASCRIPT\n" +
+ "AS 'var t=java.lang.System.currentTimeMillis()+500; while (t>java.lang.System.currentTimeMillis()) { }; 0;';");
+ assertInvalidMessage("ran longer than 250ms", "SELECT " + fName + "(dval) FROM %s WHERE key=1");
+
+ return;
+ }
+ catch (Error | RuntimeException e)
+ {
+ if (i == maxTries)
+ throw e;
+ }
+ finally
+ {
+ // reset to defaults
+ DatabaseDescriptor.setUserDefinedFunctionWarnTimeout(udfWarnTimeout);
+ DatabaseDescriptor.setUserDefinedFunctionFailTimeout(udfFailTimeout);
+ }
+ }
+ }
+
+}
[08/10] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.X
Posted by sn...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.X
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/eca51a03
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/eca51a03
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/eca51a03
Branch: refs/heads/cassandra-3.X
Commit: eca51a0348141f45ea9bb31b76571ed81284c32c
Parents: 5f3b359 6e716c6
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu Jan 5 22:26:48 2017 +0100
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu Jan 5 22:26:48 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +
NEWS.txt | 3 ++
doc/cql3/CQL.textile | 2 +-
lib/jsr223/clojure/README.txt | 8 ---
lib/jsr223/groovy/README.txt | 35 -------------
lib/jsr223/jaskell/README.txt | 5 --
lib/jsr223/jruby/README.txt | 54 --------------------
lib/jsr223/jython/README.txt | 33 ------------
lib/jsr223/scala/README.txt | 37 --------------
.../cql3/functions/ScriptBasedUDFunction.java | 35 +++++--------
.../cql3/validation/entities/UFScriptTest.java | 3 +-
.../validation/entities/UFSecurityTest.java | 12 ++++-
12 files changed, 31 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eca51a03/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index cacb7cd,77a310d..66c2ee6
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,136 -1,5 +1,138 @@@
-3.0.11
- * Fixed flacky SSTableRewriterTest: check file counts before calling validateCFS (CASSANDRA-12348)
+3.12
+ * Use new token allocation for non bootstrap case as well (CASSANDRA-13080)
+ * Avoid byte-array copy when key cache is disabled (CASSANDRA-13084)
+ * More fixes to the TokenAllocator (CASSANDRA-12990)
+ * Require forceful decommission if number of nodes is less than replication factor (CASSANDRA-12510)
+ * Allow IN restrictions on column families with collections (CASSANDRA-12654)
+ * Move to FastThreadLocalThread and FastThreadLocal (CASSANDRA-13034)
+ * nodetool stopdaemon errors out (CASSANDRA-13030)
+ * Log message size in trace message in OutboundTcpConnection (CASSANDRA-13028)
+ * Add timeUnit Days for cassandra-stress (CASSANDRA-13029)
+ * Add mutation size and batch metrics (CASSANDRA-12649)
+ * Add method to get size of endpoints to TokenMetadata (CASSANDRA-12999)
+ * Fix primary index calculation for SASI (CASSANDRA-12910)
+ * Expose time spent waiting in thread pool queue (CASSANDRA-8398)
+ * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969)
+ * NoReplicationTokenAllocator should work with zero replication factor (CASSANDRA-12983)
+ * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946)
+ * Add support for arithmetic operators (CASSANDRA-11935)
+ * Tables in system_distributed should not use gcgs of 0 (CASSANDRA-12954)
++Merged from 3.0:
++ * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
+
+
+3.10
+ * Fixed query monitoring for range queries (CASSANDRA-13050)
+ * Remove outboundBindAny configuration property (CASSANDRA-12673)
+ * Use correct bounds for all-data range when filtering (CASSANDRA-12666)
+ * Remove timing window in test case (CASSANDRA-12875)
+ * Resolve unit testing without JCE security libraries installed (CASSANDRA-12945)
+ * Fix inconsistencies in cassandra-stress load balancing policy (CASSANDRA-12919)
+ * Fix validation of non-frozen UDT cells (CASSANDRA-12916)
+ * Don't shut down socket input/output on StreamSession (CASSANDRA-12903)
+ * Fix Murmur3PartitionerTest (CASSANDRA-12858)
+ * Move cqlsh syntax rules into separate module and allow easier customization (CASSANDRA-12897)
+ * Fix CommitLogSegmentManagerTest (CASSANDRA-12283)
+ * Fix cassandra-stress truncate option (CASSANDRA-12695)
+ * Fix crossNode value when receiving messages (CASSANDRA-12791)
+ * Don't load MX4J beans twice (CASSANDRA-12869)
+ * Extend native protocol request flags, add versions to SUPPORTED, and introduce ProtocolVersion enum (CASSANDRA-12838)
+ * Set JOINING mode when running pre-join tasks (CASSANDRA-12836)
+ * remove net.mintern.primitive library due to license issue (CASSANDRA-12845)
+ * Properly format IPv6 addresses when logging JMX service URL (CASSANDRA-12454)
+ * Optimize the vnode allocation for single replica per DC (CASSANDRA-12777)
+ * Use non-token restrictions for bounds when token restrictions are overridden (CASSANDRA-12419)
+ * Fix CQLSH auto completion for PER PARTITION LIMIT (CASSANDRA-12803)
+ * Use different build directories for Eclipse and Ant (CASSANDRA-12466)
+ * Avoid potential AttributeError in cqlsh due to no table metadata (CASSANDRA-12815)
+ * Fix RandomReplicationAwareTokenAllocatorTest.testExistingCluster (CASSANDRA-12812)
+ * Upgrade commons-codec to 1.9 (CASSANDRA-12790)
+ * Make the fanout size for LeveledCompactionStrategy to be configurable (CASSANDRA-11550)
+ * Add duration data type (CASSANDRA-11873)
+ * Fix timeout in ReplicationAwareTokenAllocatorTest (CASSANDRA-12784)
+ * Improve sum aggregate functions (CASSANDRA-12417)
+ * Make cassandra.yaml docs for batch_size_*_threshold_in_kb reflect changes in CASSANDRA-10876 (CASSANDRA-12761)
+ * cqlsh fails to format collections when using aliases (CASSANDRA-11534)
+ * Check for hash conflicts in prepared statements (CASSANDRA-12733)
+ * Exit query parsing upon first error (CASSANDRA-12598)
+ * Fix cassandra-stress to use single seed in UUID generation (CASSANDRA-12729)
+ * CQLSSTableWriter does not allow Update statement (CASSANDRA-12450)
+ * Config class uses boxed types but DD exposes primitive types (CASSANDRA-12199)
+ * Add pre- and post-shutdown hooks to Storage Service (CASSANDRA-12461)
+ * Add hint delivery metrics (CASSANDRA-12693)
+ * Remove IndexInfo cache from FileIndexInfoRetriever (CASSANDRA-12731)
+ * ColumnIndex does not reuse buffer (CASSANDRA-12502)
+ * cdc column addition still breaks schema migration tasks (CASSANDRA-12697)
+ * Upgrade metrics-reporter dependencies (CASSANDRA-12089)
+ * Tune compaction thread count via nodetool (CASSANDRA-12248)
+ * Add +=/-= shortcut syntax for update queries (CASSANDRA-12232)
+ * Include repair session IDs in repair start message (CASSANDRA-12532)
+ * Add a blocking task to Index, run before joining the ring (CASSANDRA-12039)
+ * Fix NPE when using CQLSSTableWriter (CASSANDRA-12667)
+ * Support optional backpressure strategies at the coordinator (CASSANDRA-9318)
+ * Make randompartitioner work with new vnode allocation (CASSANDRA-12647)
+ * Fix cassandra-stress graphing (CASSANDRA-12237)
+ * Allow filtering on partition key columns for queries without secondary indexes (CASSANDRA-11031)
+ * Fix Cassandra Stress reporting thread model and precision (CASSANDRA-12585)
+ * Add JMH benchmarks.jar (CASSANDRA-12586)
+ * Cleanup uses of AlterTableStatementColumn (CASSANDRA-12567)
+ * Add keep-alive to streaming (CASSANDRA-11841)
+ * Tracing payload is passed through newSession(..) (CASSANDRA-11706)
+ * avoid deleting non existing sstable files and improve related log messages (CASSANDRA-12261)
+ * json/yaml output format for nodetool compactionhistory (CASSANDRA-12486)
+ * Retry all internode messages once after a connection is
+ closed and reopened (CASSANDRA-12192)
+ * Add support to rebuild from targeted replica (CASSANDRA-9875)
+ * Add sequence distribution type to cassandra stress (CASSANDRA-12490)
+ * "SELECT * FROM foo LIMIT ;" does not error out (CASSANDRA-12154)
+ * Define executeLocally() at the ReadQuery Level (CASSANDRA-12474)
+ * Extend read/write failure messages with a map of replica addresses
+ to error codes in the v5 native protocol (CASSANDRA-12311)
+ * Fix rebuild of SASI indexes with existing index files (CASSANDRA-12374)
+ * Let DatabaseDescriptor not implicitly startup services (CASSANDRA-9054, 12550)
+ * Fix clustering indexes in presence of static columns in SASI (CASSANDRA-12378)
+ * Fix queries on columns with reversed type on SASI indexes (CASSANDRA-12223)
+ * Added slow query log (CASSANDRA-12403)
+ * Count full coordinated request against timeout (CASSANDRA-12256)
+ * Allow TTL with null value on insert and update (CASSANDRA-12216)
+ * Make decommission operation resumable (CASSANDRA-12008)
+ * Add support to one-way targeted repair (CASSANDRA-9876)
+ * Remove clientutil jar (CASSANDRA-11635)
+ * Fix compaction throughput throttle (CASSANDRA-12366, CASSANDRA-12717)
+ * Delay releasing Memtable memory on flush until PostFlush has finished running (CASSANDRA-12358)
+ * Cassandra stress should dump all setting on startup (CASSANDRA-11914)
+ * Make it possible to compact a given token range (CASSANDRA-10643)
+ * Allow updating DynamicEndpointSnitch properties via JMX (CASSANDRA-12179)
+ * Collect metrics on queries by consistency level (CASSANDRA-7384)
+ * Add support for GROUP BY to SELECT statement (CASSANDRA-10707)
+ * Deprecate memtable_cleanup_threshold and update default for memtable_flush_writers (CASSANDRA-12228)
+ * Upgrade to OHC 0.4.4 (CASSANDRA-12133)
+ * Add version command to cassandra-stress (CASSANDRA-12258)
+ * Create compaction-stress tool (CASSANDRA-11844)
+ * Garbage-collecting compaction operation and schema option (CASSANDRA-7019)
+ * Add beta protocol flag for v5 native protocol (CASSANDRA-12142)
+ * Support filtering on non-PRIMARY KEY columns in the CREATE
+ MATERIALIZED VIEW statement's WHERE clause (CASSANDRA-10368)
+ * Unify STDOUT and SYSTEMLOG logback format (CASSANDRA-12004)
+ * COPY FROM should raise error for non-existing input files (CASSANDRA-12174)
+ * Faster write path (CASSANDRA-12269)
+ * Option to leave omitted columns in INSERT JSON unset (CASSANDRA-11424)
+ * Support json/yaml output in nodetool tpstats (CASSANDRA-12035)
+ * Expose metrics for successful/failed authentication attempts (CASSANDRA-10635)
+ * Prepend snapshot name with "truncated" or "dropped" when a snapshot
+ is taken before truncating or dropping a table (CASSANDRA-12178)
+ * Optimize RestrictionSet (CASSANDRA-12153)
+ * cqlsh does not automatically downgrade CQL version (CASSANDRA-12150)
+ * Omit (de)serialization of state variable in UDAs (CASSANDRA-9613)
+ * Create a system table to expose prepared statements (CASSANDRA-8831)
+ * Reuse DataOutputBuffer from ColumnIndex (CASSANDRA-11970)
+ * Remove DatabaseDescriptor dependency from SegmentedFile (CASSANDRA-11580)
+ * Add supplied username to authentication error messages (CASSANDRA-12076)
+ * Remove pre-startup check for open JMX port (CASSANDRA-12074)
+ * Remove compaction Severity from DynamicEndpointSnitch (CASSANDRA-11738)
+ * Restore resumable hints delivery (CASSANDRA-11960)
+ * Properly report LWT contention (CASSANDRA-12626)
+Merged from 3.0:
* Fix deserialization of 2.x DeletedCells (CASSANDRA-12620)
* Add parent repair session id to anticompaction log message (CASSANDRA-12186)
* Improve contention handling on failure to acquire MV lock for streaming and hints (CASSANDRA-12905)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eca51a03/NEWS.txt
----------------------------------------------------------------------
diff --cc NEWS.txt
index e6c8c10,b4e6551..9f376ff
--- a/NEWS.txt
+++ b/NEWS.txt
@@@ -21,125 -21,23 +21,128 @@@ Upgradin
- Specifying the default_time_to_live option when creating or altering a
materialized view was erroneously accepted (and ignored). It is now
properly rejected.
+ - Only Java and JavaScript are now supported UDF languages.
+ The sandbox in 3.0 already prevented the use of script languages except Java
+ and JavaScript.
-3.0.10
-=====
+3.10
+====
-Upgrading
----------
- - memtable_allocation_type: offheap_buffers is no longer allowed to be specified in the 3.0 series.
- This was an oversight that can cause segfaults. Offheap was re-introduced in 3.4 see CASSANDRA-11039
- and CASSANDRA-9472 for details.
+New features
+------------
+ - New `DurationType` (cql duration). See CASSANDRA-11873
+ - Runtime modification of concurrent_compactors is now available via nodetool
+ - Support for the assignment operators +=/-= has been added for update queries.
+ - An Index implementation may now provide a task which runs prior to joining
+ the ring. See CASSANDRA-12039
+ - Filtering on partition key columns is now also supported for queries without
+ secondary indexes.
+ - A slow query log has been added: slow queries will be logged at DEBUG level.
+ For more details refer to CASSANDRA-12403 and slow_query_log_timeout_in_ms
+ in cassandra.yaml.
+ - Support for GROUP BY queries has been added.
+ - A new compaction-stress tool has been added to test the throughput of compaction
+ for any cassandra-stress user schema. see compaction-stress help for how to use.
+ - Compaction can now take into account overlapping tables that don't take part
+ in the compaction to look for deleted or overwritten data in the compacted tables.
+ Then such data is found, it can be safely discarded, which in turn should enable
+ the removal of tombstones over that data.
+
+ The behavior can be engaged in two ways:
+ - as a "nodetool garbagecollect -g CELL/ROW" operation, which applies
+ single-table compaction on all sstables to discard deleted data in one step.
+ - as a "provide_overlapping_tombstones:CELL/ROW/NONE" compaction strategy flag,
+ which uses overlapping tables as a source of deletions/overwrites during all
+ compactions.
+ The argument specifies the granularity at which deleted data is to be found:
+ - If ROW is specified, only whole deleted rows (or sets of rows) will be
+ discarded.
+ - If CELL is specified, any columns whose value is overwritten or deleted
+ will also be discarded.
+ - NONE (default) specifies the old behavior, overlapping tables are not used to
+ decide when to discard data.
+ Which option to use depends on your workload, both ROW and CELL increase the
+ disk load on compaction (especially with the size-tiered compaction strategy),
+ with CELL being more resource-intensive. Both should lead to better read
+ performance if deleting rows (resp. overwriting or deleting cells) is common.
+ - Prepared statements are now persisted in the table prepared_statements in
+ the system keyspace. Upon startup, this table is used to preload all
+ previously prepared statements - i.e. in many cases clients do not need to
+ re-prepare statements against restarted nodes.
+ - cqlsh can now connect to older Cassandra versions by downgrading the native
+ protocol version. Please note that this is currently not part of our release
+ testing and, as a consequence, it is not guaranteed to work in all cases.
+ See CASSANDRA-12150 for more details.
+ - Snapshots that are automatically taken before a table is dropped or truncated
+ will have a "dropped" or "truncated" prefix on their snapshot tag name.
+ - Metrics are exposed for successful and failed authentication attempts.
+ These can be located using the object names org.apache.cassandra.metrics:type=Client,name=AuthSuccess
+ and org.apache.cassandra.metrics:type=Client,name=AuthFailure respectively.
+ - Add support to "unset" JSON fields in prepared statements by specifying DEFAULT UNSET.
+ See CASSANDRA-11424 for details
+ - Allow TTL with null value on insert and update. It will be treated as equivalent to inserting a 0.
+ - Removed outboundBindAny configuration property. See CASSANDRA-12673 for details.
+
+Upgrading
+---------
+ - Request timeouts in cassandra.yaml (read_request_timeout_in_ms, etc) now apply to the
+ "full" request time on the coordinator. Previously, they only covered the time from
+ when the coordinator sent a message to a replica until the time that the replica
+ responded. Additionally, the previous behavior was to reset the timeout when performing
+ a read repair, making a second read to fix a short read, and when subranges were read
+ as part of a range scan or secondary index query. In 3.10 and higher, the timeout
+ is no longer reset for these "subqueries". The entire request must complete within
+ the specified timeout. As a consequence, your timeouts may need to be adjusted
+ to account for this. See CASSANDRA-12256 for more details.
+ - Logs written to stdout are now consistent with logs written to files.
+ Time is now local (it was UTC on the console and local in files). Date, thread, file
+ and line info where added to stdout. (see CASSANDRA-12004)
+ - The 'clientutil' jar, which has been somewhat broken on the 3.x branch, is not longer provided.
+ The features provided by that jar are provided by any good java driver and we advise relying on drivers rather on
+ that jar, but if you need that jar for backward compatiblity until you do so, you should use the version provided
+ on previous Cassandra branch, like the 3.0 branch (by design, the functionality provided by that jar are stable
+ accross versions so using the 3.0 jar for a client connecting to 3.x should work without issues).
+ - (Tools development) DatabaseDescriptor no longer implicitly startups components/services like
+ commit log replay. This may break existing 3rd party tools and clients. In order to startup
+ a standalone tool or client application, use the DatabaseDescriptor.toolInitialization() or
+ DatabaseDescriptor.clientInitialization() methods. Tool initialization sets up partitioner,
+ snitch, encryption context. Client initialization just applies the configuration but does not
+ setup anything. Instead of using Config.setClientMode() or Config.isClientMode(), which are
+ deprecated now, use one of the appropiate new methods in DatabaseDescriptor.
+ - Application layer keep-alives were added to the streaming protocol to prevent idle incoming connections from
+ timing out and failing the stream session (CASSANDRA-11839). This effectively deprecates the streaming_socket_timeout_in_ms
+ property in favor of streaming_keep_alive_period_in_secs. See cassandra.yaml for more details about this property.
+ - Duration litterals support the ISO 8601 format. By consequence, identifiers matching that format
+ (e.g P2Y or P1MT6H) will not be supported anymore (CASSANDRA-11873).
+
+3.8
+===
-3.0.9
-=====
+New features
+------------
+ - Shared pool threads are now named according to the stage they are executing
+ tasks for. Thread names mentioned in traced queries change accordingly.
+ - A new option has been added to cassandra-stress "-rate fixed={number}/s"
+ that forces a scheduled rate of operations/sec over time. Using this, stress can
+ accurately account for coordinated ommission from the stress process.
+ - The cassandra-stress "-rate limit=" option has been renamed to "-rate throttle="
+ - hdr histograms have been added to stress runs, it's output can be saved to disk using:
+ "-log hdrfile=" option. This histogram includes response/service/wait times when used with the
+ fixed or throttle rate options. The histogram file can be plotted on
+ http://hdrhistogram.github.io/HdrHistogram/plotFiles.html
+ - TimeWindowCompactionStrategy has been added. This has proven to be a better approach
+ to time series compaction and new tables should use this instead of DTCS. See
+ CASSANDRA-9666 for details.
+ - Change-Data-Capture is now available. See cassandra.yaml and for cdc-specific flags and
+ a brief explanation of on-disk locations for archived data in CommitLog form. This can
+ be enabled via ALTER TABLE ... WITH cdc=true.
+ Upon flush, CommitLogSegments containing data for CDC-enabled tables are moved to
+ the data/cdc_raw directory until removed by the user and writes to CDC-enabled tables
+ will be rejected with a WriteTimeoutException once cdc_total_space_in_mb is reached
+ between unflushed CommitLogSegments and cdc_raw.
+ NOTE: CDC is disabled by default in the .yaml file. Do not enable CDC on a mixed-version
+ cluster as it will lead to exceptions which can interrupt traffic. Once all nodes
+ have been upgraded to 3.8 it is safe to enable this feature and restart the cluster.
Upgrading
---------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eca51a03/doc/cql3/CQL.textile
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eca51a03/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
index 3ad60d0,47deafa..c568972
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
@@@ -27,7 -27,9 +27,10 @@@ import java.util.*
import java.util.concurrent.ExecutorService;
import javax.script.*;
+import jdk.nashorn.api.scripting.AbstractJSObject;
+ import jdk.nashorn.api.scripting.ClassFilter;
+ import jdk.nashorn.api.scripting.NashornScriptEngine;
+ import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.marshal.AbstractType;
@@@ -158,13 -149,6 +152,10 @@@ final class ScriptBasedUDFunction exten
throw new InvalidRequestException(
String.format("Failed to compile function '%s' for language %s: %s", name, language, e));
}
+
+ // It's not always possible to simply pass a plain Java object as a binding to Nashorn and
+ // let the script execute methods on it.
- udfContextBinding =
- ("Oracle Nashorn".equals(((ScriptEngine) scriptEngine).getFactory().getEngineName()))
- ? new UDFContextWrapper()
- : udfContext;
++ udfContextBinding = new UDFContextWrapper();
}
protected ExecutorService executor()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eca51a03/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
----------------------------------------------------------------------
[02/10] cassandra git commit: Remove support for non-JavaScript UDFs
Posted by sn...@apache.org.
Remove support for non-JavaScript UDFs
patch by Robert Stupp; reviewed by Tyler Hobbs for CASSANDRA-12883
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6f360b6d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6f360b6d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6f360b6d
Branch: refs/heads/cassandra-3.0
Commit: 6f360b6d239a7b3d3b625e1bcefcaeb2f6b2be1d
Parents: 0cf0f67
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu Jan 5 22:18:37 2017 +0100
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu Jan 5 22:19:50 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 16 ++++++++++++++++
.../cassandra/cql3/functions/ScriptBasedUDF.java | 5 +++++
3 files changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d31ffc8..b41313d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.9
+ * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
* Fix DynamicEndpointSnitch noop in multi-datacenter situations (CASSANDRA-13074)
* cqlsh copy-from: encode column names to avoid primary key parsing errors (CASSANDRA-12909)
* Temporarily fix bug that creates commit log when running offline tools (CASSANDRA-8616)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index 0a3ab36..37949a1 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -13,6 +13,19 @@ restore snapshots created with the previous major version using the
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
+2.2.9
+=====
+
+Deprecation
+-----------
+
+(See note about the new feature User-Defined-Functions in 2.2.0.)
+
+Since the security manager added in 3.0 only allows Java and JavaScript
+UDFs to be run, UDFs for other languages are deprecated and support for
+non-Java and non-JavaScript UDFs is deprecated in 2.2 and has been removed
+in version 3.0.11.
+
2.2.8
=====
@@ -230,6 +243,9 @@ New features
3.0. This will inherently be backwards-incompatible with any 2.2
UDF that perform insecure operations such as opening a socket or
writing to the filesystem.
+
+ Per the previous note about adding a security manager in 3.0, this security manager
+ means that non JavaScipt UDF's won't run, there for their use is deprecated.
************************************************************************
- Row-cache is now fully off-heap.
- jemalloc is now automatically preloaded and used on Linux and OS-X if
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
index e55d450..2d46934 100644
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
@@ -73,6 +73,11 @@ public class ScriptBasedUDF extends UDFunction
{
super(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
+ if (!"JavaScript".equalsIgnoreCase(language))
+ logger.warn("Support for UDFs using '" + language + "' has been deprecated and removed in 3.0. If '" +
+ language + "' actually is JavaScript, change the language used in CREATE/ALTER FUNCTION to " +
+ "'javascript'.");
+
Compilable scriptEngine = scriptEngines.get(language);
if (scriptEngine == null)
throw new InvalidRequestException(String.format("Invalid language '%s' for function '%s'", language, name));
[07/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by sn...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6e716c6d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6e716c6d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6e716c6d
Branch: refs/heads/trunk
Commit: 6e716c6da41900950e32a5549b3bb1e858ecad18
Parents: c0765ed 6f360b6
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu Jan 5 22:20:31 2017 +0100
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu Jan 5 22:20:31 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 3 ++
doc/cql3/CQL.textile | 2 +-
lib/jsr223/clojure/README.txt | 8 ---
lib/jsr223/groovy/README.txt | 35 -------------
lib/jsr223/jaskell/README.txt | 5 --
lib/jsr223/jruby/README.txt | 54 --------------------
lib/jsr223/jython/README.txt | 33 ------------
lib/jsr223/scala/README.txt | 37 --------------
.../cql3/functions/ScriptBasedUDFunction.java | 30 +++++------
.../cql3/validation/entities/UFScriptTest.java | 3 +-
.../validation/entities/UFSecurityTest.java | 12 ++++-
12 files changed, 29 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 666a771,b41313d..77a310d
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,23 -1,5 +1,24 @@@
-2.2.9
+3.0.11
+ * Fixed flacky SSTableRewriterTest: check file counts before calling validateCFS (CASSANDRA-12348)
+ * Fix deserialization of 2.x DeletedCells (CASSANDRA-12620)
+ * Add parent repair session id to anticompaction log message (CASSANDRA-12186)
+ * Improve contention handling on failure to acquire MV lock for streaming and hints (CASSANDRA-12905)
+ * Fix DELETE and UPDATE queries with empty IN restrictions (CASSANDRA-12829)
+ * Mark MVs as built after successful bootstrap (CASSANDRA-12984)
+ * Estimated TS drop-time histogram updated with Cell.NO_DELETION_TIME (CASSANDRA-13040)
+ * Nodetool compactionstats fails with NullPointerException (CASSANDRA-13021)
+ * Thread local pools never cleaned up (CASSANDRA-13033)
+ * Set RPC_READY to false when draining or if a node is marked as shutdown (CASSANDRA-12781)
+ * CQL often queries static columns unnecessarily (CASSANDRA-12768)
+ * Make sure sstables only get committed when it's safe to discard commit log records (CASSANDRA-12956)
+ * Reject default_time_to_live option when creating or altering MVs (CASSANDRA-12868)
+ * Nodetool should use a more sane max heap size (CASSANDRA-12739)
+ * LocalToken ensures token values are cloned on heap (CASSANDRA-12651)
+ * AnticompactionRequestSerializer serializedSize is incorrect (CASSANDRA-12934)
+ * Prevent reloading of logback.xml from UDF sandbox (CASSANDRA-12535)
+ * Reenable HeapPool (CASSANDRA-12900)
+Merged from 2.2:
+ * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
* Fix DynamicEndpointSnitch noop in multi-datacenter situations (CASSANDRA-13074)
* cqlsh copy-from: encode column names to avoid primary key parsing errors (CASSANDRA-12909)
* Temporarily fix bug that creates commit log when running offline tools (CASSANDRA-8616)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/NEWS.txt
----------------------------------------------------------------------
diff --cc NEWS.txt
index 32b5084,37949a1..b4e6551
--- a/NEWS.txt
+++ b/NEWS.txt
@@@ -13,77 -13,20 +13,80 @@@ restore snapshots created with the prev
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
-2.2.9
+3.0.11
=====
+Upgrading
+---------
+ - Nothing specific to this release, but please see previous versions upgrading section,
+ especially if you are upgrading from 2.2.
+ - Specifying the default_time_to_live option when creating or altering a
+ materialized view was erroneously accepted (and ignored). It is now
+ properly rejected.
++ - Only Java and JavaScript are now supported UDF languages.
++ The sandbox in 3.0 already prevented the use of script languages except Java
++ and JavaScript.
+
+3.0.10
+=====
+
+Upgrading
+---------
+ - memtable_allocation_type: offheap_buffers is no longer allowed to be specified in the 3.0 series.
+ This was an oversight that can cause segfaults. Offheap was re-introduced in 3.4 see CASSANDRA-11039
+ and CASSANDRA-9472 for details.
+
+3.0.9
+=====
+
+Upgrading
+---------
+ - The ReversedType behaviour has been corrected for clustering columns of
+ BYTES type containing empty value. Scrub should be run on the existing
+ SSTables containing a descending clustering column of BYTES type to correct
+ their ordering. See CASSANDRA-12127 for more details.
+
+3.0.8
+=====
+
+Upgrading
+---------
+ - Ec2MultiRegionSnitch will no longer automatically set broadcast_rpc_address
+ to the public instance IP if this property is defined on cassandra.yaml.
+
+3.0.7
+=====
+
+Upgrading
+---------
+ - A maximum size for SSTables values has been introduced, to prevent out of memory
+ exceptions when reading corrupt SSTables. This maximum size can be set via
+ max_value_size_in_mb in cassandra.yaml. The default is 256MB, which matches the default
+ value of native_transport_max_frame_size_in_mb. SSTables will be considered corrupt if
+ they contain values whose size exceeds this limit. See CASSANDRA-9530 for more details.
+
Deprecation
-----------
+ - DateTieredCompactionStrategy has been deprecated - new tables should use
+ TimeWindowCompactionStrategy. Note that migrating an existing DTCS-table to TWCS might
+ cause increased compaction load for a while after the migration so make sure you run
+ tests before migrating. Read CASSANDRA-9666 for background on this.
-(See note about the new feature User-Defined-Functions in 2.2.0.)
+New features
+------------
+ - TimeWindowCompactionStrategy has been added. This has proven to be a better approach
+ to time series compaction and new tables should use this instead of DTCS. See
+ CASSANDRA-9666 for details.
-Since the security manager added in 3.0 only allows Java and JavaScript
-UDFs to be run, UDFs for other languages are deprecated and support for
-non-Java and non-JavaScript UDFs is deprecated in 2.2 and has been removed
-in version 3.0.11.
+3.0.6
+=====
+
+New features
+------------
+ - JSON timestamps are now in UTC and contain the timezone information, see
+ CASSANDRA-11137 for more details.
-2.2.8
+3.0.5
=====
Upgrading
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/doc/cql3/CQL.textile
----------------------------------------------------------------------
diff --cc doc/cql3/CQL.textile
index 2a37452,af584d0..2544878
--- a/doc/cql3/CQL.textile
+++ b/doc/cql3/CQL.textile
@@@ -1977,7 -1910,7 +1977,7 @@@ SELECT AVG(players) FROM plays
h2(#udfs). User-Defined Functions
--User-defined functions allow execution of user-provided code in Cassandra. By default, Cassandra supports defining functions in _Java_ and _JavaScript_. Support for other JSR 223 compliant scripting languages (such as Python, Ruby, and Scala) can be added by adding a JAR to the classpath.
++User-defined functions allow execution of user-provided code in Cassandra. By default, Cassandra supports defining functions in _Java_ and _JavaScript_. Support for other JSR 223 compliant scripting languages (such as Python, Ruby, and Scala) has been removed in 3.0.11.
UDFs are part of the Cassandra schema. As such, they are automatically propagated to all nodes in the cluster.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/clojure/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/clojure/README.txt
index 7ed7551,7ed7551..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/clojure/README.txt
+++ /dev/null
@@@ -1,8 -1,8 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Unfortunately the JSR-223 support provided by the project https://github.com/ato/clojure-jsr223
--and the related ones do not provide compileable script support.
--
--The JSR-223 javax.script.Compilable implementation takes source file names or readers but not script sources
--as all other JSR-223 implementations do.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/groovy/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/groovy/README.txt
index 09fef93,09fef93..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/groovy/README.txt
+++ /dev/null
@@@ -1,35 -1,35 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable Groovy
--
--Tested with version 2.3.6
--
--Installation
--------------
--
--1. Download Groovy binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy the jar groovy-all-2.3.6-indy.jar from the Groovy embeddable directory to $CASSANDRA_HOME/lib/jsr223/groovy
-- "indy" means "invokedynamic" and is a JVM instruction for scripting languages new to Java 7.
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 10:49:45 Found scripting engine Groovy Scripting Engine 2.0 - Groovy 2.3.6 - language names: [groovy, Groovy]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--Smoke Test
------------
--
--To test Groovy functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE groovy AS 'return "foo";' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language groovy for 'foobar'"
--Groovy for Apache Cassandra has not been installed correctly.
--
--Notes / Java7 invokedynamic
-----------------------------
--
--Groovy provides jars that support invokedynamic bytecode instruction. These jars are whose ending with
--"-indy.jar".
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jaskell/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/jaskell/README.txt
index 53e942e,53e942e..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/jaskell/README.txt
+++ /dev/null
@@@ -1,5 -1,5 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Unfortunately Jaskell JSR-223 support is quite old and the Jaskell engine seems to be quite
--unsupported. If you find a solution, please open a ticket at Apache Cassandra JIRA.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jruby/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/jruby/README.txt
index cbc12dc,cbc12dc..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/jruby/README.txt
+++ /dev/null
@@@ -1,54 -1,54 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable JRuby
--
--Tested with version 1.7.15
--
--Installation
--------------
--
--1. Download JRuby binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy everything from the JRuby lib directory to $CASSANDRA_HOME/lib/jsr223/jruby
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 10:29:03 Found scripting engine JSR 223 JRuby Engine 1.7.15 - ruby jruby 1.7.15 - language names: [ruby, jruby]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--
--Smoke Test
------------
--
--To test JRuby functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE ruby AS 'return "foo";' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language ruby for 'foobar'"
--JRuby for Apache Cassandra has not been installed correctly.
--
--
--Ruby require/include
----------------------
--
--You can use Ruby require and include in your scripts as in the following example:
--
--
--CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE ruby AS '
--require "bigdecimal"
--require "bigdecimal/math"
--
--include BigMath
--
--a = BigDecimal((PI(100)/2).to_s)
--
--return "foo " + a.to_s;
--' ;
--
--
--Notes / Java7 invokedynamic
-----------------------------
--
--See JRuby wiki pages https://github.com/jruby/jruby/wiki/ConfiguringJRuby and
--https://github.com/jruby/jruby/wiki/PerformanceTuning for more information and optimization tips.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jython/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/jython/README.txt
index bef3c83,bef3c83..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/jython/README.txt
+++ /dev/null
@@@ -1,33 -1,33 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable Jython
--
--Tested with version 2.3.5
--
--Installation
--------------
--
--1. Download Jython binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy the jar jython.jar from the Jython directory to $CASSANDRA_HOME/lib/jsr223/jython
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 10:58:18 Found scripting engine jython 2.5.3 - python 2.5 - language names: [python, jython]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--Smoke Test
------------
--
--To test Jython functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE python AS '''foo''' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language python for 'foobar'"
--Jython for Apache Cassandra has not been installed correctly.
--
--Notes / Java7 invokedynamic
-----------------------------
--
--Jython currently targets Java6 only. They want to switch to Java7 + invokedynamic in Jython 3.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/scala/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/scala/README.txt
index 7f5d6fe,7f5d6fe..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/scala/README.txt
+++ /dev/null
@@@ -1,37 -1,37 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable Scala
--
--Tested with version 2.11.2
--
--Installation
--------------
--
--1. Download Scala binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy the following jars from the Scala lib directory to $CASSANDRA_HOME/lib/jsr223/scala
-- scala-compiler.jar
-- scala-library.jar
-- scala-reflect.jar
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 11:42:35 Found scripting engine Scala Interpreter 1.0 - Scala version 2.11.2 - language names: [scala]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--Smoke Test
------------
--
--To test Scala functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE scala AS 'return "foo";' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language scala for 'foobar'"
--Scala for Apache Cassandra has not been installed correctly.
--
--Notes / Java7 invokedynamic
-----------------------------
--
--Scala 2.10 has Java6 support only. 2.11 has experimental invokedynamic support (use at your own risk!).
--2.12 introduces an upgrade directly to Java8 - see https://stackoverflow.com/questions/14285894/advantages-of-scala-emitting-bytecode-for-the-jvm-1-7
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
index 8743a20,0000000..47deafa
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
@@@ -1,246 -1,0 +1,240 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.*;
+import java.nio.ByteBuffer;
+import java.security.*;
+import java.security.cert.Certificate;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import javax.script.*;
+
++import jdk.nashorn.api.scripting.ClassFilter;
++import jdk.nashorn.api.scripting.NashornScriptEngine;
++import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+final class ScriptBasedUDFunction extends UDFunction
+{
- static final Map<String, Compilable> scriptEngines = new HashMap<>();
-
+ private static final ProtectionDomain protectionDomain;
+ private static final AccessControlContext accessControlContext;
+
+ //
+ // For scripted UDFs we have to rely on the security mechanisms of the scripting engine and
+ // SecurityManager - especially SecurityManager.checkPackageAccess(). Unlike Java-UDFs, strict checking
+ // of class access via the UDF class loader is not possible, since e.g. Nashorn builds its own class loader
+ // (jdk.nashorn.internal.runtime.ScriptLoader / jdk.nashorn.internal.runtime.NashornLoader) configured with
+ // a system class loader.
+ //
+ private static final String[] allowedPackagesArray =
+ {
+ // following required by jdk.nashorn.internal.objects.Global.initJavaAccess()
+ "",
+ "com",
+ "edu",
+ "java",
+ "javax",
+ "javafx",
+ "org",
+ // following required by Nashorn runtime
+ "java.lang",
+ "java.lang.invoke",
+ "java.lang.reflect",
+ "java.nio.charset",
+ "java.util",
+ "java.util.concurrent",
+ "javax.script",
+ "sun.reflect",
+ "jdk.internal.org.objectweb.asm.commons",
+ "jdk.nashorn.internal.runtime",
+ "jdk.nashorn.internal.runtime.linker",
+ // following required by Java Driver
+ "java.math",
+ "java.nio",
+ "java.text",
+ "com.google.common.base",
+ "com.google.common.collect",
+ "com.google.common.reflect",
+ // following required by UDF
+ "com.datastax.driver.core",
+ "com.datastax.driver.core.utils"
+ };
+
+ // use a JVM standard ExecutorService as DebuggableThreadPoolExecutor references internal
+ // classes, which triggers AccessControlException from the UDF sandbox
+ private static final UDFExecutorService executor =
+ new UDFExecutorService(new NamedThreadFactory("UserDefinedScriptFunctions",
+ Thread.MIN_PRIORITY,
+ udfClassLoader,
+ new SecurityThreadGroup("UserDefinedScriptFunctions",
+ Collections.unmodifiableSet(new HashSet<>(Arrays.asList(allowedPackagesArray))),
+ UDFunction::initializeThread)),
+ "userscripts");
+
++ private static final ClassFilter classFilter = clsName -> secureResource(clsName.replace('.', '/') + ".class");
++
++ private static final NashornScriptEngine scriptEngine;
++
++
+ static
+ {
+ ScriptEngineManager scriptEngineManager = new ScriptEngineManager();
- for (ScriptEngineFactory scriptEngineFactory : scriptEngineManager.getEngineFactories())
- {
- ScriptEngine scriptEngine = scriptEngineFactory.getScriptEngine();
- boolean compilable = scriptEngine instanceof Compilable;
- if (compilable)
- {
- logger.info("Found scripting engine {} {} - {} {} - language names: {}",
- scriptEngineFactory.getEngineName(), scriptEngineFactory.getEngineVersion(),
- scriptEngineFactory.getLanguageName(), scriptEngineFactory.getLanguageVersion(),
- scriptEngineFactory.getNames());
- for (String name : scriptEngineFactory.getNames())
- scriptEngines.put(name, (Compilable) scriptEngine);
- }
- }
++ ScriptEngine engine = scriptEngineManager.getEngineByName("nashorn");
++ NashornScriptEngineFactory factory = engine != null ? (NashornScriptEngineFactory) engine.getFactory() : null;
++ scriptEngine = factory != null ? (NashornScriptEngine) factory.getScriptEngine(new String[]{}, udfClassLoader, classFilter) : null;
+
+ try
+ {
+ protectionDomain = new ProtectionDomain(new CodeSource(new URL("udf", "localhost", 0, "/script", new URLStreamHandler()
+ {
+ protected URLConnection openConnection(URL u)
+ {
+ return null;
+ }
+ }), (Certificate[]) null), ThreadAwareSecurityManager.noPermissions);
+ }
+ catch (MalformedURLException e)
+ {
+ throw new RuntimeException(e);
+ }
+ accessControlContext = new AccessControlContext(new ProtectionDomain[]{ protectionDomain });
+ }
+
+ private final CompiledScript script;
+
+ ScriptBasedUDFunction(FunctionName name,
+ List<ColumnIdentifier> argNames,
+ List<AbstractType<?>> argTypes,
+ AbstractType<?> returnType,
+ boolean calledOnNullInput,
+ String language,
+ String body)
+ {
+ super(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
+
- Compilable scriptEngine = scriptEngines.get(language);
- if (scriptEngine == null)
++ if (!"JavaScript".equalsIgnoreCase(language) || scriptEngine == null)
+ throw new InvalidRequestException(String.format("Invalid language '%s' for function '%s'", language, name));
+
+ // execute compilation with no-permissions to prevent evil code e.g. via "static code blocks" / "class initialization"
+ try
+ {
+ this.script = AccessController.doPrivileged((PrivilegedExceptionAction<CompiledScript>) () -> scriptEngine.compile(body),
+ accessControlContext);
+ }
+ catch (PrivilegedActionException x)
+ {
+ Throwable e = x.getCause();
+ logger.info("Failed to compile function '{}' for language {}: ", name, language, e);
+ throw new InvalidRequestException(
+ String.format("Failed to compile function '%s' for language %s: %s", name, language, e));
+ }
+ }
+
+ protected ExecutorService executor()
+ {
+ return executor;
+ }
+
+ public ByteBuffer executeUserDefined(int protocolVersion, List<ByteBuffer> parameters)
+ {
+ Object[] params = new Object[argTypes.size()];
+ for (int i = 0; i < params.length; i++)
+ params[i] = compose(protocolVersion, i, parameters.get(i));
+
+ ScriptContext scriptContext = new SimpleScriptContext();
+ scriptContext.setAttribute("javax.script.filename", this.name.toString(), ScriptContext.ENGINE_SCOPE);
+ Bindings bindings = scriptContext.getBindings(ScriptContext.ENGINE_SCOPE);
+ for (int i = 0; i < params.length; i++)
+ bindings.put(argNames.get(i).toString(), params[i]);
+
+ Object result;
+ try
+ {
+ // How to prevent Class.forName() _without_ "help" from the script engine ?
+ // NOTE: Nashorn enforces a special permission to allow class-loading, which is not granted - so it's fine.
+
+ result = script.eval(scriptContext);
+ }
+ catch (ScriptException e)
+ {
+ throw new RuntimeException(e);
+ }
+ if (result == null)
+ return null;
+
+ Class<?> javaReturnType = UDHelper.asJavaClass(returnCodec);
+ Class<?> resultType = result.getClass();
+ if (!javaReturnType.isAssignableFrom(resultType))
+ {
+ if (result instanceof Number)
+ {
+ Number rNumber = (Number) result;
+ if (javaReturnType == Integer.class)
+ result = rNumber.intValue();
+ else if (javaReturnType == Long.class)
+ result = rNumber.longValue();
+ else if (javaReturnType == Short.class)
+ result = rNumber.shortValue();
+ else if (javaReturnType == Byte.class)
+ result = rNumber.byteValue();
+ else if (javaReturnType == Float.class)
+ result = rNumber.floatValue();
+ else if (javaReturnType == Double.class)
+ result = rNumber.doubleValue();
+ else if (javaReturnType == BigInteger.class)
+ {
+ if (javaReturnType == Integer.class)
+ result = rNumber.intValue();
+ else if (javaReturnType == Short.class)
+ result = rNumber.shortValue();
+ else if (javaReturnType == Byte.class)
+ result = rNumber.byteValue();
+ else if (javaReturnType == Long.class)
+ result = rNumber.longValue();
+ else if (javaReturnType == Float.class)
+ result = rNumber.floatValue();
+ else if (javaReturnType == Double.class)
+ result = rNumber.doubleValue();
+ else if (javaReturnType == BigInteger.class)
+ {
+ if (rNumber instanceof BigDecimal)
+ result = ((BigDecimal) rNumber).toBigInteger();
+ else if (rNumber instanceof Double || rNumber instanceof Float)
+ result = new BigDecimal(rNumber.toString()).toBigInteger();
+ else
+ result = BigInteger.valueOf(rNumber.longValue());
+ }
+ else if (javaReturnType == BigDecimal.class)
+ // String c'tor of BigDecimal is more accurate than valueOf(double)
+ result = new BigDecimal(rNumber.toString());
+ }
+ else if (javaReturnType == BigDecimal.class)
+ // String c'tor of BigDecimal is more accurate than valueOf(double)
+ result = new BigDecimal(rNumber.toString());
+ }
+ }
+
+ return decompose(protocolVersion, result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
index af3c894,d3050a5..9c931e8
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
@@@ -382,47 -501,4 +382,46 @@@ public class UFScriptTest extends CQLTe
row(1, expected1, expected2));
}
}
+
+ @Test
+ public void testJavascriptDisabled() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key int primary key, val double)");
+
+ DatabaseDescriptor.enableScriptedUserDefinedFunctions(false);
+ try
+ {
- assertInvalid("double",
- "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".assertNotEnabled(val double) " +
++ assertInvalid("CREATE OR REPLACE FUNCTION " + KEYSPACE + ".assertNotEnabled(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS 'Math.sin(val);';");
+ }
+ finally
+ {
+ DatabaseDescriptor.enableScriptedUserDefinedFunctions(true);
+ }
+ }
+
+ @Test
+ public void testJavascriptCompileFailure() throws Throwable
+ {
+ assertInvalidMessage("Failed to compile function 'cql_test_keyspace.scrinv'",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS 'foo bar';");
+ }
+
+ @Test
+ public void testScriptInvalidLanguage() throws Throwable
+ {
+ assertInvalidMessage("Invalid language 'artificial_intelligence' for function 'cql_test_keyspace.scrinv'",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE artificial_intelligence\n" +
+ "AS 'question for 42?';");
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
index 5c7ca2b,0000000..4e45a8a
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
@@@ -1,258 -1,0 +1,268 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.validation.entities;
+
+import java.security.AccessControlException;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.functions.UDHelper;
+import org.apache.cassandra.exceptions.FunctionExecutionException;
+import org.apache.cassandra.service.ClientWarn;
+
+public class UFSecurityTest extends CQLTester
+{
+ @Test
+ public void testSecurityPermissions() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key int primary key, dval double)");
+ execute("INSERT INTO %s (key, dval) VALUES (?, ?)", 1, 1d);
+
+ // Java UDFs
+
+ try
+ {
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS 'System.getProperty(\"foo.bar.baz\"); return 0d;';");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ Assert.fail();
+ }
+ catch (FunctionExecutionException e)
+ {
+ assertAccessControlException("System.getProperty(\"foo.bar.baz\"); return 0d;", e);
+ }
+
+ String[][] typesAndSources =
+ {
+ {"", "try { Class.forName(\"" + UDHelper.class.getName() + "\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;"},
+ {"sun.misc.Unsafe", "sun.misc.Unsafe.getUnsafe(); return 0d;"},
+ {"", "try { Class.forName(\"sun.misc.Unsafe\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;"},
+ {"java.nio.file.FileSystems", "try {" +
+ " java.nio.file.FileSystems.getDefault(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.nio.channels.FileChannel", "try {" +
+ " java.nio.channels.FileChannel.open(java.nio.file.FileSystems.getDefault().getPath(\"/etc/passwd\")).close(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.nio.channels.SocketChannel", "try {" +
+ " java.nio.channels.SocketChannel.open().close(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.io.FileInputStream", "try {" +
+ " new java.io.FileInputStream(\"./foobar\").close(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.lang.Runtime", "try {" +
+ " java.lang.Runtime.getRuntime(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"org.apache.cassandra.service.StorageService",
+ "try {" +
+ " org.apache.cassandra.service.StorageService v = org.apache.cassandra.service.StorageService.instance; v.isShutdown(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.net.ServerSocket", "try {" +
+ " new java.net.ServerSocket().bind(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.io.FileOutputStream","try {" +
+ " new java.io.FileOutputStream(\".foo\"); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.lang.Runtime", "try {" +
+ " java.lang.Runtime.getRuntime().exec(\"/tmp/foo\"); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'}
+ };
+
+ for (String[] typeAndSource : typesAndSources)
+ {
+ assertInvalidMessage(typeAndSource[0] + " cannot be resolved",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".invalid_class_access(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS '" + typeAndSource[1] + "';");
+ }
+
+ // JavaScript UDFs
+
+ try
+ {
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS 'org.apache.cassandra.service.StorageService.instance.isShutdown(); 0;';");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ Assert.fail("Javascript security check failed");
+ }
+ catch (FunctionExecutionException e)
+ {
+ assertAccessControlException("", e);
+ }
+
+ String[] javascript =
+ {
+ "java.lang.management.ManagmentFactory.getThreadMXBean(); 0;",
+ "new java.io.FileInputStream(\"/tmp/foo\"); 0;",
+ "new java.io.FileOutputStream(\"/tmp/foo\"); 0;",
+ "java.nio.file.FileSystems.getDefault().createFileExclusively(\"./foo_bar_baz\"); 0;",
+ "java.nio.channels.FileChannel.open(java.nio.file.FileSystems.getDefault().getPath(\"/etc/passwd\")); 0;",
+ "java.nio.channels.SocketChannel.open(); 0;",
+ "new java.net.ServerSocket().bind(null); 0;",
+ "var thread = new java.lang.Thread(); thread.start(); 0;",
+ "java.lang.System.getProperty(\"foo.bar.baz\"); 0;",
- "java.lang.Class.forName(\"java.lang.System\"); 0;",
+ "java.lang.Runtime.getRuntime().exec(\"/tmp/foo\"); 0;",
+ "java.lang.Runtime.getRuntime().loadLibrary(\"foobar\"); 0;",
+ "java.lang.Runtime.getRuntime().loadLibrary(\"foobar\"); 0;",
+ // TODO these (ugly) calls are still possible - these can consume CPU (as one could do with an evil loop, too)
+// "java.lang.Runtime.getRuntime().traceMethodCalls(true); 0;",
+// "java.lang.Runtime.getRuntime().gc(); 0;",
+// "java.lang.Runtime.getRuntime(); 0;",
+ };
+
+ for (String script : javascript)
+ {
+ try
+ {
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS '" + script + "';");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ Assert.fail("Javascript security check failed: " + script);
+ }
+ catch (FunctionExecutionException e)
+ {
+ assertAccessControlException(script, e);
+ }
+ }
++
++ String script = "java.lang.Class.forName(\"java.lang.System\"); 0;";
++ String fName = createFunction(KEYSPACE_PER_TEST, "double",
++ "CREATE OR REPLACE FUNCTION %s(val double) " +
++ "RETURNS NULL ON NULL INPUT " +
++ "RETURNS double " +
++ "LANGUAGE javascript\n" +
++ "AS '" + script + "';");
++ assertInvalidThrowMessage("Java reflection not supported when class filter is present",
++ FunctionExecutionException.class,
++ "SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ }
+
+ private static void assertAccessControlException(String script, FunctionExecutionException e)
+ {
+ for (Throwable t = e; t != null && t != t.getCause(); t = t.getCause())
+ if (t instanceof AccessControlException)
+ return;
+ Assert.fail("no AccessControlException for " + script + " (got " + e + ')');
+ }
+
+ @Test
+ public void testAmokUDF() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key int primary key, dval double)");
+ execute("INSERT INTO %s (key, dval) VALUES (?, ?)", 1, 1d);
+
+ long udfWarnTimeout = DatabaseDescriptor.getUserDefinedFunctionWarnTimeout();
+ long udfFailTimeout = DatabaseDescriptor.getUserDefinedFunctionFailTimeout();
+ int maxTries = 5;
+ for (int i = 1; i <= maxTries; i++)
+ {
+ try
+ {
+ // short timeout
+ DatabaseDescriptor.setUserDefinedFunctionWarnTimeout(10);
+ DatabaseDescriptor.setUserDefinedFunctionFailTimeout(250);
+ // don't kill the unit test... - default policy is "die"
+ DatabaseDescriptor.setUserFunctionTimeoutPolicy(Config.UserFunctionTimeoutPolicy.ignore);
+
+ ClientWarn.instance.captureWarnings();
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS 'long t=System.currentTimeMillis()+110; while (t>System.currentTimeMillis()) { }; return 0d;'");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ List<String> warnings = ClientWarn.instance.getWarnings();
+ Assert.assertNotNull(warnings);
+ Assert.assertFalse(warnings.isEmpty());
+ ClientWarn.instance.resetWarnings();
+
+ // Java UDF
+
+ fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS 'long t=System.currentTimeMillis()+500; while (t>System.currentTimeMillis()) { }; return 0d;';");
+ assertInvalidMessage("ran longer than 250ms", "SELECT " + fName + "(dval) FROM %s WHERE key=1");
+
+ // Javascript UDF
+
+ fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVASCRIPT\n" +
+ "AS 'var t=java.lang.System.currentTimeMillis()+500; while (t>java.lang.System.currentTimeMillis()) { }; 0;';");
+ assertInvalidMessage("ran longer than 250ms", "SELECT " + fName + "(dval) FROM %s WHERE key=1");
+
+ return;
+ }
+ catch (Error | RuntimeException e)
+ {
+ if (i == maxTries)
+ throw e;
+ }
+ finally
+ {
+ // reset to defaults
+ DatabaseDescriptor.setUserDefinedFunctionWarnTimeout(udfWarnTimeout);
+ DatabaseDescriptor.setUserDefinedFunctionFailTimeout(udfFailTimeout);
+ }
+ }
+ }
+
+}
[05/10] cassandra git commit: Merge branch 'cassandra-2.2' into
cassandra-3.0
Posted by sn...@apache.org.
Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6e716c6d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6e716c6d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6e716c6d
Branch: refs/heads/cassandra-3.X
Commit: 6e716c6da41900950e32a5549b3bb1e858ecad18
Parents: c0765ed 6f360b6
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu Jan 5 22:20:31 2017 +0100
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu Jan 5 22:20:31 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 3 ++
doc/cql3/CQL.textile | 2 +-
lib/jsr223/clojure/README.txt | 8 ---
lib/jsr223/groovy/README.txt | 35 -------------
lib/jsr223/jaskell/README.txt | 5 --
lib/jsr223/jruby/README.txt | 54 --------------------
lib/jsr223/jython/README.txt | 33 ------------
lib/jsr223/scala/README.txt | 37 --------------
.../cql3/functions/ScriptBasedUDFunction.java | 30 +++++------
.../cql3/validation/entities/UFScriptTest.java | 3 +-
.../validation/entities/UFSecurityTest.java | 12 ++++-
12 files changed, 29 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 666a771,b41313d..77a310d
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,23 -1,5 +1,24 @@@
-2.2.9
+3.0.11
+ * Fixed flacky SSTableRewriterTest: check file counts before calling validateCFS (CASSANDRA-12348)
+ * Fix deserialization of 2.x DeletedCells (CASSANDRA-12620)
+ * Add parent repair session id to anticompaction log message (CASSANDRA-12186)
+ * Improve contention handling on failure to acquire MV lock for streaming and hints (CASSANDRA-12905)
+ * Fix DELETE and UPDATE queries with empty IN restrictions (CASSANDRA-12829)
+ * Mark MVs as built after successful bootstrap (CASSANDRA-12984)
+ * Estimated TS drop-time histogram updated with Cell.NO_DELETION_TIME (CASSANDRA-13040)
+ * Nodetool compactionstats fails with NullPointerException (CASSANDRA-13021)
+ * Thread local pools never cleaned up (CASSANDRA-13033)
+ * Set RPC_READY to false when draining or if a node is marked as shutdown (CASSANDRA-12781)
+ * CQL often queries static columns unnecessarily (CASSANDRA-12768)
+ * Make sure sstables only get committed when it's safe to discard commit log records (CASSANDRA-12956)
+ * Reject default_time_to_live option when creating or altering MVs (CASSANDRA-12868)
+ * Nodetool should use a more sane max heap size (CASSANDRA-12739)
+ * LocalToken ensures token values are cloned on heap (CASSANDRA-12651)
+ * AnticompactionRequestSerializer serializedSize is incorrect (CASSANDRA-12934)
+ * Prevent reloading of logback.xml from UDF sandbox (CASSANDRA-12535)
+ * Reenable HeapPool (CASSANDRA-12900)
+Merged from 2.2:
+ * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
* Fix DynamicEndpointSnitch noop in multi-datacenter situations (CASSANDRA-13074)
* cqlsh copy-from: encode column names to avoid primary key parsing errors (CASSANDRA-12909)
* Temporarily fix bug that creates commit log when running offline tools (CASSANDRA-8616)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/NEWS.txt
----------------------------------------------------------------------
diff --cc NEWS.txt
index 32b5084,37949a1..b4e6551
--- a/NEWS.txt
+++ b/NEWS.txt
@@@ -13,77 -13,20 +13,80 @@@ restore snapshots created with the prev
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
-2.2.9
+3.0.11
=====
+Upgrading
+---------
+ - Nothing specific to this release, but please see previous versions upgrading section,
+ especially if you are upgrading from 2.2.
+ - Specifying the default_time_to_live option when creating or altering a
+ materialized view was erroneously accepted (and ignored). It is now
+ properly rejected.
++ - Only Java and JavaScript are now supported UDF languages.
++ The sandbox in 3.0 already prevented the use of script languages except Java
++ and JavaScript.
+
+3.0.10
+=====
+
+Upgrading
+---------
+ - memtable_allocation_type: offheap_buffers is no longer allowed to be specified in the 3.0 series.
+ This was an oversight that can cause segfaults. Offheap was re-introduced in 3.4 see CASSANDRA-11039
+ and CASSANDRA-9472 for details.
+
+3.0.9
+=====
+
+Upgrading
+---------
+ - The ReversedType behaviour has been corrected for clustering columns of
+ BYTES type containing empty value. Scrub should be run on the existing
+ SSTables containing a descending clustering column of BYTES type to correct
+ their ordering. See CASSANDRA-12127 for more details.
+
+3.0.8
+=====
+
+Upgrading
+---------
+ - Ec2MultiRegionSnitch will no longer automatically set broadcast_rpc_address
+ to the public instance IP if this property is defined on cassandra.yaml.
+
+3.0.7
+=====
+
+Upgrading
+---------
+ - A maximum size for SSTables values has been introduced, to prevent out of memory
+ exceptions when reading corrupt SSTables. This maximum size can be set via
+ max_value_size_in_mb in cassandra.yaml. The default is 256MB, which matches the default
+ value of native_transport_max_frame_size_in_mb. SSTables will be considered corrupt if
+ they contain values whose size exceeds this limit. See CASSANDRA-9530 for more details.
+
Deprecation
-----------
+ - DateTieredCompactionStrategy has been deprecated - new tables should use
+ TimeWindowCompactionStrategy. Note that migrating an existing DTCS-table to TWCS might
+ cause increased compaction load for a while after the migration so make sure you run
+ tests before migrating. Read CASSANDRA-9666 for background on this.
-(See note about the new feature User-Defined-Functions in 2.2.0.)
+New features
+------------
+ - TimeWindowCompactionStrategy has been added. This has proven to be a better approach
+ to time series compaction and new tables should use this instead of DTCS. See
+ CASSANDRA-9666 for details.
-Since the security manager added in 3.0 only allows Java and JavaScript
-UDFs to be run, UDFs for other languages are deprecated and support for
-non-Java and non-JavaScript UDFs is deprecated in 2.2 and has been removed
-in version 3.0.11.
+3.0.6
+=====
+
+New features
+------------
+ - JSON timestamps are now in UTC and contain the timezone information, see
+ CASSANDRA-11137 for more details.
-2.2.8
+3.0.5
=====
Upgrading
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/doc/cql3/CQL.textile
----------------------------------------------------------------------
diff --cc doc/cql3/CQL.textile
index 2a37452,af584d0..2544878
--- a/doc/cql3/CQL.textile
+++ b/doc/cql3/CQL.textile
@@@ -1977,7 -1910,7 +1977,7 @@@ SELECT AVG(players) FROM plays
h2(#udfs). User-Defined Functions
--User-defined functions allow execution of user-provided code in Cassandra. By default, Cassandra supports defining functions in _Java_ and _JavaScript_. Support for other JSR 223 compliant scripting languages (such as Python, Ruby, and Scala) can be added by adding a JAR to the classpath.
++User-defined functions allow execution of user-provided code in Cassandra. By default, Cassandra supports defining functions in _Java_ and _JavaScript_. Support for other JSR 223 compliant scripting languages (such as Python, Ruby, and Scala) has been removed in 3.0.11.
UDFs are part of the Cassandra schema. As such, they are automatically propagated to all nodes in the cluster.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/clojure/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/clojure/README.txt
index 7ed7551,7ed7551..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/clojure/README.txt
+++ /dev/null
@@@ -1,8 -1,8 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Unfortunately the JSR-223 support provided by the project https://github.com/ato/clojure-jsr223
--and the related ones do not provide compileable script support.
--
--The JSR-223 javax.script.Compilable implementation takes source file names or readers but not script sources
--as all other JSR-223 implementations do.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/groovy/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/groovy/README.txt
index 09fef93,09fef93..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/groovy/README.txt
+++ /dev/null
@@@ -1,35 -1,35 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable Groovy
--
--Tested with version 2.3.6
--
--Installation
--------------
--
--1. Download Groovy binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy the jar groovy-all-2.3.6-indy.jar from the Groovy embeddable directory to $CASSANDRA_HOME/lib/jsr223/groovy
-- "indy" means "invokedynamic" and is a JVM instruction for scripting languages new to Java 7.
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 10:49:45 Found scripting engine Groovy Scripting Engine 2.0 - Groovy 2.3.6 - language names: [groovy, Groovy]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--Smoke Test
------------
--
--To test Groovy functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE groovy AS 'return "foo";' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language groovy for 'foobar'"
--Groovy for Apache Cassandra has not been installed correctly.
--
--Notes / Java7 invokedynamic
-----------------------------
--
--Groovy provides jars that support invokedynamic bytecode instruction. These jars are whose ending with
--"-indy.jar".
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jaskell/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/jaskell/README.txt
index 53e942e,53e942e..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/jaskell/README.txt
+++ /dev/null
@@@ -1,5 -1,5 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Unfortunately Jaskell JSR-223 support is quite old and the Jaskell engine seems to be quite
--unsupported. If you find a solution, please open a ticket at Apache Cassandra JIRA.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jruby/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/jruby/README.txt
index cbc12dc,cbc12dc..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/jruby/README.txt
+++ /dev/null
@@@ -1,54 -1,54 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable JRuby
--
--Tested with version 1.7.15
--
--Installation
--------------
--
--1. Download JRuby binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy everything from the JRuby lib directory to $CASSANDRA_HOME/lib/jsr223/jruby
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 10:29:03 Found scripting engine JSR 223 JRuby Engine 1.7.15 - ruby jruby 1.7.15 - language names: [ruby, jruby]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--
--Smoke Test
------------
--
--To test JRuby functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE ruby AS 'return "foo";' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language ruby for 'foobar'"
--JRuby for Apache Cassandra has not been installed correctly.
--
--
--Ruby require/include
----------------------
--
--You can use Ruby require and include in your scripts as in the following example:
--
--
--CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE ruby AS '
--require "bigdecimal"
--require "bigdecimal/math"
--
--include BigMath
--
--a = BigDecimal((PI(100)/2).to_s)
--
--return "foo " + a.to_s;
--' ;
--
--
--Notes / Java7 invokedynamic
-----------------------------
--
--See JRuby wiki pages https://github.com/jruby/jruby/wiki/ConfiguringJRuby and
--https://github.com/jruby/jruby/wiki/PerformanceTuning for more information and optimization tips.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/jython/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/jython/README.txt
index bef3c83,bef3c83..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/jython/README.txt
+++ /dev/null
@@@ -1,33 -1,33 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable Jython
--
--Tested with version 2.3.5
--
--Installation
--------------
--
--1. Download Jython binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy the jar jython.jar from the Jython directory to $CASSANDRA_HOME/lib/jsr223/jython
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 10:58:18 Found scripting engine jython 2.5.3 - python 2.5 - language names: [python, jython]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--Smoke Test
------------
--
--To test Jython functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE python AS '''foo''' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language python for 'foobar'"
--Jython for Apache Cassandra has not been installed correctly.
--
--Notes / Java7 invokedynamic
-----------------------------
--
--Jython currently targets Java6 only. They want to switch to Java7 + invokedynamic in Jython 3.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/lib/jsr223/scala/README.txt
----------------------------------------------------------------------
diff --cc lib/jsr223/scala/README.txt
index 7f5d6fe,7f5d6fe..0000000
deleted file mode 100644,100644
--- a/lib/jsr223/scala/README.txt
+++ /dev/null
@@@ -1,37 -1,37 +1,0 @@@
--Apache Cassandra User-Defined-Functions JSR 223 scripting
--=========================================================
--
--Using JSR-223 capable Scala
--
--Tested with version 2.11.2
--
--Installation
--------------
--
--1. Download Scala binary release
--2. Unpack the downloaded archive into a temporary directory
--3. Copy the following jars from the Scala lib directory to $CASSANDRA_HOME/lib/jsr223/scala
-- scala-compiler.jar
-- scala-library.jar
-- scala-reflect.jar
--4. Restart your Cassandra daemon if it's already running
--
--Cassandra log should contain a line like this:
-- INFO 11:42:35 Found scripting engine Scala Interpreter 1.0 - Scala version 2.11.2 - language names: [scala]
--Such a line appears when you already have scripted UDFs in your system or add a scripted UDF for the first time (see below).
--
--Smoke Test
------------
--
--To test Scala functionality, open cqlsh and execute the following command:
-- CREATE OR REPLACE FUNCTION foobar ( input text ) RETURNS text LANGUAGE scala AS 'return "foo";' ;
--
--If you get the error
-- code=2200 [Invalid query] message="Invalid language scala for 'foobar'"
--Scala for Apache Cassandra has not been installed correctly.
--
--Notes / Java7 invokedynamic
-----------------------------
--
--Scala 2.10 has Java6 support only. 2.11 has experimental invokedynamic support (use at your own risk!).
--2.12 introduces an upgrade directly to Java8 - see https://stackoverflow.com/questions/14285894/advantages-of-scala-emitting-bytecode-for-the-jvm-1-7
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
index 8743a20,0000000..47deafa
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
@@@ -1,246 -1,0 +1,240 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3.functions;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.net.*;
+import java.nio.ByteBuffer;
+import java.security.*;
+import java.security.cert.Certificate;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import javax.script.*;
+
++import jdk.nashorn.api.scripting.ClassFilter;
++import jdk.nashorn.api.scripting.NashornScriptEngine;
++import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
+import org.apache.cassandra.concurrent.NamedThreadFactory;
+import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+
+final class ScriptBasedUDFunction extends UDFunction
+{
- static final Map<String, Compilable> scriptEngines = new HashMap<>();
-
+ private static final ProtectionDomain protectionDomain;
+ private static final AccessControlContext accessControlContext;
+
+ //
+ // For scripted UDFs we have to rely on the security mechanisms of the scripting engine and
+ // SecurityManager - especially SecurityManager.checkPackageAccess(). Unlike Java-UDFs, strict checking
+ // of class access via the UDF class loader is not possible, since e.g. Nashorn builds its own class loader
+ // (jdk.nashorn.internal.runtime.ScriptLoader / jdk.nashorn.internal.runtime.NashornLoader) configured with
+ // a system class loader.
+ //
+ private static final String[] allowedPackagesArray =
+ {
+ // following required by jdk.nashorn.internal.objects.Global.initJavaAccess()
+ "",
+ "com",
+ "edu",
+ "java",
+ "javax",
+ "javafx",
+ "org",
+ // following required by Nashorn runtime
+ "java.lang",
+ "java.lang.invoke",
+ "java.lang.reflect",
+ "java.nio.charset",
+ "java.util",
+ "java.util.concurrent",
+ "javax.script",
+ "sun.reflect",
+ "jdk.internal.org.objectweb.asm.commons",
+ "jdk.nashorn.internal.runtime",
+ "jdk.nashorn.internal.runtime.linker",
+ // following required by Java Driver
+ "java.math",
+ "java.nio",
+ "java.text",
+ "com.google.common.base",
+ "com.google.common.collect",
+ "com.google.common.reflect",
+ // following required by UDF
+ "com.datastax.driver.core",
+ "com.datastax.driver.core.utils"
+ };
+
+ // use a JVM standard ExecutorService as DebuggableThreadPoolExecutor references internal
+ // classes, which triggers AccessControlException from the UDF sandbox
+ private static final UDFExecutorService executor =
+ new UDFExecutorService(new NamedThreadFactory("UserDefinedScriptFunctions",
+ Thread.MIN_PRIORITY,
+ udfClassLoader,
+ new SecurityThreadGroup("UserDefinedScriptFunctions",
+ Collections.unmodifiableSet(new HashSet<>(Arrays.asList(allowedPackagesArray))),
+ UDFunction::initializeThread)),
+ "userscripts");
+
++ private static final ClassFilter classFilter = clsName -> secureResource(clsName.replace('.', '/') + ".class");
++
++ private static final NashornScriptEngine scriptEngine;
++
++
+ static
+ {
+ ScriptEngineManager scriptEngineManager = new ScriptEngineManager();
- for (ScriptEngineFactory scriptEngineFactory : scriptEngineManager.getEngineFactories())
- {
- ScriptEngine scriptEngine = scriptEngineFactory.getScriptEngine();
- boolean compilable = scriptEngine instanceof Compilable;
- if (compilable)
- {
- logger.info("Found scripting engine {} {} - {} {} - language names: {}",
- scriptEngineFactory.getEngineName(), scriptEngineFactory.getEngineVersion(),
- scriptEngineFactory.getLanguageName(), scriptEngineFactory.getLanguageVersion(),
- scriptEngineFactory.getNames());
- for (String name : scriptEngineFactory.getNames())
- scriptEngines.put(name, (Compilable) scriptEngine);
- }
- }
++ ScriptEngine engine = scriptEngineManager.getEngineByName("nashorn");
++ NashornScriptEngineFactory factory = engine != null ? (NashornScriptEngineFactory) engine.getFactory() : null;
++ scriptEngine = factory != null ? (NashornScriptEngine) factory.getScriptEngine(new String[]{}, udfClassLoader, classFilter) : null;
+
+ try
+ {
+ protectionDomain = new ProtectionDomain(new CodeSource(new URL("udf", "localhost", 0, "/script", new URLStreamHandler()
+ {
+ protected URLConnection openConnection(URL u)
+ {
+ return null;
+ }
+ }), (Certificate[]) null), ThreadAwareSecurityManager.noPermissions);
+ }
+ catch (MalformedURLException e)
+ {
+ throw new RuntimeException(e);
+ }
+ accessControlContext = new AccessControlContext(new ProtectionDomain[]{ protectionDomain });
+ }
+
+ private final CompiledScript script;
+
+ ScriptBasedUDFunction(FunctionName name,
+ List<ColumnIdentifier> argNames,
+ List<AbstractType<?>> argTypes,
+ AbstractType<?> returnType,
+ boolean calledOnNullInput,
+ String language,
+ String body)
+ {
+ super(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
+
- Compilable scriptEngine = scriptEngines.get(language);
- if (scriptEngine == null)
++ if (!"JavaScript".equalsIgnoreCase(language) || scriptEngine == null)
+ throw new InvalidRequestException(String.format("Invalid language '%s' for function '%s'", language, name));
+
+ // execute compilation with no-permissions to prevent evil code e.g. via "static code blocks" / "class initialization"
+ try
+ {
+ this.script = AccessController.doPrivileged((PrivilegedExceptionAction<CompiledScript>) () -> scriptEngine.compile(body),
+ accessControlContext);
+ }
+ catch (PrivilegedActionException x)
+ {
+ Throwable e = x.getCause();
+ logger.info("Failed to compile function '{}' for language {}: ", name, language, e);
+ throw new InvalidRequestException(
+ String.format("Failed to compile function '%s' for language %s: %s", name, language, e));
+ }
+ }
+
+ protected ExecutorService executor()
+ {
+ return executor;
+ }
+
+ public ByteBuffer executeUserDefined(int protocolVersion, List<ByteBuffer> parameters)
+ {
+ Object[] params = new Object[argTypes.size()];
+ for (int i = 0; i < params.length; i++)
+ params[i] = compose(protocolVersion, i, parameters.get(i));
+
+ ScriptContext scriptContext = new SimpleScriptContext();
+ scriptContext.setAttribute("javax.script.filename", this.name.toString(), ScriptContext.ENGINE_SCOPE);
+ Bindings bindings = scriptContext.getBindings(ScriptContext.ENGINE_SCOPE);
+ for (int i = 0; i < params.length; i++)
+ bindings.put(argNames.get(i).toString(), params[i]);
+
+ Object result;
+ try
+ {
+ // How to prevent Class.forName() _without_ "help" from the script engine ?
+ // NOTE: Nashorn enforces a special permission to allow class-loading, which is not granted - so it's fine.
+
+ result = script.eval(scriptContext);
+ }
+ catch (ScriptException e)
+ {
+ throw new RuntimeException(e);
+ }
+ if (result == null)
+ return null;
+
+ Class<?> javaReturnType = UDHelper.asJavaClass(returnCodec);
+ Class<?> resultType = result.getClass();
+ if (!javaReturnType.isAssignableFrom(resultType))
+ {
+ if (result instanceof Number)
+ {
+ Number rNumber = (Number) result;
+ if (javaReturnType == Integer.class)
+ result = rNumber.intValue();
+ else if (javaReturnType == Long.class)
+ result = rNumber.longValue();
+ else if (javaReturnType == Short.class)
+ result = rNumber.shortValue();
+ else if (javaReturnType == Byte.class)
+ result = rNumber.byteValue();
+ else if (javaReturnType == Float.class)
+ result = rNumber.floatValue();
+ else if (javaReturnType == Double.class)
+ result = rNumber.doubleValue();
+ else if (javaReturnType == BigInteger.class)
+ {
+ if (javaReturnType == Integer.class)
+ result = rNumber.intValue();
+ else if (javaReturnType == Short.class)
+ result = rNumber.shortValue();
+ else if (javaReturnType == Byte.class)
+ result = rNumber.byteValue();
+ else if (javaReturnType == Long.class)
+ result = rNumber.longValue();
+ else if (javaReturnType == Float.class)
+ result = rNumber.floatValue();
+ else if (javaReturnType == Double.class)
+ result = rNumber.doubleValue();
+ else if (javaReturnType == BigInteger.class)
+ {
+ if (rNumber instanceof BigDecimal)
+ result = ((BigDecimal) rNumber).toBigInteger();
+ else if (rNumber instanceof Double || rNumber instanceof Float)
+ result = new BigDecimal(rNumber.toString()).toBigInteger();
+ else
+ result = BigInteger.valueOf(rNumber.longValue());
+ }
+ else if (javaReturnType == BigDecimal.class)
+ // String c'tor of BigDecimal is more accurate than valueOf(double)
+ result = new BigDecimal(rNumber.toString());
+ }
+ else if (javaReturnType == BigDecimal.class)
+ // String c'tor of BigDecimal is more accurate than valueOf(double)
+ result = new BigDecimal(rNumber.toString());
+ }
+ }
+
+ return decompose(protocolVersion, result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
index af3c894,d3050a5..9c931e8
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
@@@ -382,47 -501,4 +382,46 @@@ public class UFScriptTest extends CQLTe
row(1, expected1, expected2));
}
}
+
+ @Test
+ public void testJavascriptDisabled() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key int primary key, val double)");
+
+ DatabaseDescriptor.enableScriptedUserDefinedFunctions(false);
+ try
+ {
- assertInvalid("double",
- "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".assertNotEnabled(val double) " +
++ assertInvalid("CREATE OR REPLACE FUNCTION " + KEYSPACE + ".assertNotEnabled(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS 'Math.sin(val);';");
+ }
+ finally
+ {
+ DatabaseDescriptor.enableScriptedUserDefinedFunctions(true);
+ }
+ }
+
+ @Test
+ public void testJavascriptCompileFailure() throws Throwable
+ {
+ assertInvalidMessage("Failed to compile function 'cql_test_keyspace.scrinv'",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS 'foo bar';");
+ }
+
+ @Test
+ public void testScriptInvalidLanguage() throws Throwable
+ {
+ assertInvalidMessage("Invalid language 'artificial_intelligence' for function 'cql_test_keyspace.scrinv'",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".scrinv(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE artificial_intelligence\n" +
+ "AS 'question for 42?';");
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6e716c6d/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
index 5c7ca2b,0000000..4e45a8a
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/UFSecurityTest.java
@@@ -1,258 -1,0 +1,268 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3.validation.entities;
+
+import java.security.AccessControlException;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.functions.UDHelper;
+import org.apache.cassandra.exceptions.FunctionExecutionException;
+import org.apache.cassandra.service.ClientWarn;
+
+public class UFSecurityTest extends CQLTester
+{
+ @Test
+ public void testSecurityPermissions() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key int primary key, dval double)");
+ execute("INSERT INTO %s (key, dval) VALUES (?, ?)", 1, 1d);
+
+ // Java UDFs
+
+ try
+ {
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS 'System.getProperty(\"foo.bar.baz\"); return 0d;';");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ Assert.fail();
+ }
+ catch (FunctionExecutionException e)
+ {
+ assertAccessControlException("System.getProperty(\"foo.bar.baz\"); return 0d;", e);
+ }
+
+ String[][] typesAndSources =
+ {
+ {"", "try { Class.forName(\"" + UDHelper.class.getName() + "\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;"},
+ {"sun.misc.Unsafe", "sun.misc.Unsafe.getUnsafe(); return 0d;"},
+ {"", "try { Class.forName(\"sun.misc.Unsafe\"); } catch (Exception e) { throw new RuntimeException(e); } return 0d;"},
+ {"java.nio.file.FileSystems", "try {" +
+ " java.nio.file.FileSystems.getDefault(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.nio.channels.FileChannel", "try {" +
+ " java.nio.channels.FileChannel.open(java.nio.file.FileSystems.getDefault().getPath(\"/etc/passwd\")).close(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.nio.channels.SocketChannel", "try {" +
+ " java.nio.channels.SocketChannel.open().close(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.io.FileInputStream", "try {" +
+ " new java.io.FileInputStream(\"./foobar\").close(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.lang.Runtime", "try {" +
+ " java.lang.Runtime.getRuntime(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"org.apache.cassandra.service.StorageService",
+ "try {" +
+ " org.apache.cassandra.service.StorageService v = org.apache.cassandra.service.StorageService.instance; v.isShutdown(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.net.ServerSocket", "try {" +
+ " new java.net.ServerSocket().bind(); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.io.FileOutputStream","try {" +
+ " new java.io.FileOutputStream(\".foo\"); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'},
+ {"java.lang.Runtime", "try {" +
+ " java.lang.Runtime.getRuntime().exec(\"/tmp/foo\"); return 0d;" +
+ "} catch (Exception t) {" +
+ " throw new RuntimeException(t);" +
+ '}'}
+ };
+
+ for (String[] typeAndSource : typesAndSources)
+ {
+ assertInvalidMessage(typeAndSource[0] + " cannot be resolved",
+ "CREATE OR REPLACE FUNCTION " + KEYSPACE + ".invalid_class_access(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS '" + typeAndSource[1] + "';");
+ }
+
+ // JavaScript UDFs
+
+ try
+ {
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS 'org.apache.cassandra.service.StorageService.instance.isShutdown(); 0;';");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ Assert.fail("Javascript security check failed");
+ }
+ catch (FunctionExecutionException e)
+ {
+ assertAccessControlException("", e);
+ }
+
+ String[] javascript =
+ {
+ "java.lang.management.ManagmentFactory.getThreadMXBean(); 0;",
+ "new java.io.FileInputStream(\"/tmp/foo\"); 0;",
+ "new java.io.FileOutputStream(\"/tmp/foo\"); 0;",
+ "java.nio.file.FileSystems.getDefault().createFileExclusively(\"./foo_bar_baz\"); 0;",
+ "java.nio.channels.FileChannel.open(java.nio.file.FileSystems.getDefault().getPath(\"/etc/passwd\")); 0;",
+ "java.nio.channels.SocketChannel.open(); 0;",
+ "new java.net.ServerSocket().bind(null); 0;",
+ "var thread = new java.lang.Thread(); thread.start(); 0;",
+ "java.lang.System.getProperty(\"foo.bar.baz\"); 0;",
- "java.lang.Class.forName(\"java.lang.System\"); 0;",
+ "java.lang.Runtime.getRuntime().exec(\"/tmp/foo\"); 0;",
+ "java.lang.Runtime.getRuntime().loadLibrary(\"foobar\"); 0;",
+ "java.lang.Runtime.getRuntime().loadLibrary(\"foobar\"); 0;",
+ // TODO these (ugly) calls are still possible - these can consume CPU (as one could do with an evil loop, too)
+// "java.lang.Runtime.getRuntime().traceMethodCalls(true); 0;",
+// "java.lang.Runtime.getRuntime().gc(); 0;",
+// "java.lang.Runtime.getRuntime(); 0;",
+ };
+
+ for (String script : javascript)
+ {
+ try
+ {
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE javascript\n" +
+ "AS '" + script + "';");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ Assert.fail("Javascript security check failed: " + script);
+ }
+ catch (FunctionExecutionException e)
+ {
+ assertAccessControlException(script, e);
+ }
+ }
++
++ String script = "java.lang.Class.forName(\"java.lang.System\"); 0;";
++ String fName = createFunction(KEYSPACE_PER_TEST, "double",
++ "CREATE OR REPLACE FUNCTION %s(val double) " +
++ "RETURNS NULL ON NULL INPUT " +
++ "RETURNS double " +
++ "LANGUAGE javascript\n" +
++ "AS '" + script + "';");
++ assertInvalidThrowMessage("Java reflection not supported when class filter is present",
++ FunctionExecutionException.class,
++ "SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ }
+
+ private static void assertAccessControlException(String script, FunctionExecutionException e)
+ {
+ for (Throwable t = e; t != null && t != t.getCause(); t = t.getCause())
+ if (t instanceof AccessControlException)
+ return;
+ Assert.fail("no AccessControlException for " + script + " (got " + e + ')');
+ }
+
+ @Test
+ public void testAmokUDF() throws Throwable
+ {
+ createTable("CREATE TABLE %s (key int primary key, dval double)");
+ execute("INSERT INTO %s (key, dval) VALUES (?, ?)", 1, 1d);
+
+ long udfWarnTimeout = DatabaseDescriptor.getUserDefinedFunctionWarnTimeout();
+ long udfFailTimeout = DatabaseDescriptor.getUserDefinedFunctionFailTimeout();
+ int maxTries = 5;
+ for (int i = 1; i <= maxTries; i++)
+ {
+ try
+ {
+ // short timeout
+ DatabaseDescriptor.setUserDefinedFunctionWarnTimeout(10);
+ DatabaseDescriptor.setUserDefinedFunctionFailTimeout(250);
+ // don't kill the unit test... - default policy is "die"
+ DatabaseDescriptor.setUserFunctionTimeoutPolicy(Config.UserFunctionTimeoutPolicy.ignore);
+
+ ClientWarn.instance.captureWarnings();
+ String fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS 'long t=System.currentTimeMillis()+110; while (t>System.currentTimeMillis()) { }; return 0d;'");
+ execute("SELECT " + fName + "(dval) FROM %s WHERE key=1");
+ List<String> warnings = ClientWarn.instance.getWarnings();
+ Assert.assertNotNull(warnings);
+ Assert.assertFalse(warnings.isEmpty());
+ ClientWarn.instance.resetWarnings();
+
+ // Java UDF
+
+ fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVA\n" +
+ "AS 'long t=System.currentTimeMillis()+500; while (t>System.currentTimeMillis()) { }; return 0d;';");
+ assertInvalidMessage("ran longer than 250ms", "SELECT " + fName + "(dval) FROM %s WHERE key=1");
+
+ // Javascript UDF
+
+ fName = createFunction(KEYSPACE_PER_TEST, "double",
+ "CREATE OR REPLACE FUNCTION %s(val double) " +
+ "RETURNS NULL ON NULL INPUT " +
+ "RETURNS double " +
+ "LANGUAGE JAVASCRIPT\n" +
+ "AS 'var t=java.lang.System.currentTimeMillis()+500; while (t>java.lang.System.currentTimeMillis()) { }; 0;';");
+ assertInvalidMessage("ran longer than 250ms", "SELECT " + fName + "(dval) FROM %s WHERE key=1");
+
+ return;
+ }
+ catch (Error | RuntimeException e)
+ {
+ if (i == maxTries)
+ throw e;
+ }
+ finally
+ {
+ // reset to defaults
+ DatabaseDescriptor.setUserDefinedFunctionWarnTimeout(udfWarnTimeout);
+ DatabaseDescriptor.setUserDefinedFunctionFailTimeout(udfFailTimeout);
+ }
+ }
+ }
+
+}
[04/10] cassandra git commit: Remove support for non-JavaScript UDFs
Posted by sn...@apache.org.
Remove support for non-JavaScript UDFs
patch by Robert Stupp; reviewed by Tyler Hobbs for CASSANDRA-12883
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6f360b6d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6f360b6d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6f360b6d
Branch: refs/heads/trunk
Commit: 6f360b6d239a7b3d3b625e1bcefcaeb2f6b2be1d
Parents: 0cf0f67
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu Jan 5 22:18:37 2017 +0100
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu Jan 5 22:19:50 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
NEWS.txt | 16 ++++++++++++++++
.../cassandra/cql3/functions/ScriptBasedUDF.java | 5 +++++
3 files changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d31ffc8..b41313d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.2.9
+ * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
* Fix DynamicEndpointSnitch noop in multi-datacenter situations (CASSANDRA-13074)
* cqlsh copy-from: encode column names to avoid primary key parsing errors (CASSANDRA-12909)
* Temporarily fix bug that creates commit log when running offline tools (CASSANDRA-8616)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index 0a3ab36..37949a1 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -13,6 +13,19 @@ restore snapshots created with the previous major version using the
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
+2.2.9
+=====
+
+Deprecation
+-----------
+
+(See note about the new feature User-Defined-Functions in 2.2.0.)
+
+Since the security manager added in 3.0 only allows Java and JavaScript
+UDFs to be run, UDFs for other languages are deprecated and support for
+non-Java and non-JavaScript UDFs is deprecated in 2.2 and has been removed
+in version 3.0.11.
+
2.2.8
=====
@@ -230,6 +243,9 @@ New features
3.0. This will inherently be backwards-incompatible with any 2.2
UDF that perform insecure operations such as opening a socket or
writing to the filesystem.
+
+ Per the previous note about adding a security manager in 3.0, this security manager
+ means that non JavaScipt UDF's won't run, there for their use is deprecated.
************************************************************************
- Row-cache is now fully off-heap.
- jemalloc is now automatically preloaded and used on Linux and OS-X if
http://git-wip-us.apache.org/repos/asf/cassandra/blob/6f360b6d/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
index e55d450..2d46934 100644
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDF.java
@@ -73,6 +73,11 @@ public class ScriptBasedUDF extends UDFunction
{
super(name, argNames, argTypes, returnType, calledOnNullInput, language, body);
+ if (!"JavaScript".equalsIgnoreCase(language))
+ logger.warn("Support for UDFs using '" + language + "' has been deprecated and removed in 3.0. If '" +
+ language + "' actually is JavaScript, change the language used in CREATE/ALTER FUNCTION to " +
+ "'javascript'.");
+
Compilable scriptEngine = scriptEngines.get(language);
if (scriptEngine == null)
throw new InvalidRequestException(String.format("Invalid language '%s' for function '%s'", language, name));
[10/10] cassandra git commit: Merge branch 'cassandra-3.X' into trunk
Posted by sn...@apache.org.
Merge branch 'cassandra-3.X' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2cabf9c4
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2cabf9c4
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2cabf9c4
Branch: refs/heads/trunk
Commit: 2cabf9c49d0061a9325181a2b66711faecfec31d
Parents: d384e78 eca51a0
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu Jan 5 22:28:04 2017 +0100
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu Jan 5 22:28:04 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +
NEWS.txt | 3 ++
doc/cql3/CQL.textile | 2 +-
lib/jsr223/clojure/README.txt | 8 ---
lib/jsr223/groovy/README.txt | 35 -------------
lib/jsr223/jaskell/README.txt | 5 --
lib/jsr223/jruby/README.txt | 54 --------------------
lib/jsr223/jython/README.txt | 33 ------------
lib/jsr223/scala/README.txt | 37 --------------
.../cql3/functions/ScriptBasedUDFunction.java | 35 +++++--------
.../cql3/validation/entities/UFScriptTest.java | 3 +-
.../validation/entities/UFSecurityTest.java | 12 ++++-
12 files changed, 31 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2cabf9c4/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2cabf9c4/NEWS.txt
----------------------------------------------------------------------
[09/10] cassandra git commit: Merge branch 'cassandra-3.0' into
cassandra-3.X
Posted by sn...@apache.org.
Merge branch 'cassandra-3.0' into cassandra-3.X
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/eca51a03
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/eca51a03
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/eca51a03
Branch: refs/heads/trunk
Commit: eca51a0348141f45ea9bb31b76571ed81284c32c
Parents: 5f3b359 6e716c6
Author: Robert Stupp <sn...@snazy.de>
Authored: Thu Jan 5 22:26:48 2017 +0100
Committer: Robert Stupp <sn...@snazy.de>
Committed: Thu Jan 5 22:26:48 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 2 +
NEWS.txt | 3 ++
doc/cql3/CQL.textile | 2 +-
lib/jsr223/clojure/README.txt | 8 ---
lib/jsr223/groovy/README.txt | 35 -------------
lib/jsr223/jaskell/README.txt | 5 --
lib/jsr223/jruby/README.txt | 54 --------------------
lib/jsr223/jython/README.txt | 33 ------------
lib/jsr223/scala/README.txt | 37 --------------
.../cql3/functions/ScriptBasedUDFunction.java | 35 +++++--------
.../cql3/validation/entities/UFScriptTest.java | 3 +-
.../validation/entities/UFSecurityTest.java | 12 ++++-
12 files changed, 31 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eca51a03/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index cacb7cd,77a310d..66c2ee6
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,136 -1,5 +1,138 @@@
-3.0.11
- * Fixed flacky SSTableRewriterTest: check file counts before calling validateCFS (CASSANDRA-12348)
+3.12
+ * Use new token allocation for non bootstrap case as well (CASSANDRA-13080)
+ * Avoid byte-array copy when key cache is disabled (CASSANDRA-13084)
+ * More fixes to the TokenAllocator (CASSANDRA-12990)
+ * Require forceful decommission if number of nodes is less than replication factor (CASSANDRA-12510)
+ * Allow IN restrictions on column families with collections (CASSANDRA-12654)
+ * Move to FastThreadLocalThread and FastThreadLocal (CASSANDRA-13034)
+ * nodetool stopdaemon errors out (CASSANDRA-13030)
+ * Log message size in trace message in OutboundTcpConnection (CASSANDRA-13028)
+ * Add timeUnit Days for cassandra-stress (CASSANDRA-13029)
+ * Add mutation size and batch metrics (CASSANDRA-12649)
+ * Add method to get size of endpoints to TokenMetadata (CASSANDRA-12999)
+ * Fix primary index calculation for SASI (CASSANDRA-12910)
+ * Expose time spent waiting in thread pool queue (CASSANDRA-8398)
+ * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969)
+ * NoReplicationTokenAllocator should work with zero replication factor (CASSANDRA-12983)
+ * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946)
+ * Add support for arithmetic operators (CASSANDRA-11935)
+ * Tables in system_distributed should not use gcgs of 0 (CASSANDRA-12954)
++Merged from 3.0:
++ * Remove support for non-JavaScript UDFs (CASSANDRA-12883)
+
+
+3.10
+ * Fixed query monitoring for range queries (CASSANDRA-13050)
+ * Remove outboundBindAny configuration property (CASSANDRA-12673)
+ * Use correct bounds for all-data range when filtering (CASSANDRA-12666)
+ * Remove timing window in test case (CASSANDRA-12875)
+ * Resolve unit testing without JCE security libraries installed (CASSANDRA-12945)
+ * Fix inconsistencies in cassandra-stress load balancing policy (CASSANDRA-12919)
+ * Fix validation of non-frozen UDT cells (CASSANDRA-12916)
+ * Don't shut down socket input/output on StreamSession (CASSANDRA-12903)
+ * Fix Murmur3PartitionerTest (CASSANDRA-12858)
+ * Move cqlsh syntax rules into separate module and allow easier customization (CASSANDRA-12897)
+ * Fix CommitLogSegmentManagerTest (CASSANDRA-12283)
+ * Fix cassandra-stress truncate option (CASSANDRA-12695)
+ * Fix crossNode value when receiving messages (CASSANDRA-12791)
+ * Don't load MX4J beans twice (CASSANDRA-12869)
+ * Extend native protocol request flags, add versions to SUPPORTED, and introduce ProtocolVersion enum (CASSANDRA-12838)
+ * Set JOINING mode when running pre-join tasks (CASSANDRA-12836)
+ * remove net.mintern.primitive library due to license issue (CASSANDRA-12845)
+ * Properly format IPv6 addresses when logging JMX service URL (CASSANDRA-12454)
+ * Optimize the vnode allocation for single replica per DC (CASSANDRA-12777)
+ * Use non-token restrictions for bounds when token restrictions are overridden (CASSANDRA-12419)
+ * Fix CQLSH auto completion for PER PARTITION LIMIT (CASSANDRA-12803)
+ * Use different build directories for Eclipse and Ant (CASSANDRA-12466)
+ * Avoid potential AttributeError in cqlsh due to no table metadata (CASSANDRA-12815)
+ * Fix RandomReplicationAwareTokenAllocatorTest.testExistingCluster (CASSANDRA-12812)
+ * Upgrade commons-codec to 1.9 (CASSANDRA-12790)
+ * Make the fanout size for LeveledCompactionStrategy to be configurable (CASSANDRA-11550)
+ * Add duration data type (CASSANDRA-11873)
+ * Fix timeout in ReplicationAwareTokenAllocatorTest (CASSANDRA-12784)
+ * Improve sum aggregate functions (CASSANDRA-12417)
+ * Make cassandra.yaml docs for batch_size_*_threshold_in_kb reflect changes in CASSANDRA-10876 (CASSANDRA-12761)
+ * cqlsh fails to format collections when using aliases (CASSANDRA-11534)
+ * Check for hash conflicts in prepared statements (CASSANDRA-12733)
+ * Exit query parsing upon first error (CASSANDRA-12598)
+ * Fix cassandra-stress to use single seed in UUID generation (CASSANDRA-12729)
+ * CQLSSTableWriter does not allow Update statement (CASSANDRA-12450)
+ * Config class uses boxed types but DD exposes primitive types (CASSANDRA-12199)
+ * Add pre- and post-shutdown hooks to Storage Service (CASSANDRA-12461)
+ * Add hint delivery metrics (CASSANDRA-12693)
+ * Remove IndexInfo cache from FileIndexInfoRetriever (CASSANDRA-12731)
+ * ColumnIndex does not reuse buffer (CASSANDRA-12502)
+ * cdc column addition still breaks schema migration tasks (CASSANDRA-12697)
+ * Upgrade metrics-reporter dependencies (CASSANDRA-12089)
+ * Tune compaction thread count via nodetool (CASSANDRA-12248)
+ * Add +=/-= shortcut syntax for update queries (CASSANDRA-12232)
+ * Include repair session IDs in repair start message (CASSANDRA-12532)
+ * Add a blocking task to Index, run before joining the ring (CASSANDRA-12039)
+ * Fix NPE when using CQLSSTableWriter (CASSANDRA-12667)
+ * Support optional backpressure strategies at the coordinator (CASSANDRA-9318)
+ * Make randompartitioner work with new vnode allocation (CASSANDRA-12647)
+ * Fix cassandra-stress graphing (CASSANDRA-12237)
+ * Allow filtering on partition key columns for queries without secondary indexes (CASSANDRA-11031)
+ * Fix Cassandra Stress reporting thread model and precision (CASSANDRA-12585)
+ * Add JMH benchmarks.jar (CASSANDRA-12586)
+ * Cleanup uses of AlterTableStatementColumn (CASSANDRA-12567)
+ * Add keep-alive to streaming (CASSANDRA-11841)
+ * Tracing payload is passed through newSession(..) (CASSANDRA-11706)
+ * avoid deleting non existing sstable files and improve related log messages (CASSANDRA-12261)
+ * json/yaml output format for nodetool compactionhistory (CASSANDRA-12486)
+ * Retry all internode messages once after a connection is
+ closed and reopened (CASSANDRA-12192)
+ * Add support to rebuild from targeted replica (CASSANDRA-9875)
+ * Add sequence distribution type to cassandra stress (CASSANDRA-12490)
+ * "SELECT * FROM foo LIMIT ;" does not error out (CASSANDRA-12154)
+ * Define executeLocally() at the ReadQuery Level (CASSANDRA-12474)
+ * Extend read/write failure messages with a map of replica addresses
+ to error codes in the v5 native protocol (CASSANDRA-12311)
+ * Fix rebuild of SASI indexes with existing index files (CASSANDRA-12374)
+ * Let DatabaseDescriptor not implicitly startup services (CASSANDRA-9054, 12550)
+ * Fix clustering indexes in presence of static columns in SASI (CASSANDRA-12378)
+ * Fix queries on columns with reversed type on SASI indexes (CASSANDRA-12223)
+ * Added slow query log (CASSANDRA-12403)
+ * Count full coordinated request against timeout (CASSANDRA-12256)
+ * Allow TTL with null value on insert and update (CASSANDRA-12216)
+ * Make decommission operation resumable (CASSANDRA-12008)
+ * Add support to one-way targeted repair (CASSANDRA-9876)
+ * Remove clientutil jar (CASSANDRA-11635)
+ * Fix compaction throughput throttle (CASSANDRA-12366, CASSANDRA-12717)
+ * Delay releasing Memtable memory on flush until PostFlush has finished running (CASSANDRA-12358)
+ * Cassandra stress should dump all setting on startup (CASSANDRA-11914)
+ * Make it possible to compact a given token range (CASSANDRA-10643)
+ * Allow updating DynamicEndpointSnitch properties via JMX (CASSANDRA-12179)
+ * Collect metrics on queries by consistency level (CASSANDRA-7384)
+ * Add support for GROUP BY to SELECT statement (CASSANDRA-10707)
+ * Deprecate memtable_cleanup_threshold and update default for memtable_flush_writers (CASSANDRA-12228)
+ * Upgrade to OHC 0.4.4 (CASSANDRA-12133)
+ * Add version command to cassandra-stress (CASSANDRA-12258)
+ * Create compaction-stress tool (CASSANDRA-11844)
+ * Garbage-collecting compaction operation and schema option (CASSANDRA-7019)
+ * Add beta protocol flag for v5 native protocol (CASSANDRA-12142)
+ * Support filtering on non-PRIMARY KEY columns in the CREATE
+ MATERIALIZED VIEW statement's WHERE clause (CASSANDRA-10368)
+ * Unify STDOUT and SYSTEMLOG logback format (CASSANDRA-12004)
+ * COPY FROM should raise error for non-existing input files (CASSANDRA-12174)
+ * Faster write path (CASSANDRA-12269)
+ * Option to leave omitted columns in INSERT JSON unset (CASSANDRA-11424)
+ * Support json/yaml output in nodetool tpstats (CASSANDRA-12035)
+ * Expose metrics for successful/failed authentication attempts (CASSANDRA-10635)
+ * Prepend snapshot name with "truncated" or "dropped" when a snapshot
+ is taken before truncating or dropping a table (CASSANDRA-12178)
+ * Optimize RestrictionSet (CASSANDRA-12153)
+ * cqlsh does not automatically downgrade CQL version (CASSANDRA-12150)
+ * Omit (de)serialization of state variable in UDAs (CASSANDRA-9613)
+ * Create a system table to expose prepared statements (CASSANDRA-8831)
+ * Reuse DataOutputBuffer from ColumnIndex (CASSANDRA-11970)
+ * Remove DatabaseDescriptor dependency from SegmentedFile (CASSANDRA-11580)
+ * Add supplied username to authentication error messages (CASSANDRA-12076)
+ * Remove pre-startup check for open JMX port (CASSANDRA-12074)
+ * Remove compaction Severity from DynamicEndpointSnitch (CASSANDRA-11738)
+ * Restore resumable hints delivery (CASSANDRA-11960)
+ * Properly report LWT contention (CASSANDRA-12626)
+Merged from 3.0:
* Fix deserialization of 2.x DeletedCells (CASSANDRA-12620)
* Add parent repair session id to anticompaction log message (CASSANDRA-12186)
* Improve contention handling on failure to acquire MV lock for streaming and hints (CASSANDRA-12905)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eca51a03/NEWS.txt
----------------------------------------------------------------------
diff --cc NEWS.txt
index e6c8c10,b4e6551..9f376ff
--- a/NEWS.txt
+++ b/NEWS.txt
@@@ -21,125 -21,23 +21,128 @@@ Upgradin
- Specifying the default_time_to_live option when creating or altering a
materialized view was erroneously accepted (and ignored). It is now
properly rejected.
+ - Only Java and JavaScript are now supported UDF languages.
+ The sandbox in 3.0 already prevented the use of script languages except Java
+ and JavaScript.
-3.0.10
-=====
+3.10
+====
-Upgrading
----------
- - memtable_allocation_type: offheap_buffers is no longer allowed to be specified in the 3.0 series.
- This was an oversight that can cause segfaults. Offheap was re-introduced in 3.4 see CASSANDRA-11039
- and CASSANDRA-9472 for details.
+New features
+------------
+ - New `DurationType` (cql duration). See CASSANDRA-11873
+ - Runtime modification of concurrent_compactors is now available via nodetool
+ - Support for the assignment operators +=/-= has been added for update queries.
+ - An Index implementation may now provide a task which runs prior to joining
+ the ring. See CASSANDRA-12039
+ - Filtering on partition key columns is now also supported for queries without
+ secondary indexes.
+ - A slow query log has been added: slow queries will be logged at DEBUG level.
+ For more details refer to CASSANDRA-12403 and slow_query_log_timeout_in_ms
+ in cassandra.yaml.
+ - Support for GROUP BY queries has been added.
+ - A new compaction-stress tool has been added to test the throughput of compaction
+ for any cassandra-stress user schema. see compaction-stress help for how to use.
+ - Compaction can now take into account overlapping tables that don't take part
+ in the compaction to look for deleted or overwritten data in the compacted tables.
+ Then such data is found, it can be safely discarded, which in turn should enable
+ the removal of tombstones over that data.
+
+ The behavior can be engaged in two ways:
+ - as a "nodetool garbagecollect -g CELL/ROW" operation, which applies
+ single-table compaction on all sstables to discard deleted data in one step.
+ - as a "provide_overlapping_tombstones:CELL/ROW/NONE" compaction strategy flag,
+ which uses overlapping tables as a source of deletions/overwrites during all
+ compactions.
+ The argument specifies the granularity at which deleted data is to be found:
+ - If ROW is specified, only whole deleted rows (or sets of rows) will be
+ discarded.
+ - If CELL is specified, any columns whose value is overwritten or deleted
+ will also be discarded.
+ - NONE (default) specifies the old behavior, overlapping tables are not used to
+ decide when to discard data.
+ Which option to use depends on your workload, both ROW and CELL increase the
+ disk load on compaction (especially with the size-tiered compaction strategy),
+ with CELL being more resource-intensive. Both should lead to better read
+ performance if deleting rows (resp. overwriting or deleting cells) is common.
+ - Prepared statements are now persisted in the table prepared_statements in
+ the system keyspace. Upon startup, this table is used to preload all
+ previously prepared statements - i.e. in many cases clients do not need to
+ re-prepare statements against restarted nodes.
+ - cqlsh can now connect to older Cassandra versions by downgrading the native
+ protocol version. Please note that this is currently not part of our release
+ testing and, as a consequence, it is not guaranteed to work in all cases.
+ See CASSANDRA-12150 for more details.
+ - Snapshots that are automatically taken before a table is dropped or truncated
+ will have a "dropped" or "truncated" prefix on their snapshot tag name.
+ - Metrics are exposed for successful and failed authentication attempts.
+ These can be located using the object names org.apache.cassandra.metrics:type=Client,name=AuthSuccess
+ and org.apache.cassandra.metrics:type=Client,name=AuthFailure respectively.
+ - Add support to "unset" JSON fields in prepared statements by specifying DEFAULT UNSET.
+ See CASSANDRA-11424 for details
+ - Allow TTL with null value on insert and update. It will be treated as equivalent to inserting a 0.
+ - Removed outboundBindAny configuration property. See CASSANDRA-12673 for details.
+
+Upgrading
+---------
+ - Request timeouts in cassandra.yaml (read_request_timeout_in_ms, etc) now apply to the
+ "full" request time on the coordinator. Previously, they only covered the time from
+ when the coordinator sent a message to a replica until the time that the replica
+ responded. Additionally, the previous behavior was to reset the timeout when performing
+ a read repair, making a second read to fix a short read, and when subranges were read
+ as part of a range scan or secondary index query. In 3.10 and higher, the timeout
+ is no longer reset for these "subqueries". The entire request must complete within
+ the specified timeout. As a consequence, your timeouts may need to be adjusted
+ to account for this. See CASSANDRA-12256 for more details.
+ - Logs written to stdout are now consistent with logs written to files.
+ Time is now local (it was UTC on the console and local in files). Date, thread, file
+ and line info where added to stdout. (see CASSANDRA-12004)
+ - The 'clientutil' jar, which has been somewhat broken on the 3.x branch, is not longer provided.
+ The features provided by that jar are provided by any good java driver and we advise relying on drivers rather on
+ that jar, but if you need that jar for backward compatiblity until you do so, you should use the version provided
+ on previous Cassandra branch, like the 3.0 branch (by design, the functionality provided by that jar are stable
+ accross versions so using the 3.0 jar for a client connecting to 3.x should work without issues).
+ - (Tools development) DatabaseDescriptor no longer implicitly startups components/services like
+ commit log replay. This may break existing 3rd party tools and clients. In order to startup
+ a standalone tool or client application, use the DatabaseDescriptor.toolInitialization() or
+ DatabaseDescriptor.clientInitialization() methods. Tool initialization sets up partitioner,
+ snitch, encryption context. Client initialization just applies the configuration but does not
+ setup anything. Instead of using Config.setClientMode() or Config.isClientMode(), which are
+ deprecated now, use one of the appropiate new methods in DatabaseDescriptor.
+ - Application layer keep-alives were added to the streaming protocol to prevent idle incoming connections from
+ timing out and failing the stream session (CASSANDRA-11839). This effectively deprecates the streaming_socket_timeout_in_ms
+ property in favor of streaming_keep_alive_period_in_secs. See cassandra.yaml for more details about this property.
+ - Duration litterals support the ISO 8601 format. By consequence, identifiers matching that format
+ (e.g P2Y or P1MT6H) will not be supported anymore (CASSANDRA-11873).
+
+3.8
+===
-3.0.9
-=====
+New features
+------------
+ - Shared pool threads are now named according to the stage they are executing
+ tasks for. Thread names mentioned in traced queries change accordingly.
+ - A new option has been added to cassandra-stress "-rate fixed={number}/s"
+ that forces a scheduled rate of operations/sec over time. Using this, stress can
+ accurately account for coordinated ommission from the stress process.
+ - The cassandra-stress "-rate limit=" option has been renamed to "-rate throttle="
+ - hdr histograms have been added to stress runs, it's output can be saved to disk using:
+ "-log hdrfile=" option. This histogram includes response/service/wait times when used with the
+ fixed or throttle rate options. The histogram file can be plotted on
+ http://hdrhistogram.github.io/HdrHistogram/plotFiles.html
+ - TimeWindowCompactionStrategy has been added. This has proven to be a better approach
+ to time series compaction and new tables should use this instead of DTCS. See
+ CASSANDRA-9666 for details.
+ - Change-Data-Capture is now available. See cassandra.yaml and for cdc-specific flags and
+ a brief explanation of on-disk locations for archived data in CommitLog form. This can
+ be enabled via ALTER TABLE ... WITH cdc=true.
+ Upon flush, CommitLogSegments containing data for CDC-enabled tables are moved to
+ the data/cdc_raw directory until removed by the user and writes to CDC-enabled tables
+ will be rejected with a WriteTimeoutException once cdc_total_space_in_mb is reached
+ between unflushed CommitLogSegments and cdc_raw.
+ NOTE: CDC is disabled by default in the .yaml file. Do not enable CDC on a mixed-version
+ cluster as it will lead to exceptions which can interrupt traffic. Once all nodes
+ have been upgraded to 3.8 it is safe to enable this feature and restart the cluster.
Upgrading
---------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eca51a03/doc/cql3/CQL.textile
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eca51a03/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
index 3ad60d0,47deafa..c568972
--- a/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
+++ b/src/java/org/apache/cassandra/cql3/functions/ScriptBasedUDFunction.java
@@@ -27,7 -27,9 +27,10 @@@ import java.util.*
import java.util.concurrent.ExecutorService;
import javax.script.*;
+import jdk.nashorn.api.scripting.AbstractJSObject;
+ import jdk.nashorn.api.scripting.ClassFilter;
+ import jdk.nashorn.api.scripting.NashornScriptEngine;
+ import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
import org.apache.cassandra.concurrent.NamedThreadFactory;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.db.marshal.AbstractType;
@@@ -158,13 -149,6 +152,10 @@@ final class ScriptBasedUDFunction exten
throw new InvalidRequestException(
String.format("Failed to compile function '%s' for language %s: %s", name, language, e));
}
+
+ // It's not always possible to simply pass a plain Java object as a binding to Nashorn and
+ // let the script execute methods on it.
- udfContextBinding =
- ("Oracle Nashorn".equals(((ScriptEngine) scriptEngine).getFactory().getEngineName()))
- ? new UDFContextWrapper()
- : udfContext;
++ udfContextBinding = new UDFContextWrapper();
}
protected ExecutorService executor()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eca51a03/test/unit/org/apache/cassandra/cql3/validation/entities/UFScriptTest.java
----------------------------------------------------------------------