You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by dj...@apache.org on 2020/02/14 18:36:44 UTC
[cassandra-dtest] branch master updated: Make cqlsh and cqlshlib
Python 2 & 3 compatible
This is an automated email from the ASF dual-hosted git repository.
djoshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/cassandra-dtest.git
The following commit(s) were added to refs/heads/master by this push:
new 175a083 Make cqlsh and cqlshlib Python 2 & 3 compatible
175a083 is described below
commit 175a083a6f3b4d5d58f3702d31ce6920af519669
Author: Patrick Bannister <pt...@gmail.com>
AuthorDate: Fri Apr 13 22:18:57 2018 -0400
Make cqlsh and cqlshlib Python 2 & 3 compatible
Patch by Patrick Bannister; reviewed by Dinesh Joshi, Andy Tolbert and David Capwell for CASSANDRA-10190
---
conftest.py | 11 +-
cql_test.py | 2 +-
cqlsh_tests/cqlsh_test_types.py | 136 ++++++++++++++++
cqlsh_tests/cqlsh_tools.py | 4 +-
cqlsh_tests/test_cqlsh.py | 335 +++++++++++++++++++++++++++++++++-------
cqlsh_tests/test_cqlsh_copy.py | 211 ++++++++++++-------------
dtest.py | 2 +-
dtest_setup.py | 2 +-
requirements.txt | 1 -
run_dtests.py | 6 +-
tools/metadata_wrapper.py | 4 +-
11 files changed, 530 insertions(+), 184 deletions(-)
diff --git a/conftest.py b/conftest.py
index e680ca9..b46f8b5 100644
--- a/conftest.py
+++ b/conftest.py
@@ -7,8 +7,7 @@ import re
import platform
import copy
import inspect
-
-from itertools import zip_longest
+import subprocess
from dtest import running_in_docker, cleanup_docker_environment_before_test_execution
@@ -25,6 +24,9 @@ from dtest_config import DTestConfig
from dtest_setup import DTestSetup
from dtest_setup_overrides import DTestSetupOverrides
+# Python 3 imports
+from itertools import zip_longest
+
logger = logging.getLogger(__name__)
def check_required_loopback_interfaces_available():
@@ -497,9 +499,8 @@ def pytest_collection_modifyitems(items, config):
if config.getoption("use_off_heap_memtables"):
deselect_test = True
- # temporarily deselect tests in cqlsh_copy_tests that depend on cqlshlib,
- # until cqlshlib is Python 3 compatibile
- if item.get_marker("depends_cqlshlib"):
+ # deselect cqlsh tests that depend on fixing a driver behavior
+ if item.get_closest_marker("depends_driver"):
deselect_test = True
if deselect_test:
diff --git a/cql_test.py b/cql_test.py
index 84ded2b..659cbae 100644
--- a/cql_test.py
+++ b/cql_test.py
@@ -1105,7 +1105,7 @@ class TestCQLSlowQuery(CQLTester):
@jira_ticket CASSANDRA-12403
"""
cluster = self.cluster
- cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 10,
+ cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 1,
'request_timeout_in_ms': 120000,
'read_request_timeout_in_ms': 120000,
'range_request_timeout_in_ms': 120000})
diff --git a/cqlsh_tests/cqlsh_test_types.py b/cqlsh_tests/cqlsh_test_types.py
new file mode 100644
index 0000000..d78864e
--- /dev/null
+++ b/cqlsh_tests/cqlsh_test_types.py
@@ -0,0 +1,136 @@
+import datetime
+import sys
+
+from collections import namedtuple
+from contextlib import contextmanager
+
+from cassandra.util import SortedSet
+
+
+@contextmanager
+def _cqlshlib(cqlshlib_path):
+ """
+ Returns the cqlshlib module found at the specified path.
+ """
+ # This method accomplishes its goal by manually adding the library to
+ # sys.path, returning the module, then restoring the old path once the
+ # context manager exits. This isn't great for maintainability and should
+ # be replaced if cqlshlib is made easier to interact with.
+ saved_path = list(sys.path)
+
+ try:
+ sys.path = sys.path + [cqlshlib_path]
+ import cqlshlib
+ yield cqlshlib
+ finally:
+ sys.path = saved_path
+
+
+def maybe_quote(s):
+ """
+ Return a quoted string representation for strings, unicode and date time parameters,
+ otherwise return a string representation of the parameter.
+ """
+ return "'{}'".format(s) if isinstance(s, (str, Datetime)) else str(s)
+
+
+class Address(namedtuple('Address', ('name', 'number', 'street', 'phones'))):
+ __slots__ = ()
+
+ def __repr__(self):
+ phones_str = "{{{}}}".format(', '.join(maybe_quote(p) for p in sorted(self.phones)))
+ return "{{name: {}, number: {}, street: '{}', phones: {}}}".format(self.name,
+ self.number,
+ self.street,
+ phones_str)
+
+ def __str__(self):
+ phones_str = "{{{}}}".format(', '.join(maybe_quote(p) for p in sorted(self.phones)))
+ return "{{name: {}, number: {}, street: '{}', phones: {}}}".format(self.name,
+ self.number,
+ self.street,
+ phones_str)
+
+
+class Datetime(datetime.datetime):
+ """
+ Extend standard datetime.datetime class with cql formatting.
+ This could be cleaner if this class was declared inside TestCqlshCopy, but then pickle
+ wouldn't have access to the class.
+ """
+ def __new__(cls, year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, cqlshlib_path=None):
+ self = datetime.datetime.__new__(cls, year, month, day, hour, minute, second, microsecond, tzinfo)
+ if (cqlshlib_path is not None):
+ with _cqlshlib(cqlshlib_path) as cqlshlib:
+ from cqlshlib.formatting import DEFAULT_TIMESTAMP_FORMAT, round_microseconds
+ self.default_time_format = DEFAULT_TIMESTAMP_FORMAT
+ self.round_microseconds = round_microseconds
+ else:
+ self.default_time_format = '%Y-%m-%d %H:%M:%S%z'
+ return self
+
+ def __repr__(self):
+ return self._format_for_csv()
+
+ def __str__(self):
+ return self._format_for_csv()
+
+ def _format_for_csv(self):
+ ret = self.strftime(self.default_time_format)
+ return self.round_microseconds(ret) if self.round_microseconds else ret
+
+
+class ImmutableDict(frozenset):
+ """
+ Immutable dictionary implementation to represent map types.
+ We need to pass BoundStatement.bind() a dict() because it calls iteritems(),
+ except we can't create a dict with another dict as the key, hence we use a class
+ that adds iteritems to a frozen set of tuples (which is how dict are normally made
+ immutable in python).
+ Must be declared in the top level of the module to be available for pickling.
+ """
+ iteritems = frozenset.__iter__
+
+ def items(self):
+ for k, v in self.iteritems():
+ yield k, v
+
+ def __repr__(self):
+ return '{{{}}}'.format(', '.join(['{0}: {1}'.format(maybe_quote(k), maybe_quote(v)) for k, v in sorted(self.items())]))
+
+
+class ImmutableSet(SortedSet):
+
+ def __repr__(self):
+ return '{{{}}}'.format(', '.join([maybe_quote(t) for t in sorted(self._items)]))
+
+ def __str__(self):
+ return '{{{}}}'.format(', '.join([maybe_quote(t) for t in sorted(self._items)]))
+
+ def __hash__(self):
+ return hash(tuple([e for e in self]))
+
+
+class Name(namedtuple('Name', ('firstname', 'lastname'))):
+ __slots__ = ()
+
+ def __repr__(self):
+ return "{{firstname: '{}', lastname: '{}'}}".format(self.firstname, self.lastname)
+
+ def __str__(self):
+ return "{{firstname: '{}', lastname: '{}'}}".format(self.firstname, self.lastname)
+
+
+class UTC(datetime.tzinfo):
+ """
+ A utility class to specify a UTC timezone.
+ """
+
+ def utcoffset(self, dt):
+ return datetime.timedelta(0)
+
+ def tzname(self, dt):
+ return "UTC"
+
+ def dst(self, dt):
+ return datetime.timedelta(0)
diff --git a/cqlsh_tests/cqlsh_tools.py b/cqlsh_tests/cqlsh_tools.py
index 9544d3e..0b49232 100644
--- a/cqlsh_tests/cqlsh_tools.py
+++ b/cqlsh_tests/cqlsh_tools.py
@@ -1,10 +1,12 @@
+from __future__ import unicode_literals
+
import csv
import random
+from typing import List
import cassandra
from cassandra.cluster import ResultSet
-from typing import List
class DummyColorMap(object):
diff --git a/cqlsh_tests/test_cqlsh.py b/cqlsh_tests/test_cqlsh.py
index ab905c5..c9a11e2 100644
--- a/cqlsh_tests/test_cqlsh.py
+++ b/cqlsh_tests/test_cqlsh.py
@@ -1,11 +1,14 @@
# coding=utf-8
+from __future__ import unicode_literals
+
import binascii
import csv
import datetime
import locale
import os
import re
+import six
import subprocess
import sys
import logging
@@ -23,8 +26,10 @@ from ccmlib import common
from .cqlsh_tools import monkeypatch_driver, unmonkeypatch_driver
from dtest import Tester, create_ks, create_cf
+from dtest_setup_overrides import DTestSetupOverrides
from tools.assertions import assert_all, assert_none
from tools.data import create_c1c2_table, insert_c1c2, rows_to_list
+from tools.misc import ImmutableMapping
since = pytest.mark.since
logger = logging.getLogger(__name__)
@@ -32,6 +37,15 @@ logger = logging.getLogger(__name__)
class TestCqlsh(Tester):
+ # override cluster options to enable user defined functions
+ # currently only needed for test_describe
+ @pytest.fixture
+ def fixture_dtest_setup_overrides(self):
+ dtest_setup_overrides = DTestSetupOverrides()
+ dtest_setup_overrides.cluster_options = ImmutableMapping({'enable_user_defined_functions': 'true',
+ 'enable_scripted_user_defined_functions': 'true'})
+ return dtest_setup_overrides
+
@classmethod
def setUpClass(cls):
cls._cached_driver_methods = monkeypatch_driver()
@@ -53,11 +67,11 @@ class TestCqlsh(Tester):
"""
@jira_ticket CASSANDRA-10066
Checks that cqlsh is compliant with pycodestyle (formally known as pep8) with the following command:
- pycodestyle --ignore E501,E402,E731 pylib/cqlshlib/*.py bin/cqlsh.py
+ pycodestyle --ignore E501,E402,E731,W503 pylib/cqlshlib/*.py bin/cqlsh.py
"""
cluster = self.cluster
- if cluster.version() < '2.2':
+ if cluster.version() < LooseVersion('2.2'):
cqlsh_path = os.path.join(cluster.get_install_dir(), 'bin', 'cqlsh')
else:
cqlsh_path = os.path.join(cluster.get_install_dir(), 'bin', 'cqlsh.py')
@@ -66,7 +80,7 @@ class TestCqlsh(Tester):
cqlshlib_paths = os.listdir(cqlshlib_path)
cqlshlib_paths = [os.path.join(cqlshlib_path, x) for x in cqlshlib_paths if '.py' in x and '.pyc' not in x]
- cmds = ['pycodestyle', '--ignore', 'E501,E402,E731', cqlsh_path] + cqlshlib_paths
+ cmds = ['pycodestyle', '--ignore', 'E501,E402,E731,W503', cqlsh_path] + cqlshlib_paths
logger.debug(cmds)
@@ -303,7 +317,7 @@ class TestCqlsh(Tester):
'I can eat glass and it does not hurt me': 1400
})
- output, err = self.run_cqlsh(node, 'use testks; SELECT * FROM varcharmaptable', ['--encoding=utf-8'])
+ output, _ = self.run_cqlsh(node, 'use testks; SELECT * FROM varcharmaptable', ['--encoding=utf-8'])
assert output.count('Можам да јадам стакло, а не ме штета.') == 16
assert output.count(' ⠊⠀⠉⠁⠝⠀⠑⠁⠞⠀⠛⠇⠁⠎⠎⠀⠁⠝⠙⠀⠊⠞⠀⠙⠕⠑⠎⠝⠞⠀⠓⠥⠗⠞⠀⠍⠑') == 16
@@ -316,7 +330,7 @@ class TestCqlsh(Tester):
node1, = self.cluster.nodelist()
- node1.run_cqlsh(cmds="""create KEYSPACE testks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
+ cmds = """create KEYSPACE testks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
use testks;
CREATE TABLE varcharmaptable (
@@ -427,7 +441,8 @@ INSERT INTO varcharmaptable (varcharkey, varcharvarintmap ) VALUES ('᚛᚛
UPDATE varcharmaptable SET varcharvarintmap = varcharvarintmap + {'Vitrum edere possum, mihi non nocet.':20000} WHERE varcharkey= '᚛᚛ᚉᚑᚅᚔᚉᚉᚔᚋ ᚔᚈᚔ ᚍᚂᚐᚅᚑ ᚅᚔᚋᚌᚓᚅᚐ᚜';
UPDATE varcharmaptable SET varcharvarintmap['Vitrum edere possum, mihi non nocet.'] = 1010010101020400204143243 WHERE varcharkey= '᚛᚛ᚉᚑᚅᚔᚉᚉᚔᚋ ᚔᚈᚔ ᚍᚂᚐᚅᚑ ᚅᚔᚋᚌᚓᚅᚐ᚜'
- """)
+ """
+ node1.run_cqlsh(cmds=cmds)
self.verify_glass(node1)
@@ -452,7 +467,9 @@ UPDATE varcharmaptable SET varcharvarintmap['Vitrum edere possum, mihi non nocet
node1, = self.cluster.nodelist()
- output, err, _ = node1.run_cqlsh(cmds="ä;")
+ cmds = "ä;"
+ _, err, _ = node1.run_cqlsh(cmds=cmds)
+
assert 'Invalid syntax' in err
assert 'ä' in err
@@ -468,7 +485,7 @@ UPDATE varcharmaptable SET varcharvarintmap['Vitrum edere possum, mihi non nocet
node1, = self.cluster.nodelist()
cmd = '''create keyspace "ä" WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'};'''
- output, err, _ = node1.run_cqlsh(cmds=cmd, cqlsh_options=["--debug"])
+ _, err, _ = node1.run_cqlsh(cmds=cmd, cqlsh_options=["--debug"])
if self.cluster.version() >= LooseVersion('4.0'):
assert "Keyspace name must not be empty, more than 48 characters long, or contain non-alphanumeric-underscore characters (got 'ä')" in err
@@ -484,7 +501,7 @@ UPDATE varcharmaptable SET varcharvarintmap['Vitrum edere possum, mihi non nocet
node1, = self.cluster.nodelist()
- node1.run_cqlsh(cmds="""create keyspace CASSANDRA_7196 WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1} ;
+ cmds = """create keyspace CASSANDRA_7196 WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1} ;
use CASSANDRA_7196;
@@ -539,9 +556,13 @@ INSERT INTO has_all_types (num, intcol, asciicol, bigintcol, blobcol, booleancol
timestampcol, uuidcol, varcharcol, varintcol)
VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDecimal(0x),
blobAsDouble(0x), blobAsFloat(0x), '', blobAsTimestamp(0x), blobAsUuid(0x), '',
- blobAsVarint(0x))""")
+ blobAsVarint(0x))"""
+
+ node1.run_cqlsh(cmds=cmds)
+
+ select_cmd = "select intcol, bigintcol, varintcol from CASSANDRA_7196.has_all_types where num in (0, 1, 2, 3, 4)"
+ output, err = self.run_cqlsh(node1, cmds=select_cmd)
- output, err = self.run_cqlsh(node1, "select intcol, bigintcol, varintcol from CASSANDRA_7196.has_all_types where num in (0, 1, 2, 3, 4)")
if common.is_win():
output = output.replace('\r', '')
@@ -637,7 +658,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
conn.execute("CREATE USER user1 WITH PASSWORD 'user1'")
conn.execute("GRANT ALL ON ks.t1 TO user1")
- if self.cluster.version() >= '4.0':
+ if self.cluster.version() >= LooseVersion('4.0'):
self.verify_output("LIST USERS", node1, """
name | super | datacenters
-----------+-------+-------------
@@ -646,7 +667,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
(2 rows)
""")
- elif self.cluster.version() >= '2.2':
+ elif self.cluster.version() >= LooseVersion('2.2'):
self.verify_output("LIST USERS", node1, """
name | super
-----------+-------
@@ -665,7 +686,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
(2 rows)
""")
- if self.cluster.version() >= '2.2':
+ if self.cluster.version() >= LooseVersion('2.2'):
self.verify_output("LIST ALL PERMISSIONS OF user1", node1, """
role | username | resource | permission
-------+----------+---------------+------------
@@ -698,6 +719,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
self.cluster.populate(1)
self.cluster.start(wait_for_binary_proto=True)
node1, = self.cluster.nodelist()
+
self.execute(
cql="""
CREATE KEYSPACE test WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1};
@@ -715,21 +737,21 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
assert "system" in output
# Describe keyspace
- self.execute(cql="DESCRIBE KEYSPACE test", expected_output=self.get_keyspace_output())
- self.execute(cql="DESCRIBE test", expected_output=self.get_keyspace_output())
+ self.execute(cql="DESCRIBE KEYSPACE test", expected_output=self.get_keyspace_output(), output_is_ordered=False)
+ self.execute(cql="DESCRIBE test", expected_output=self.get_keyspace_output(), output_is_ordered=False)
self.execute(cql="DESCRIBE test2", expected_err="'test2' not found in keyspaces")
- self.execute(cql="USE test; DESCRIBE KEYSPACE", expected_output=self.get_keyspace_output())
+ self.execute(cql="USE test; DESCRIBE KEYSPACE", expected_output=self.get_keyspace_output(), output_is_ordered=False)
# Describe table
- self.execute(cql="DESCRIBE TABLE test.test", expected_output=self.get_test_table_output())
- self.execute(cql="DESCRIBE TABLE test.users", expected_output=self.get_users_table_output())
- self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output())
- self.execute(cql="DESCRIBE test.users", expected_output=self.get_users_table_output())
+ self.execute(cql="DESCRIBE TABLE test.test", expected_output=self.get_test_table_output(), output_is_ordered=False)
+ self.execute(cql="DESCRIBE TABLE test.users", expected_output=self.get_users_table_output(), output_is_ordered=False)
+ self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output(), output_is_ordered=False)
+ self.execute(cql="DESCRIBE test.users", expected_output=self.get_users_table_output(), output_is_ordered=False)
self.execute(cql="DESCRIBE test.users2", expected_err="'users2' not found in keyspace 'test'")
- self.execute(cql="USE test; DESCRIBE TABLE test", expected_output=self.get_test_table_output())
- self.execute(cql="USE test; DESCRIBE TABLE users", expected_output=self.get_users_table_output())
- self.execute(cql="USE test; DESCRIBE test", expected_output=self.get_keyspace_output())
- self.execute(cql="USE test; DESCRIBE users", expected_output=self.get_users_table_output())
+ self.execute(cql="USE test; DESCRIBE TABLE test", expected_output=self.get_test_table_output(), output_is_ordered=False)
+ self.execute(cql="USE test; DESCRIBE TABLE users", expected_output=self.get_users_table_output(), output_is_ordered=False)
+ self.execute(cql="USE test; DESCRIBE test", expected_output=self.get_keyspace_output(), output_is_ordered=False)
+ self.execute(cql="USE test; DESCRIBE users", expected_output=self.get_users_table_output(), output_is_ordered=False)
self.execute(cql="USE test; DESCRIBE users2", expected_err="'users2' not found in keyspace 'test'")
# Describe index
@@ -757,7 +779,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
CREATE INDEX myindex ON test.users (age);
CREATE INDEX "QuotedNameIndex" on test.users (firstname)
""")
- self.execute(cql="DESCRIBE test.users", expected_output=self.get_users_table_output())
+ self.execute(cql="DESCRIBE test.users", expected_output=self.get_users_table_output(), output_is_ordered=False)
self.execute(cql='DESCRIBE test.myindex', expected_output=self.get_index_output('myindex', 'test', 'users', 'age'))
# Drop index and recreate
@@ -774,10 +796,10 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
# Prior to 3.0 the index would have been automatically dropped, but now we need to explicitly do that.
self.execute(cql='DROP INDEX test.test_val_idx')
self.execute(cql='ALTER TABLE test.test DROP val')
- self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output(has_val=False, has_val_idx=False))
+ self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output(has_val=False, has_val_idx=False), output_is_ordered=False)
self.execute(cql='DESCRIBE test.test_val_idx', expected_err="'test_val_idx' not found in keyspace 'test'")
self.execute(cql='ALTER TABLE test.test ADD val text')
- self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output(has_val=True, has_val_idx=False))
+ self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output(has_val=True, has_val_idx=False), output_is_ordered=False)
self.execute(cql='DESCRIBE test.test_val_idx', expected_err="'test_val_idx' not found in keyspace 'test'")
def test_describe_describes_non_default_compaction_parameters(self):
@@ -794,6 +816,97 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
assert "'min_threshold': '10'" in stdout
assert "'max_threshold': '100'" in stdout
+ def test_describe_functions(self, fixture_dtest_setup_overrides):
+ """Test DESCRIBE statements for functions and aggregate functions"""
+ self.cluster.populate(1)
+ self.cluster.start(wait_for_binary_proto=True)
+
+ create_ks_statement = "CREATE KEYSPACE test WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}"
+ create_function_statement = """
+CREATE FUNCTION test.some_function(arg int)
+ RETURNS NULL ON NULL INPUT
+ RETURNS int
+ LANGUAGE java
+ AS $$ return arg;
+ $$"""
+ create_aggregate_dependencies_statement = """
+CREATE OR REPLACE FUNCTION test.average_state(state tuple<int,bigint>, val int)
+ CALLED ON NULL INPUT
+ RETURNS tuple<int,bigint>
+ LANGUAGE java
+ AS $$ if (val != null) {
+ state.setInt(0, state.getInt(0)+1);
+ state.setLong(1, state.getLong(1)+val.intValue());
+ }
+ return state;
+ $$;
+CREATE OR REPLACE FUNCTION test.average_final (state tuple<int,bigint>)
+ CALLED ON NULL INPUT
+ RETURNS double
+ LANGUAGE java
+ AS $$
+ double r = 0;
+ if (state.getInt(0) == 0) return null;
+ r = state.getLong(1);
+ r /= state.getInt(0);
+ return Double.valueOf(r);
+ $$
+"""
+ create_aggregate_statement = """
+CREATE OR REPLACE AGGREGATE test.average(int)
+ SFUNC average_state
+ STYPE tuple<int,bigint>
+ FINALFUNC average_final
+ INITCOND (0, 0)
+"""
+ # the expected output of a DESCRIBE AGGREGATE statement
+ # does not look like a valid CREATE AGGREGATE statement
+ describe_aggregate_expected = """
+CREATE AGGREGATE test.average(int)
+ SFUNC average_state
+ STYPE frozen<tuple<int, bigint>>
+ FINALFUNC average_final
+ INITCOND (0, 0);
+"""
+
+ # create keyspace, scalar function, and aggregate function
+ self.execute(cql=create_ks_statement)
+ self.execute(cql=create_function_statement)
+ self.execute(cql=create_aggregate_dependencies_statement)
+ self.execute(cql=create_aggregate_statement)
+ # describe scalar functions
+ self.execute(cql='DESCRIBE FUNCTION test.some_function', expected_output='{};'.format(create_function_statement))
+ # describe aggregate functions
+ self.execute(cql='DESCRIBE AGGREGATE test.average', expected_output=describe_aggregate_expected)
+
+ def test_describe_types(self):
+ """Test DESCRIBE statements for user defined datatypes"""
+ self.cluster.populate(1)
+ self.cluster.start(wait_for_binary_proto=True)
+
+ create_ks_statement = "CREATE KEYSPACE test WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}"
+ create_name_type_statement = """
+CREATE TYPE test.name_type (
+ firstname text,
+ lastname text
+)"""
+ create_address_type_statement = """
+CREATE TYPE test.address_type (
+ name frozen<name_type>,
+ number int,
+ street text,
+ phones set<text>
+)"""
+
+ # create test keyspace and some user defined types
+ self.execute(cql=create_ks_statement)
+ self.execute(create_name_type_statement)
+ self.execute(create_address_type_statement)
+
+ # DESCRIBE user defined types
+ self.execute(cql='DESCRIBE TYPE test.name_type', expected_output='{};'.format(create_name_type_statement))
+ self.execute(cql='DESCRIBE TYPE test.address_type', expected_output='{};'.format(create_address_type_statement))
+
def test_describe_on_non_reserved_keywords(self):
"""
@jira_ticket CASSANDRA-9232
@@ -842,13 +955,14 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
self.execute(cql='USE test; DESCRIBE "users_by_state"', expected_output=self.get_users_by_state_mv_output())
def get_keyspace_output(self):
- return ("CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;" +
- self.get_test_table_output() +
- self.get_users_table_output())
+ return ["CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;",
+ self.get_test_table_output(),
+ self.get_users_table_output()]
def get_test_table_output(self, has_val=True, has_val_idx=True):
+ create_table = None
if has_val:
- ret = """
+ create_table = """
CREATE TABLE test.test (
id int,
col int,
@@ -856,7 +970,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
PRIMARY KEY (id, col)
"""
else:
- ret = """
+ create_table = """
CREATE TABLE test.test (
id int,
col int,
@@ -864,7 +978,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
"""
if self.cluster.version() >= LooseVersion('4.0'):
- ret += """
+ create_table += """
) WITH CLUSTERING ORDER BY (col ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
@@ -882,7 +996,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
AND speculative_retry = '99p';
"""
elif self.cluster.version() >= LooseVersion('3.9'):
- ret += """
+ create_table += """
) WITH CLUSTERING ORDER BY (col ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
@@ -900,7 +1014,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
AND speculative_retry = '99PERCENTILE';
"""
elif self.cluster.version() >= LooseVersion('3.0'):
- ret += """
+ create_table += """
) WITH CLUSTERING ORDER BY (col ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
@@ -918,7 +1032,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
AND speculative_retry = '99PERCENTILE';
"""
else:
- ret += """
+ create_table += """
) WITH CLUSTERING ORDER BY (col ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}'
@@ -936,22 +1050,18 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
"""
col_idx_def = self.get_index_output('test_col_idx', 'test', 'test', 'col')
-
+ expected_output = [create_table, col_idx_def]
if has_val_idx:
- val_idx_def = self.get_index_output('test_val_idx', 'test', 'test', 'val')
- if self.cluster.version() >= LooseVersion('2.2'):
- return ret + "\n" + val_idx_def + "\n" + col_idx_def
- else:
- return ret + "\n" + col_idx_def + "\n" + val_idx_def
- else:
- return ret + "\n" + col_idx_def
+ expected_output.append(self.get_index_output('test_val_idx', 'test', 'test', 'val'))
+ return expected_output
def get_users_table_output(self):
quoted_index_output = self.get_index_output('"QuotedNameIndex"', 'test', 'users', 'firstname')
myindex_output = self.get_index_output('myindex', 'test', 'users', 'age')
+ create_table = None
if self.cluster.version() >= LooseVersion('4.0'):
- return """
+ create_table = """
CREATE TABLE test.users (
userid text PRIMARY KEY,
age int,
@@ -971,9 +1081,9 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99p';
- """ + quoted_index_output + "\n" + myindex_output
+ """
elif self.cluster.version() >= LooseVersion('3.9'):
- return """
+ create_table = """
CREATE TABLE test.users (
userid text PRIMARY KEY,
age int,
@@ -993,9 +1103,9 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99PERCENTILE';
- """ + quoted_index_output + "\n" + myindex_output
+ """
elif self.cluster.version() >= LooseVersion('3.0'):
- return """
+ create_table = """
CREATE TABLE test.users (
userid text PRIMARY KEY,
age int,
@@ -1015,9 +1125,9 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99PERCENTILE';
- """ + quoted_index_output + "\n" + myindex_output
+ """
else:
- return """
+ create_table = """
CREATE TABLE test.users (
userid text PRIMARY KEY,
age int,
@@ -1036,8 +1146,8 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99.0PERCENTILE';
- """ + (quoted_index_output + "\n" + myindex_output if self.cluster.version() >= LooseVersion('2.2') else
- myindex_output + "\n" + quoted_index_output)
+ """
+ return [create_table, quoted_index_output, myindex_output]
def get_index_output(self, index, ks, table, col):
# a quoted index name (e.g. "FooIndex") is only correctly echoed by DESCRIBE
@@ -1123,7 +1233,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
AND speculative_retry = '99PERCENTILE';
"""
- def execute(self, cql, expected_output=None, expected_err=None, env_vars=None):
+ def execute(self, cql, expected_output=None, expected_err=None, env_vars=None, output_is_ordered=True, err_is_ordered=True):
logger.debug(cql)
node1, = self.cluster.nodelist()
@@ -1132,13 +1242,19 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
if err:
if expected_err:
err = err[10:] # strip <stdin>:2:
- self.check_response(err, expected_err)
+ if err_is_ordered:
+ self.check_response(err, expected_err)
+ else:
+ self.check_response_unordered(err, expected_err)
return
else:
assert False, err
if expected_output:
- self.check_response(output, expected_output)
+ if output_is_ordered:
+ self.check_response(output, expected_output)
+ else:
+ self.check_response_unordered(output, expected_output)
return output
@@ -1160,6 +1276,31 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec
describe_statement = re.sub(r"WITH[\s]*;", "", describe_statement)
return describe_statement
+ def check_response_unordered(self, response, expected_response):
+ """
+ Assert that a response matches a concatenation of expected
+ strings in an arbitrary order. This is useful for features
+ such as DESCRIBE KEYSPACE, where output is arbitrarily
+ ordered.
+ expected_response should be a list of strings that form the
+ expected output.
+ """
+
+ def consume_expected(observed, expected):
+ unconsumed = observed
+ if isinstance(expected, list):
+ for unit in expected:
+ unconsumed = consume_expected(unconsumed, unit)
+ else:
+ expected_stripped = "\n".join([s.strip() for s in expected.split("\n") if s.strip()])
+ assert unconsumed.find(expected_stripped) >= 0
+ unconsumed = unconsumed.replace(expected_stripped, "")
+ return unconsumed
+
+ stripped_response = "\n".join([s.strip() for s in response.split("\n") if s.strip()])
+ unconsumed = consume_expected(stripped_response, expected_response)
+ assert unconsumed.replace("\n", "") == ""
+
def strip_read_repair_chance(self, describe_statement):
"""
Remove read_repair_chance and dclocal_read_repair_chance options
@@ -1425,7 +1566,7 @@ CREATE TABLE int_checks.values (
val4 tinyint
""")
- @since('2.2')
+ @since('2.2', max_version='4')
def test_datetime_values(self):
""" Tests for CASSANDRA-9399, check tables with date and time values"""
self.cluster.populate(1)
@@ -1468,6 +1609,52 @@ CREATE TABLE datetime_checks.values (
PRIMARY KEY (d, t)
""")
+ """
+ Starting with 4.0, date/time format needs to conform to java.time.format.DateTimeFormatter
+ See CASSANDRA-15257 for more details
+ """
+ @since('4.0')
+ def test_datetime_values_40(self):
+ self.cluster.populate(1)
+ self.cluster.start(wait_for_binary_proto=True)
+
+ node1, = self.cluster.nodelist()
+
+ stdout, stderr = self.run_cqlsh(node1, cmds="""
+ CREATE KEYSPACE datetime_checks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};
+ USE datetime_checks;
+ CREATE TABLE values (d date, t time, PRIMARY KEY (d, t));
+ INSERT INTO values (d, t) VALUES ('9800-12-31', '23:59:59.999999999');
+ INSERT INTO values (d, t) VALUES ('2015-05-14', '16:30:00.555555555');
+ INSERT INTO values (d, t) VALUES ('1582-01-01', '00:00:00.000000000');
+ INSERT INTO values (d, t) VALUES ('%04d-01-01', '00:00:00.000000000');
+ INSERT INTO values (d, t) VALUES ('%04d-01-01', '01:00:00.000000000');
+ INSERT INTO values (d, t) VALUES ('%02d-01-01', '02:00:00.000000000');
+ INSERT INTO values (d, t) VALUES ('+%02d-01-01', '03:00:00.000000000')"""
+ % (datetime.MINYEAR - 1, datetime.MINYEAR, datetime.MAXYEAR, datetime.MAXYEAR+1))
+
+ assert 0 == len(stderr), "Failed to execute cqlsh: {}".format(stderr)
+
+ self.verify_output("select * from datetime_checks.values", node1, """
+ d | t
+------------+--------------------
+ -719528 | 00:00:00.000000000
+ 9800-12-31 | 23:59:59.999999999
+ 0001-01-01 | 01:00:00.000000000
+ 1582-01-01 | 00:00:00.000000000
+ 2932897 | 03:00:00.000000000
+ 9999-01-01 | 02:00:00.000000000
+ 2015-05-14 | 16:30:00.555555555
+""")
+
+ self.verify_output("DESCRIBE TABLE datetime_checks.values", node1, """
+CREATE TABLE datetime_checks.values (
+ d date,
+ t time,
+ PRIMARY KEY (d, t)
+""")
+
+
@since('2.2')
def test_tracing(self):
"""
@@ -1767,7 +1954,7 @@ Tracing session:""")
# Can't check escape sequence on cmd prompt. Assume no errors is good enough metric.
if not common.is_win():
- assert re.search(chr(27) + r"\[[0,1,2]?J", out)
+ assert re.search((chr(27) + "\[[0,1,2]?J"), out)
def test_batch(self):
"""
@@ -2088,6 +2275,34 @@ class TestCqlshSmoke(Tester):
self.session.cluster.refresh_schema_metadata()
return [table.name for table in list(self.session.cluster.metadata.keyspaces[keyspace].tables.values())]
+ def test_cjk_output(self):
+ """Confirm cqlsh outputs CJK text properly"""
+ create_ks(self.session, 'ks', 1)
+ create_cf(self.session, 'iroha', key_type='int', columns={'manyogana': 'text', 'modern': 'text', 'kana': 'text'})
+
+ self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (1, '以呂波耳本部止', '色は匂へど', 'いろはにほへと')")
+ self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (2, '千利奴流乎', '散りぬるを', 'ちりぬるを')")
+ self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (3, '和加餘多連曽', '我が世誰ぞ', 'わかよたれそ')")
+ self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (4, '津祢那良牟', '常ならん', 'つねならむ')")
+ self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (5, '有為能於久耶万', '有為の奥山', 'うゐのおくやま')")
+ self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (6, '計不己衣天阿', '今日越えて', 'けふこえて')")
+ self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (7, '佐伎喩女美之', '浅き夢見じ', 'あさきゆめみし')")
+ self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (8, '恵比毛勢須', '酔ひもせず', 'ゑひもせす')")
+
+ stdout, _, _ = self.node1.run_cqlsh('SELECT key, manyogana, modern, kana FROM ks.iroha')
+ stdout_lines_sorted = '\n'.join(sorted(stdout.split('\n')))
+ expected = """
+ 1 | 以呂波耳本部止 | 色は匂へど | いろはにほへと
+ 2 | 千利奴流乎 | 散りぬるを | ちりぬるを
+ 3 | 和加餘多連曽 | 我が世誰ぞ | わかよたれそ
+ 4 | 津祢那良牟 | 常ならん | つねならむ
+ 5 | 有為能於久耶万 | 有為の奥山 | うゐのおくやま
+ 6 | 計不己衣天阿 | 今日越えて | けふこえて
+ 7 | 佐伎喩女美之 | 浅き夢見じ | あさきゆめみし
+ 8 | 恵比毛勢須 | 酔ひもせず | ゑひもせす
+"""
+ assert stdout_lines_sorted.find(expected) >= 0
+
class TestCqlLogin(Tester):
"""
@@ -2145,7 +2360,7 @@ class TestCqlLogin(Tester):
create_cf(self.session, 'ks1table')
self.session.execute("CREATE USER user1 WITH PASSWORD 'changeme';")
- if self.cluster.version() >= '2.2':
+ if self.cluster.version() >= LooseVersion('2.2'):
query = '''
LOGIN user1 'changeme';
CREATE USER user2 WITH PASSWORD 'fail' SUPERUSER;
diff --git a/cqlsh_tests/test_cqlsh_copy.py b/cqlsh_tests/test_cqlsh_copy.py
index a170c7e..d634655 100644
--- a/cqlsh_tests/test_cqlsh_copy.py
+++ b/cqlsh_tests/test_cqlsh_copy.py
@@ -1,14 +1,17 @@
+# coding=utf-8
+
import csv
import datetime
import glob
+import io
import json
import locale
+import logging
import os
+import pytest
import re
import sys
import time
-import pytest
-import logging
from collections import namedtuple
from contextlib import contextmanager
@@ -25,9 +28,11 @@ from cassandra.murmur3 import murmur3
from cassandra.util import SortedSet
from ccmlib.common import is_win
-from .cqlsh_tools import (DummyColorMap, assert_csvs_items_equal, csv_rows,
- monkeypatch_driver, random_list, unmonkeypatch_driver,
- write_rows_to_csv)
+from .cqlsh_test_types import (Address, Datetime, ImmutableDict,
+ ImmutableSet, Name, UTC)
+from .cqlsh_tools import (DummyColorMap, assert_csvs_items_equal,
+ csv_rows, monkeypatch_driver, random_list,
+ unmonkeypatch_driver, write_rows_to_csv)
from dtest import (Tester, create_ks)
from dtest import (FlakyRetryPolicy, Tester, create_ks)
from tools.data import rows_to_list
@@ -45,21 +50,6 @@ PARTITIONERS = {
}
-class UTC(datetime.tzinfo):
- """
- A utility class to specify a UTC timezone.
- """
-
- def utcoffset(self, dt):
- return datetime.timedelta(0)
-
- def tzname(self, dt):
- return "UTC"
-
- def dst(self, dt):
- return datetime.timedelta(0)
-
-
class TestCqlshCopy(Tester):
"""
Tests the COPY TO and COPY FROM features in cqlsh.
@@ -205,7 +195,7 @@ class TestCqlshCopy(Tester):
try:
return self._default_time_format
except AttributeError:
- with self._cqlshlib():
+ with self._cqlshlib() as cqlshlib:
try:
from cqlshlib.formatting import DEFAULT_TIMESTAMP_FORMAT
self._default_time_format = DEFAULT_TIMESTAMP_FORMAT
@@ -250,68 +240,14 @@ class TestCqlshCopy(Tester):
x map<text, frozen<list<text>>>
)''')
- default_time_format = self.default_time_format
-
- try:
- from cqlshlib.formatting import round_microseconds
- except ImportError:
- round_microseconds = None
-
- class Datetime(datetime.datetime):
-
- def _format_for_csv(self):
- ret = self.strftime(default_time_format)
- return round_microseconds(ret) if round_microseconds else ret
-
- def __str__(self):
- return self._format_for_csv()
-
- def __repr__(self):
- return self._format_for_csv()
-
- def maybe_quote(s):
- """
- Return a quoted string representation for strings, unicode and date time parameters,
- otherwise return a string representation of the parameter.
- """
- return "'{}'".format(s) if isinstance(s, (str, Datetime)) else str(s)
-
- class ImmutableDict(frozenset):
- iteritems = frozenset.__iter__
-
- def __repr__(self):
- return '{{{}}}'.format(', '.join(['{}: {}'.format(maybe_quote(t[0]), maybe_quote(t[1]))
- for t in sorted(self)]))
-
- class ImmutableSet(SortedSet):
-
- def __repr__(self):
- return '{{{}}}'.format(', '.join([maybe_quote(t) for t in sorted(self._items)]))
-
- def __hash__(self):
- return hash(tuple([e for e in self]))
-
- class Name(namedtuple('Name', ('firstname', 'lastname'))):
- __slots__ = ()
-
- def __repr__(self):
- return "{{firstname: '{}', lastname: '{}'}}".format(self.firstname, self.lastname)
-
- class Address(namedtuple('Address', ('name', 'number', 'street', 'phones'))):
- __slots__ = ()
-
- def __repr__(self):
- phones_str = "{{{}}}".format(', '.join(maybe_quote(p) for p in sorted(self.phones)))
- return "{{name: {}, number: {}, street: '{}', phones: {}}}".format(self.name,
- self.number,
- self.street,
- phones_str)
-
self.session.cluster.register_user_type('ks', 'name_type', Name)
self.session.cluster.register_user_type('ks', 'address_type', Address)
- date1 = Datetime(2005, 7, 14, 12, 30, 0, 0, UTC())
- date2 = Datetime(2005, 7, 14, 13, 30, 0, 0, UTC())
+ cassandra_dirpath = self.cluster.nodelist()[0].get_install_dir()
+ cqlshlib_dirpath = os.path.join(cassandra_dirpath, 'pylib')
+
+ date1 = Datetime(2005, 7, 14, 12, 30, 0, 0, UTC(), cqlshlib_path=cqlshlib_dirpath)
+ date2 = Datetime(2005, 7, 14, 13, 30, 0, 0, UTC(), cqlshlib_path=cqlshlib_dirpath)
addr1 = Address(Name('name1', 'last1'), 1, 'street 1', ImmutableSet(['1111 2222', '3333 4444']))
addr2 = Address(Name('name2', 'last2'), 2, 'street 2', ImmutableSet(['5555 6666', '7777 8888']))
@@ -388,12 +324,11 @@ class TestCqlshCopy(Tester):
self.maxDiff = None
- if (sort_data):
- csv_results.sort()
- processed_results.sort()
-
try:
- assert csv_results == processed_results
+ if sort_data:
+ assert sorted(csv_results) == sorted(processed_results)
+ else:
+ assert csv_results == processed_results
except Exception as e:
if len(csv_results) != len(processed_results):
logger.warning("Different # of entries. CSV: " + str(len(csv_results)) +
@@ -409,15 +344,14 @@ class TestCqlshCopy(Tester):
def make_csv_formatter(self, time_format, nullval):
with self._cqlshlib() as cqlshlib: # noqa
from cqlshlib.formatting import format_value, format_value_default
- from cqlshlib.displaying import NO_COLOR_MAP
- try:
- from cqlshlib.formatting import DateTimeFormat
- date_time_format = DateTimeFormat()
- date_time_format.timestamp_format = time_format
- if hasattr(date_time_format, 'milliseconds_only'):
- date_time_format.milliseconds_only = True
- except ImportError:
- date_time_format = None
+ try:
+ from cqlshlib.formatting import DateTimeFormat
+ date_time_format = DateTimeFormat()
+ date_time_format.timestamp_format = time_format
+ if hasattr(date_time_format, 'milliseconds_only'):
+ date_time_format.milliseconds_only = True
+ except ImportError:
+ date_time_format = None
encoding_name = 'utf-8' # codecs.lookup(locale.getpreferredencoding()).name
color_map = DummyColorMap()
@@ -439,7 +373,7 @@ class TestCqlshCopy(Tester):
format_fn = format_value
if val is None or val == EMPTY or val == nullval:
- return format_value_default(nullval)
+ return format_value_default(nullval, color_map).strval
# CASSANDRA-11255 increased COPY TO DOUBLE PRECISION TO 12
if cql_type_name == 'double' and self.cluster.version() >= LooseVersion('3.6'):
@@ -804,7 +738,6 @@ class TestCqlshCopy(Tester):
result = self.session.execute("SELECT * FROM testcounter")
result_as_list = rows_to_list(result)
- result_as_list.sort()
assert data == sorted(result_as_list)
def test_reading_counter(self):
@@ -859,6 +792,7 @@ class TestCqlshCopy(Tester):
result_as_list = [tuple(r) for r in rows_to_list(result)]
assert [tuple(d) for d in data] == sorted(result_as_list)
+ @since('2.2', max_version='3.X')
@pytest.mark.depends_cqlshlib
def test_datetimeformat_round_trip(self):
"""
@@ -894,7 +828,7 @@ class TestCqlshCopy(Tester):
logger.debug('Exporting to csv file: {name}'.format(name=tempfile.name))
cmds = "COPY ks.testdatetimeformat TO '{name}'".format(name=tempfile.name)
cmds += " WITH DATETIMEFORMAT = '{}'".format(format)
- self.run_cqlsh(cmds=cmds)
+ copy_to_out, copy_to_err, _ = self.run_cqlsh(cmds=cmds)
with open(tempfile.name, 'r') as csvfile:
csv_values = list(csv.reader(csvfile))
@@ -906,7 +840,66 @@ class TestCqlshCopy(Tester):
self.session.execute("TRUNCATE testdatetimeformat")
cmds = "COPY ks.testdatetimeformat FROM '{name}'".format(name=tempfile.name)
cmds += " WITH DATETIMEFORMAT = '{}'".format(format)
- self.run_cqlsh(cmds=cmds)
+ copy_from_out, copy_from_err, _ = self.run_cqlsh(cmds=cmds)
+
+ table_meta = UpdatingTableMetadataWrapper(self.session.cluster,
+ ks_name=self.ks,
+ table_name='testdatetimeformat')
+ cql_type_names = [table_meta.columns[c].cql_type for c in table_meta.columns]
+
+ imported_results = list(self.session.execute("SELECT * FROM testdatetimeformat"))
+ assert self.result_to_csv_rows(exported_results, cql_type_names, time_format=format) \
+ == self.result_to_csv_rows(imported_results, cql_type_names, time_format=format)
+
+ @since('4.0')
+ @pytest.mark.depends_cqlshlib
+ def test_datetimeformat_round_trip_40(self):
+ """
+ @jira_ticket CASSANDRA-10633
+ @jira_ticket CASSANDRA-9303
+
+ Test COPY TO and COPY FORM with the time format specified in the WITH option by:
+
+ - creating and populating a table,
+ - exporting the contents of the table to a CSV file using COPY TO WITH DATETIMEFORMAT,
+ - checking the time format written to csv.
+ - importing the CSV back into the table
+ - comparing the table contents before and after the import
+
+ CASSANDRA-9303 renamed TIMEFORMAT to DATETIMEFORMAT
+ """
+ self.prepare()
+ self.session.execute("""
+ CREATE TABLE testdatetimeformat (
+ a int primary key,
+ b timestamp
+ )""")
+ insert_statement = self.session.prepare("INSERT INTO testdatetimeformat (a, b) VALUES (?, ?)")
+ args = [(1, datetime.datetime(2015, 1, 1, 0o7, 00, 0, 0, UTC())),
+ (2, datetime.datetime(2015, 6, 10, 12, 30, 30, 500, UTC())),
+ (3, datetime.datetime(2015, 12, 31, 23, 59, 59, 999, UTC()))]
+ execute_concurrent_with_args(self.session, insert_statement, args)
+ exported_results = list(self.session.execute("SELECT * FROM testdatetimeformat"))
+
+ format = '%Y-%m-%d %H:%M:%S%z'
+
+ tempfile = self.get_temp_file()
+ logger.debug('Exporting to csv file: {name}'.format(name=tempfile.name))
+ cmds = "COPY ks.testdatetimeformat TO '{name}'".format(name=tempfile.name)
+ cmds += " WITH DATETIMEFORMAT = '{}' AND NUMPROCESSES=1".format(format)
+ copy_to_out, copy_to_err, _ = self.run_cqlsh(cmds=cmds)
+
+ with open(tempfile.name, 'r') as csvfile:
+ csv_values = list(csv.reader(csvfile))
+
+ assert sorted(csv_values) == [['1', '2015-01-01 07:00:00+0000'],
+ ['2', '2015-06-10 12:30:30+0000'],
+ ['3', '2015-12-31 23:59:59+0000']]
+
+ self.session.execute("TRUNCATE testdatetimeformat")
+ cmds = "COPY ks.testdatetimeformat FROM '{name}'".format(name=tempfile.name)
+ cmds += " WITH DATETIMEFORMAT = '{}' AND NUMPROCESSES=1".format(format)
+ copy_from_out, copy_from_err, _ = self.run_cqlsh(cmds=cmds)
table_meta = UpdatingTableMetadataWrapper(self.session.cluster,
ks_name=self.ks,
@@ -1310,10 +1303,9 @@ class TestCqlshCopy(Tester):
cmd += " AND ERRFILE='{}'".format(err_file.name)
self.run_cqlsh(cmds=cmd)
- logger.debug('Sorting')
- results = sorted(rows_to_list(self.session.execute("SELECT * FROM ks.testparseerrors")))
+ results = rows_to_list(self.session.execute("SELECT * FROM ks.testparseerrors"))
logger.debug('Checking valid rows')
- assert valid_rows == results
+ assert sorted(valid_rows) == sorted(results)
logger.debug('Checking invalid rows')
self.assertCsvResultEqual(err_file_name, invalid_rows, cql_type_names=['text', 'int', 'text'])
@@ -1368,10 +1360,9 @@ class TestCqlshCopy(Tester):
cmd = "COPY ks.testwrongnumcols FROM '{}' WITH ERRFILE='{}'".format(tempfile.name, err_file.name)
self.run_cqlsh(cmds=cmd)
- logger.debug('Sorting')
- results = sorted(rows_to_list(self.session.execute("SELECT * FROM ks.testwrongnumcols")))
+ results = rows_to_list(self.session.execute("SELECT * FROM ks.testwrongnumcols"))
logger.debug('Checking valid rows')
- assert valid_rows == results
+ assert sorted(valid_rows) == sorted(results)
logger.debug('Checking invalid rows')
self.assertCsvResultEqual(err_file.name, invalid_rows, 'testwrongnumcols', columns=['a', 'b', 'e'])
@@ -1802,7 +1793,7 @@ class TestCqlshCopy(Tester):
def _test(prepared_statements):
logger.debug('Importing from csv file: {name}'.format(name=tempfile.name))
- self.run_cqlsh(cmds="COPY ks.testdatatype FROM '{}' WITH PREPAREDSTATEMENTS = {}"
+ out, err, _ = self.run_cqlsh(cmds="COPY ks.testdatatype FROM '{}' WITH PREPAREDSTATEMENTS = {}"
.format(tempfile.name, prepared_statements))
results = list(self.session.execute("SELECT * FROM testdatatype"))
@@ -2043,7 +2034,7 @@ class TestCqlshCopy(Tester):
exported_results = list(self.session.execute("SELECT * FROM testnumberseps"))
self.maxDiff = None
- assert expected_vals == sorted(list(csv_rows(tempfile.name)))
+ assert sorted(expected_vals) == sorted(list(csv_rows(tempfile.name)))
logger.debug('Importing from csv file: {} with thousands_sep {} and decimal_sep {}'
.format(tempfile.name, thousands_sep, decimal_sep))
@@ -2060,8 +2051,8 @@ class TestCqlshCopy(Tester):
cql_type_names = [table_meta.columns[c].cql_type for c in table_meta.columns]
# we format as if we were comparing to csv to overcome loss of precision in the import
- assert self.result_to_csv_rows(exported_results == cql_type_names,
- self.result_to_csv_rows(imported_results, cql_type_names))
+ assert self.result_to_csv_rows(exported_results, cql_type_names) \
+ == self.result_to_csv_rows(imported_results, cql_type_names)
do_test(expected_vals_usual, ',', '.')
do_test(expected_vals_inverted, '.', ',')
@@ -2532,7 +2523,8 @@ class TestCqlshCopy(Tester):
run_copy_to(tempfile1)
# check all records generated were exported
- assert num_records == sum(1 for _ in open(tempfile1.name))
+ with io.open(tempfile1.name, encoding="utf-8", newline='') as csvfile:
+ assert num_records == sum(1 for _ in csv.reader(csvfile, quotechar='"', escapechar='\\'))
# import records from the first csv file
logger.debug('Truncating {}...'.format(stress_table))
@@ -3266,6 +3258,7 @@ class TestCqlshCopy(Tester):
_test(False)
@pytest.mark.depends_cqlshlib
+ @pytest.mark.depends_driver
@since('3.0')
def test_unusual_dates(self):
"""
diff --git a/dtest.py b/dtest.py
index d6aff2d..e144a58 100644
--- a/dtest.py
+++ b/dtest.py
@@ -227,7 +227,7 @@ def test_failure_due_to_timeout(err, *args):
@flaky(rerun_filter=test_failure_due_to_timeout)
-class Tester:
+class Tester(object):
def __getattribute__(self, name):
try:
diff --git a/dtest_setup.py b/dtest_setup.py
index 4b2ece9..107028d 100644
--- a/dtest_setup.py
+++ b/dtest_setup.py
@@ -45,7 +45,7 @@ def retry_till_success(fun, *args, **kwargs):
time.sleep(0.25)
-class DTestSetup:
+class DTestSetup(object):
def __init__(self, dtest_config=None, setup_overrides=None, cluster_name="test"):
self.dtest_config = dtest_config
self.setup_overrides = setup_overrides
diff --git a/requirements.txt b/requirements.txt
index 3b17b82..139bfc5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,6 @@
-e git+https://github.com/datastax/python-driver.git@cassandra-test#egg=cassandra-driver
# Used ccm version is tracked by cassandra-test branch in ccm repo. Please create a PR there for fixes or upgrades to new releases.
-e git+https://github.com/riptano/ccm.git@cassandra-test#egg=ccm
-cqlsh
decorator
docopt
enum34
diff --git a/run_dtests.py b/run_dtests.py
index 378029d..698977b 100755
--- a/run_dtests.py
+++ b/run_dtests.py
@@ -109,9 +109,9 @@ class RunDTests():
logger.debug('Generating configurations from the following matrix:\n\t{}'.format(args))
args_to_invoke_pytest = []
- if args.pytest_options:
- for arg in args.pytest_options.split(" "):
- args_to_invoke_pytest.append("'{the_arg}'".format(the_arg=arg))
+ # if args.pytest_options:
+ # for arg in args.pytest_options.split(" "):
+ # args_to_invoke_pytest.append("'{the_arg}'".format(the_arg=arg))
for arg in argv:
if arg.startswith("--pytest-options") or arg.startswith("--dtest-"):
diff --git a/tools/metadata_wrapper.py b/tools/metadata_wrapper.py
index 43bdbfb..c166283 100644
--- a/tools/metadata_wrapper.py
+++ b/tools/metadata_wrapper.py
@@ -1,7 +1,7 @@
from abc import ABCMeta, abstractproperty
+from six import with_metaclass
-
-class UpdatingMetadataWrapperBase(object, metaclass=ABCMeta):
+class UpdatingMetadataWrapperBase(with_metaclass(ABCMeta, object)):
@abstractproperty
def _wrapped(self):
pass
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org