You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by st...@apache.org on 2020/04/28 15:55:28 UTC
[phoenix-queryserver] branch master updated: PHOENIX-5844 Feature
parity for the python client (part2: transaction and array support)
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-queryserver.git
The following commit(s) were added to refs/heads/master by this push:
new 992d555 PHOENIX-5844 Feature parity for the python client (part2: transaction and array support)
992d555 is described below
commit 992d555ce9c793f7f2630315298ec39a1626dced
Author: Istvan Toth <st...@apache.org>
AuthorDate: Mon Apr 20 06:30:33 2020 +0200
PHOENIX-5844 Feature parity for the python client (part2: transaction and array support)
includes:
PHOENIX-5857 Get the python test suite working
PHOENIX-5858 Add commit/rollback support to the python client
PHOENIX-5859 Add Array support to the python driver
---
python/phoenixdb/NEWS.rst | 4 +
python/phoenixdb/README.rst | 5 -
python/phoenixdb/phoenixdb/__init__.py | 2 +-
python/phoenixdb/phoenixdb/avatica/client.py | 28 +++-
python/phoenixdb/phoenixdb/connection.py | 11 +-
python/phoenixdb/phoenixdb/cursor.py | 49 ++++--
python/phoenixdb/phoenixdb/tests/__init__.py | 35 +++--
.../phoenixdb/phoenixdb/tests/test_connection.py | 42 ------
python/phoenixdb/phoenixdb/tests/test_db.py | 54 +++----
python/phoenixdb/phoenixdb/tests/test_dbapi20.py | 4 +-
python/phoenixdb/phoenixdb/tests/test_errors.py | 6 +-
python/phoenixdb/phoenixdb/tests/test_types.py | 61 +++++---
python/phoenixdb/phoenixdb/types.py | 165 ++++++++++++++++-----
queryserver-it/src/it/bin/test_phoenixdb.py | 4 +-
queryserver-it/src/it/bin/test_phoenixdb.sh | 25 +++-
.../phoenix/end2end/QueryServerBasicsIT.java | 45 +++++-
.../end2end/SecureQueryServerPhoenixDBIT.java | 76 ++++++++--
queryserver/src/it/bin/test_phoenixdb.py | 39 -----
queryserver/src/it/bin/test_phoenixdb.sh | 78 ----------
19 files changed, 424 insertions(+), 309 deletions(-)
diff --git a/python/phoenixdb/NEWS.rst b/python/phoenixdb/NEWS.rst
index 285b069..3d4cff1 100644
--- a/python/phoenixdb/NEWS.rst
+++ b/python/phoenixdb/NEWS.rst
@@ -8,6 +8,10 @@ Unreleased
- Support for specifying server certificate
- Support for BASIC and DIGEST authentication
- Fix HTTP error parsing
+- Add transaction support
+- Add list support
+- Rewrite type handling
+- Refactor test suite
Version 0.7
-----------
diff --git a/python/phoenixdb/README.rst b/python/phoenixdb/README.rst
index c74104c..8388517 100644
--- a/python/phoenixdb/README.rst
+++ b/python/phoenixdb/README.rst
@@ -125,12 +125,7 @@ Similarly, tox can be used to run the test suite against multiple Python version
Known issues
------------
-- You can only use the library in autocommit mode. The native Java Phoenix library also implements batched upserts, which can be committed at once, but this is not exposed over the remote server. This was previously unimplemented due to CALCITE-767, however this functionality exists in the server, but is lacking in the driver.
- (`CALCITE-767 <https://issues.apache.org/jira/browse/CALCITE-767>`_)
- TIME and DATE columns in Phoenix are stored as full timestamps with a millisecond accuracy,
but the remote protocol only exposes the time (hour/minute/second) or date (year/month/day)
parts of the columns. (`CALCITE-797 <https://issues.apache.org/jira/browse/CALCITE-797>`_, `CALCITE-798 <https://issues.apache.org/jira/browse/CALCITE-798>`_)
- TIMESTAMP columns in Phoenix are stored with a nanosecond accuracy, but the remote protocol truncates them to milliseconds. (`CALCITE-796 <https://issues.apache.org/jira/browse/CALCITE-796>`_)
-- ARRAY columns are not supported. Again, this previously lacked server-side support which has since been built. The
- driver needs to be updated to support this functionality.
- (`CALCITE-1050 <https://issues.apache.org/jira/browse/CALCITE-1050>`_, `PHOENIX-2585 <https://issues.apache.org/jira/browse/PHOENIX-2585>`_)
diff --git a/python/phoenixdb/phoenixdb/__init__.py b/python/phoenixdb/phoenixdb/__init__.py
index aee519b..0a444e0 100644
--- a/python/phoenixdb/phoenixdb/__init__.py
+++ b/python/phoenixdb/phoenixdb/__init__.py
@@ -101,7 +101,7 @@ def connect(url, max_retries=None, auth=None, authentication=None, avatica_user=
auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech = spnego)
elif auth is None and authentication is not None:
if authentication == "SPNEGO":
- auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech = spnego)
+ auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech = spnego, opportunistic_auth=True)
elif authentication == "BASIC" and avatica_user is not None and avatica_password is not None:
auth = HTTPBasicAuth(avatica_user, avatica_password)
elif authentication == "DIGEST" and avatica_user is not None and avatica_password is not None:
diff --git a/python/phoenixdb/phoenixdb/avatica/client.py b/python/phoenixdb/phoenixdb/avatica/client.py
index 864a936..e392aee 100644
--- a/python/phoenixdb/phoenixdb/avatica/client.py
+++ b/python/phoenixdb/phoenixdb/avatica/client.py
@@ -87,8 +87,12 @@ SQLSTATE_ERROR_CLASSES = [
# Relevant properties as defined by https://calcite.apache.org/avatica/docs/client_reference.html
OPEN_CONNECTION_PROPERTIES = (
- 'user', # User for the database connection
- 'password', # Password for the user
+ 'avatica_user', # User for the database connection
+ 'avatica_password', # Password for the user
+ 'auth',
+ 'authentication',
+ 'truststore',
+ 'verify'
)
@@ -503,3 +507,23 @@ class AvaticaClient(object):
response = responses_pb2.FetchResponse()
response.ParseFromString(response_data)
return response.frame
+
+ def commit(self, connection_id):
+ """TODO Commits the transaction
+
+ :param connection_id:
+ ID of the current connection.
+ """
+ request = requests_pb2.CommitRequest()
+ request.connection_id = connection_id
+ return self._apply(request)
+
+ def rollback(self, connection_id):
+ """TODO Rolls back the transaction
+
+ :param connection_id:
+ ID of the current connection.
+ """
+ request = requests_pb2.RollbackRequest()
+ request.connection_id = connection_id
+ return self._apply(request)
diff --git a/python/phoenixdb/phoenixdb/connection.py b/python/phoenixdb/phoenixdb/connection.py
index 593a242..e041425 100644
--- a/python/phoenixdb/phoenixdb/connection.py
+++ b/python/phoenixdb/phoenixdb/connection.py
@@ -97,15 +97,14 @@ class Connection(object):
return self._closed
def commit(self):
- """Commits pending database changes.
+ if self._closed:
+ raise ProgrammingError('the connection is already closed')
+ self._client.commit(self._id);
- Currently, this does nothing, because the RPC does not support
- transactions. Only defined for DB API 2.0 compatibility.
- You need to use :attr:`autocommit` mode.
- """
- # TODO can support be added for this?
+ def rollback(self):
if self._closed:
raise ProgrammingError('the connection is already closed')
+ self._client.rollback(self._id);
def cursor(self, cursor_factory=None):
"""Creates a new cursor.
diff --git a/python/phoenixdb/phoenixdb/cursor.py b/python/phoenixdb/phoenixdb/cursor.py
index 8be7bed..e1557c8 100644
--- a/python/phoenixdb/phoenixdb/cursor.py
+++ b/python/phoenixdb/phoenixdb/cursor.py
@@ -137,11 +137,11 @@ class Cursor(object):
return
for column in signature.columns:
- dtype = TypeHelper.from_class(column.column_class_name)
+ dtype = TypeHelper.from_column(column)
self._column_data_types.append(dtype)
for parameter in signature.parameters:
- dtype = TypeHelper.from_class(parameter.class_name)
+ dtype = TypeHelper.from_param(parameter)
self._parameter_data_types.append(dtype)
def _set_frame(self, frame):
@@ -173,7 +173,7 @@ class Cursor(object):
def _transform_parameters(self, parameters):
typed_parameters = []
for value, data_type in zip(parameters, self._parameter_data_types):
- field_name, rep, mutate_to, cast_from = data_type
+ field_name, rep, mutate_to, cast_from, is_array = data_type
typed_value = common_pb2.TypedValue()
if value is None:
@@ -181,13 +181,27 @@ class Cursor(object):
typed_value.type = common_pb2.NULL
else:
typed_value.null = False
-
- # use the mutator function
- if mutate_to is not None:
- value = mutate_to(value)
-
- typed_value.type = rep
- setattr(typed_value, field_name, value)
+ if is_array:
+ if type(value) in [list,tuple]:
+ for element in value:
+ if mutate_to is not None:
+ element = mutate_to(element)
+ typed_element = common_pb2.TypedValue()
+ if element is None:
+ typed_element.null = True
+ else:
+ typed_element.type = rep
+ setattr(typed_element, field_name, element)
+ typed_value.array_value.append(typed_element)
+ typed_value.type = common_pb2.ARRAY
+ typed_value.component_type = rep
+ else:
+ raise ProgrammingError('scalar value specified for array parameter')
+ else:
+ if mutate_to is not None:
+ value = mutate_to(value)
+ typed_value.type = rep
+ setattr(typed_value, field_name, value)
typed_parameters.append(typed_value)
return typed_parameters
@@ -246,10 +260,19 @@ class Cursor(object):
tmp_row = []
for i, column in enumerate(row.value):
- if column.has_array_value:
- raise NotImplementedError('array types are not supported')
- elif column.scalar_value.null:
+ if column.scalar_value.null:
tmp_row.append(None)
+ elif column.has_array_value:
+ field_name, rep, mutate_to, cast_from = self._column_data_types[i]
+
+ list_value = []
+ for j, typed_value in enumerate(column.array_value):
+ value = getattr(typed_value, field_name)
+ if cast_from is not None:
+ value = cast_from(value)
+ list_value.append(value)
+
+ tmp_row.append(list_value)
else:
field_name, rep, mutate_to, cast_from = self._column_data_types[i]
diff --git a/python/phoenixdb/phoenixdb/tests/__init__.py b/python/phoenixdb/phoenixdb/tests/__init__.py
index ec9a79b..44b62f7 100644
--- a/python/phoenixdb/phoenixdb/tests/__init__.py
+++ b/python/phoenixdb/phoenixdb/tests/__init__.py
@@ -16,29 +16,46 @@
import os
import unittest
import phoenixdb
+import time
TEST_DB_URL = os.environ.get('PHOENIXDB_TEST_DB_URL')
-
+#TEST_DB_URL = "http://localhost:8765"
+TEST_DB_TRUSTSTORE = os.environ.get('PHOENIXDB_TEST_DB_TRUSTSTORE')
+TEST_DB_AUTHENTICATION = os.environ.get('PHOENIXDB_TEST_DB_AUTHENTICATION')
+TEST_DB_AVATICA_USER = os.environ.get('PHOENIXDB_TEST_DB_AVATICA_USER')
+TEST_DB_AVATICA_PASSWORD = os.environ.get('PHOENIXDB_TEST_DB_AVATICA_PASSWORD')
+
+httpArgs = {}
+if TEST_DB_TRUSTSTORE is not None:
+ httpArgs.update(verify = TEST_DB_TRUSTSTORE)
+if TEST_DB_AUTHENTICATION is not None:
+ httpArgs.update(authentication = TEST_DB_AUTHENTICATION)
+if TEST_DB_AVATICA_USER is not None:
+ httpArgs.update(avatica_user = TEST_DB_AVATICA_USER)
+if TEST_DB_AVATICA_PASSWORD is not None:
+ httpArgs.update(avatica_password = TEST_DB_AVATICA_PASSWORD)
@unittest.skipIf(TEST_DB_URL is None, "these tests require the PHOENIXDB_TEST_DB_URL environment variable set to a clean database")
class DatabaseTestCase(unittest.TestCase):
def setUp(self):
- self.conn = phoenixdb.connect(TEST_DB_URL, autocommit=True)
- self.cleanup_tables = []
+ self.conn = phoenixdb.connect(TEST_DB_URL, autocommit=True, **httpArgs)
+ def closeDb():
+ self.conn.close()
+ self.addCleanup(closeDb)
- def tearDown(self):
- self.doCleanups()
+ def reopen(self, **avaticaArgs):
self.conn.close()
+ self.conn = phoenixdb.connect(TEST_DB_URL, **avaticaArgs, **httpArgs)
def addTableCleanup(self, name):
def dropTable():
with self.conn.cursor() as cursor:
- cursor.execute("DROP TABLE IF EXISTS {}".format(name))
+ cursor.execute("DROP TABLE IF EXISTS {table}".format(table = name))
self.addCleanup(dropTable)
- def createTable(self, name, columns):
+ def createTable(self, name, statement):
with self.conn.cursor() as cursor:
- cursor.execute("DROP TABLE IF EXISTS {}".format(name))
- cursor.execute("CREATE TABLE {} ({})".format(name, columns))
+ cursor.execute("DROP TABLE IF EXISTS {table}".format(table = name))
+ cursor.execute(statement.format(table = name))
self.addTableCleanup(name)
diff --git a/python/phoenixdb/phoenixdb/tests/test_connection.py b/python/phoenixdb/phoenixdb/tests/test_connection.py
deleted file mode 100644
index 2deacf5..0000000
--- a/python/phoenixdb/phoenixdb/tests/test_connection.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-import phoenixdb
-from phoenixdb.tests import TEST_DB_URL
-
-
-@unittest.skipIf(TEST_DB_URL is None, "these tests require the PHOENIXDB_TEST_DB_URL environment variable set to a clean database")
-class PhoenixConnectionTest(unittest.TestCase):
-
- def _connect(self, connect_kw_args):
- try:
- r = phoenixdb.connect(TEST_DB_URL, **connect_kw_args)
- except AttributeError:
- self.fail("Failed to connect")
- return r
-
- def test_connection_credentials(self):
- connect_kw_args = {'user': 'SCOTT', 'password': 'TIGER', 'readonly': 'True'}
- con = self._connect(connect_kw_args)
- try:
- self.assertEqual(
- con._connection_args, {'user': 'SCOTT', 'password': 'TIGER'},
- 'Should have extract user and password')
- self.assertEqual(
- con._filtered_args, {'readonly': 'True'},
- 'Should have not extracted foo')
- finally:
- con.close()
diff --git a/python/phoenixdb/phoenixdb/tests/test_db.py b/python/phoenixdb/phoenixdb/tests/test_db.py
index 2fb1a2a..485e223 100644
--- a/python/phoenixdb/phoenixdb/tests/test_db.py
+++ b/python/phoenixdb/phoenixdb/tests/test_db.py
@@ -14,77 +14,62 @@
# limitations under the License.
import unittest
-import phoenixdb
import phoenixdb.cursor
from phoenixdb.errors import InternalError
from phoenixdb.tests import TEST_DB_URL
+from phoenixdb.tests import DatabaseTestCase
@unittest.skipIf(TEST_DB_URL is None, "these tests require the PHOENIXDB_TEST_DB_URL environment variable set to a clean database")
-class PhoenixDatabaseTest(unittest.TestCase):
+class PhoenixDatabaseTest(DatabaseTestCase):
def test_select_literal(self):
- db = phoenixdb.connect(TEST_DB_URL, autocommit=True)
- self.addCleanup(db.close)
-
- with db.cursor() as cursor:
- cursor.execute("DROP TABLE IF EXISTS test")
- cursor.execute("CREATE TABLE test (id INTEGER PRIMARY KEY, text VARCHAR)")
+ with self.conn.cursor() as cursor:
+ self.createTable("test", "CREATE TABLE {table} (id INTEGER PRIMARY KEY, text VARCHAR)")
cursor.executemany("UPSERT INTO test VALUES (?, ?)", [[i, 'text {}'.format(i)] for i in range(10)])
- with db.cursor() as cursor:
+ with self.conn.cursor() as cursor:
cursor.itersize = 4
cursor.execute("SELECT * FROM test WHERE id>1 ORDER BY id")
self.assertEqual(cursor.fetchall(), [[i, 'text {}'.format(i)] for i in range(2, 10)])
def test_select_parameter(self):
- db = phoenixdb.connect(TEST_DB_URL, autocommit=True)
- self.addCleanup(db.close)
-
- with db.cursor() as cursor:
- cursor.execute("DROP TABLE IF EXISTS test")
- cursor.execute("CREATE TABLE test (id INTEGER PRIMARY KEY, text VARCHAR)")
+ with self.conn.cursor() as cursor:
+ self.createTable("test", "CREATE TABLE {table} (id INTEGER PRIMARY KEY, text VARCHAR)")
cursor.executemany("UPSERT INTO test VALUES (?, ?)", [[i, 'text {}'.format(i)] for i in range(10)])
- with db.cursor() as cursor:
+ with self.conn.cursor() as cursor:
cursor.itersize = 4
cursor.execute("SELECT * FROM test WHERE id>? ORDER BY id", [1])
self.assertEqual(cursor.fetchall(), [[i, 'text {}'.format(i)] for i in range(2, 10)])
def _check_dict_cursor(self, cursor):
- cursor.execute("DROP TABLE IF EXISTS test")
- cursor.execute("CREATE TABLE test (id INTEGER PRIMARY KEY, text VARCHAR)")
+ self.createTable("test", "CREATE TABLE {table} (id INTEGER PRIMARY KEY, text VARCHAR)")
cursor.execute("UPSERT INTO test VALUES (?, ?)", [1, 'text 1'])
cursor.execute("SELECT * FROM test ORDER BY id")
self.assertEqual(cursor.fetchall(), [{'ID': 1, 'TEXT': 'text 1'}])
def test_dict_cursor_default_parameter(self):
- db = phoenixdb.connect(TEST_DB_URL, autocommit=True, cursor_factory=phoenixdb.cursor.DictCursor)
- self.addCleanup(db.close)
+ self.reopen(autocommit=True, cursor_factory=phoenixdb.cursor.DictCursor)
- with db.cursor() as cursor:
+ with self.conn.cursor() as cursor:
self._check_dict_cursor(cursor)
def test_dict_cursor_default_attribute(self):
- db = phoenixdb.connect(TEST_DB_URL, autocommit=True)
- db.cursor_factory = phoenixdb.cursor.DictCursor
- self.addCleanup(db.close)
+ self.conn.cursor_factory = phoenixdb.cursor.DictCursor
- with db.cursor() as cursor:
+ with self.conn.cursor() as cursor:
self._check_dict_cursor(cursor)
def test_dict_cursor(self):
- db = phoenixdb.connect(TEST_DB_URL, autocommit=True)
- self.addCleanup(db.close)
+ self.reopen(autocommit=True, cursor_factory=phoenixdb.cursor.DictCursor)
- with db.cursor(cursor_factory=phoenixdb.cursor.DictCursor) as cursor:
+ with self.conn.cursor(cursor_factory=phoenixdb.cursor.DictCursor) as cursor:
self._check_dict_cursor(cursor)
def test_schema(self):
- db = phoenixdb.connect(TEST_DB_URL, autocommit=True)
- self.addCleanup(db.close)
- with db.cursor() as cursor:
+ with self.conn.cursor() as cursor:
try:
cursor.execute("CREATE SCHEMA IF NOT EXISTS test_schema")
except InternalError as e:
@@ -92,8 +77,11 @@ class PhoenixDatabaseTest(unittest.TestCase):
self.skipTest(e.message)
raise
- cursor.execute("DROP TABLE IF EXISTS test_schema.test")
- cursor.execute("CREATE TABLE test_schema.test (id INTEGER PRIMARY KEY, text VARCHAR)")
+ self.createTable("test_schema.test", "CREATE TABLE {table} (id INTEGER PRIMARY KEY, text VARCHAR)")
cursor.execute("UPSERT INTO test_schema.test VALUES (?, ?)", [1, 'text 1'])
cursor.execute("SELECT * FROM test_schema.test ORDER BY id")
self.assertEqual(cursor.fetchall(), [[1, 'text 1']])
+
+ def test_transaction(self):
+ #Todo write some transaction tests
+ pass
\ No newline at end of file
diff --git a/python/phoenixdb/phoenixdb/tests/test_dbapi20.py b/python/phoenixdb/phoenixdb/tests/test_dbapi20.py
index 0e5c2e4..1be6cb0 100644
--- a/python/phoenixdb/phoenixdb/tests/test_dbapi20.py
+++ b/python/phoenixdb/phoenixdb/tests/test_dbapi20.py
@@ -17,12 +17,14 @@ import unittest
import phoenixdb
from . import dbapi20
from phoenixdb.tests import TEST_DB_URL
+from phoenixdb.tests import httpArgs
@unittest.skipIf(TEST_DB_URL is None, "these tests require the PHOENIXDB_TEST_DB_URL environment variable set to a clean database")
class PhoenixDatabaseAPI20Test(dbapi20.DatabaseAPI20Test):
driver = phoenixdb
- connect_args = (TEST_DB_URL, )
+ connect_args = (TEST_DB_URL,)
+ connect_kw_args = httpArgs
ddl1 = 'create table %sbooze (name varchar(20) primary key)' % dbapi20.DatabaseAPI20Test.table_prefix
ddl2 = 'create table %sbarflys (name varchar(20) primary key, drink varchar(30))' % dbapi20.DatabaseAPI20Test.table_prefix
diff --git a/python/phoenixdb/phoenixdb/tests/test_errors.py b/python/phoenixdb/phoenixdb/tests/test_errors.py
index 191ccb1..ee0a237 100644
--- a/python/phoenixdb/phoenixdb/tests/test_errors.py
+++ b/python/phoenixdb/phoenixdb/tests/test_errors.py
@@ -30,7 +30,7 @@ class ProgrammingErrorTest(DatabaseTestCase):
class IntegrityErrorTest(DatabaseTestCase):
def test_null_in_pk(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key)")
with self.conn.cursor() as cursor:
with self.assertRaises(self.conn.IntegrityError) as cm:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (NULL)")
@@ -42,7 +42,7 @@ class IntegrityErrorTest(DatabaseTestCase):
class DataErrorTest(DatabaseTestCase):
def test_number_outside_of_range(self):
- self.createTable("phoenixdb_test_tbl1", "id tinyint primary key")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id tinyint primary key)")
with self.conn.cursor() as cursor:
with self.assertRaises(self.conn.DataError) as cm:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (10000)")
@@ -51,7 +51,7 @@ class DataErrorTest(DatabaseTestCase):
self.assertEqual("22005", cm.exception.sqlstate)
def test_division_by_zero(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key)")
with self.conn.cursor() as cursor:
with self.assertRaises(self.conn.DataError) as cm:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2/0)")
diff --git a/python/phoenixdb/phoenixdb/tests/test_types.py b/python/phoenixdb/phoenixdb/tests/test_types.py
index 2cef0f2..598b5b8 100644
--- a/python/phoenixdb/phoenixdb/tests/test_types.py
+++ b/python/phoenixdb/phoenixdb/tests/test_types.py
@@ -24,7 +24,7 @@ from phoenixdb.tests import DatabaseTestCase
class TypesTest(DatabaseTestCase):
def checkIntType(self, type_name, min_value, max_value):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val {}".format(type_name))
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val {})".format(type_name, table="{table}"))
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, 1)")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, NULL)")
@@ -73,7 +73,7 @@ class TypesTest(DatabaseTestCase):
self.checkIntType("unsigned_smallint", 0, 32767)
def checkFloatType(self, type_name, min_value, max_value):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val {}".format(type_name))
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val {})".format(type_name, table="{table}"))
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, 1)")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, NULL)")
@@ -105,7 +105,7 @@ class TypesTest(DatabaseTestCase):
self.checkFloatType("unsigned_double", 0, 1.7976931348623158E+308)
def test_decimal(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val decimal(8,3)")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val decimal(8,3))")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, 33333.333)")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, NULL)")
@@ -129,7 +129,7 @@ class TypesTest(DatabaseTestCase):
"UPSERT INTO phoenixdb_test_tbl1 VALUES (101, ?)", [Decimal('123456.789')])
def test_boolean(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val boolean")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val boolean)")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, TRUE)")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, FALSE)")
@@ -141,8 +141,9 @@ class TypesTest(DatabaseTestCase):
self.assertEqual(cursor.description[1].type_code, phoenixdb.BOOLEAN)
self.assertEqual(cursor.fetchall(), [[1, True], [2, False], [3, None], [4, True], [5, False], [6, None]])
+ @unittest.skip("https://issues.apache.org/jira/browse/PHOENIX-4664")
def test_time(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val time")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val time)")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, '1970-01-01 12:01:02')")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, NULL)")
@@ -160,7 +161,7 @@ class TypesTest(DatabaseTestCase):
@unittest.skip("https://issues.apache.org/jira/browse/CALCITE-797")
def test_time_full(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val time")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val time)")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, '2015-07-12 13:01:02.123')")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, ?)", [datetime.datetime(2015, 7, 12, 13, 1, 2, 123000)])
@@ -170,8 +171,9 @@ class TypesTest(DatabaseTestCase):
[2, datetime.datetime(2015, 7, 12, 13, 1, 2, 123000)],
])
+ @unittest.skip("https://issues.apache.org/jira/browse/PHOENIX-4664")
def test_date(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val date")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val date)")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, '2015-07-12 00:00:00')")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (3, ?)", [phoenixdb.Date(2015, 7, 12)])
@@ -185,7 +187,7 @@ class TypesTest(DatabaseTestCase):
@unittest.skip("https://issues.apache.org/jira/browse/CALCITE-798")
def test_date_full(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val date")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val date)")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, '2015-07-12 13:01:02.123')")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, ?)", [datetime.datetime(2015, 7, 12, 13, 1, 2, 123000)])
@@ -196,7 +198,7 @@ class TypesTest(DatabaseTestCase):
])
def test_date_null(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val date")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val date)")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, NULL)")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, ?)", [None])
@@ -206,8 +208,9 @@ class TypesTest(DatabaseTestCase):
[2, None],
])
+ @unittest.skip("https://issues.apache.org/jira/browse/PHOENIX-4664")
def test_timestamp(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val timestamp")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val timestamp)")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, '2015-07-12 13:01:02.123')")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, NULL)")
@@ -225,7 +228,7 @@ class TypesTest(DatabaseTestCase):
@unittest.skip("https://issues.apache.org/jira/browse/CALCITE-796")
def test_timestamp_full(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val timestamp")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val timestamp)")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, '2015-07-12 13:01:02.123456789')")
cursor.execute("SELECT id, val FROM phoenixdb_test_tbl1 ORDER BY id")
@@ -234,7 +237,7 @@ class TypesTest(DatabaseTestCase):
])
def test_varchar(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val varchar")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val varchar)")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, 'abc')")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, NULL)")
@@ -246,7 +249,7 @@ class TypesTest(DatabaseTestCase):
self.assertEqual(cursor.fetchall(), [[1, 'abc'], [2, None], [3, 'abc'], [4, None], [5, None], [6, None]])
def test_varchar_very_long(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val varchar")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val varchar)")
with self.conn.cursor() as cursor:
value = '1234567890' * 1000
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, ?)", [value])
@@ -254,7 +257,7 @@ class TypesTest(DatabaseTestCase):
self.assertEqual(cursor.fetchall(), [[1, value]])
def test_varchar_limited(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val varchar(2)")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val varchar(2))")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, 'ab')")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, NULL)")
@@ -267,7 +270,7 @@ class TypesTest(DatabaseTestCase):
self.assertRaises(self.conn.DataError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, 'abc')")
def test_char_null(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val char(2)")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val char(2))")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, NULL)")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (4, ?)", [None])
@@ -278,7 +281,7 @@ class TypesTest(DatabaseTestCase):
self.assertRaises(self.conn.DataError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, 'abc')")
def test_char(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val char(2)")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val char(2))")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, 'ab')")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, ?)", ['ab'])
@@ -289,7 +292,7 @@ class TypesTest(DatabaseTestCase):
self.assertRaises(self.conn.DataError, cursor.execute, "UPSERT INTO phoenixdb_test_tbl1 VALUES (100, 'abc')")
def test_binary(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val binary(2)")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val binary(2))")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, 'ab')")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, ?)", [phoenixdb.Binary(b'ab')])
@@ -304,7 +307,7 @@ class TypesTest(DatabaseTestCase):
])
def test_binary_all_bytes(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val binary(256)")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val binary(256))")
with self.conn.cursor() as cursor:
if sys.version_info[0] < 3:
value = ''.join(map(chr, range(256)))
@@ -314,14 +317,32 @@ class TypesTest(DatabaseTestCase):
cursor.execute("SELECT id, val FROM phoenixdb_test_tbl1 ORDER BY id")
self.assertEqual(cursor.fetchall(), [[1, value]])
- @unittest.skip("https://issues.apache.org/jira/browse/CALCITE-1050 https://issues.apache.org/jira/browse/PHOENIX-2585")
def test_array(self):
- self.createTable("phoenixdb_test_tbl1", "id integer primary key, val integer[]")
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val integer[])")
with self.conn.cursor() as cursor:
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, ARRAY[1, 2])")
cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, ?)", [[2, 3]])
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (3, ?)", [[4]])
cursor.execute("SELECT id, val FROM phoenixdb_test_tbl1 ORDER BY id")
self.assertEqual(cursor.fetchall(), [
[1, [1, 2]],
[2, [2, 3]],
+ [3, [4]],
+ ])
+
+ def test_array_boolean(self):
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val boolean[])")
+ with self.conn.cursor() as cursor:
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, ARRAY[TRUE, TRUE, FALSE])")
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, NULL)")
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (3, ?)", ((1, 0, 1),))
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (4, ?)", [[True, True, True]])
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (5, ?)", [[]])
+ cursor.execute("SELECT id, val FROM phoenixdb_test_tbl1 ORDER BY id")
+ self.assertEqual(cursor.fetchall(), [
+ [1, [True, True, False]],
+ [2, None],
+ [3, [True, False, True]],
+ [4, [True, True, True]],
+ [5, None]
])
diff --git a/python/phoenixdb/phoenixdb/types.py b/python/phoenixdb/phoenixdb/types.py
index f41355a..20f8572 100644
--- a/python/phoenixdb/phoenixdb/types.py
+++ b/python/phoenixdb/phoenixdb/types.py
@@ -18,11 +18,12 @@ import time
import datetime
from decimal import Decimal
from phoenixdb.avatica.proto import common_pb2
+from builtins import staticmethod
__all__ = [
'Date', 'Time', 'Timestamp', 'DateFromTicks', 'TimeFromTicks', 'TimestampFromTicks',
'Binary', 'STRING', 'BINARY', 'NUMBER', 'DATETIME', 'ROWID', 'BOOLEAN',
- 'JAVA_CLASSES', 'JAVA_CLASSES_MAP', 'TypeHelper',
+ 'TypeHelper',
]
@@ -89,7 +90,7 @@ def datetime_to_java_sql_timestamp(d):
td = d - datetime.datetime(1970, 1, 1)
return td.microseconds // 1000 + (td.seconds + td.days * 24 * 3600) * 1000
-
+#FIXME This doesn't seem to be used anywhere in the code
class ColumnType(object):
def __init__(self, eq_types):
@@ -129,62 +130,150 @@ ROWID = ColumnType([])
BOOLEAN = ColumnType(['BOOLEAN'])
"""Type object that can be used to describe boolean columns. This is a phoenixdb-specific extension."""
-
-# XXX ARRAY
-
if sys.version_info[0] < 3:
_long = long # noqa: F821
else:
_long = int
-JAVA_CLASSES = {
+FIELD_MAP = {
'bool_value': [
- ('java.lang.Boolean', common_pb2.BOOLEAN, None, None),
+ (common_pb2.BOOLEAN, None, None),
+ (common_pb2.PRIMITIVE_BOOLEAN, None, None),
],
'string_value': [
- ('java.lang.Character', common_pb2.CHARACTER, None, None),
- ('java.lang.String', common_pb2.STRING, None, None),
- ('java.math.BigDecimal', common_pb2.BIG_DECIMAL, str, Decimal),
+ (common_pb2.CHARACTER, None, None),
+ (common_pb2.PRIMITIVE_CHAR, None, None),
+ (common_pb2.STRING, None, None),
+ (common_pb2.BIG_DECIMAL, str, Decimal),
],
'number_value': [
- ('java.lang.Integer', common_pb2.INTEGER, None, int),
- ('java.lang.Short', common_pb2.SHORT, None, int),
- ('java.lang.Long', common_pb2.LONG, None, _long),
- ('java.lang.Byte', common_pb2.BYTE, None, int),
- ('java.sql.Time', common_pb2.JAVA_SQL_TIME, time_to_java_sql_time, time_from_java_sql_time),
- ('java.sql.Date', common_pb2.JAVA_SQL_DATE, date_to_java_sql_date, date_from_java_sql_date),
- ('java.sql.Timestamp', common_pb2.JAVA_SQL_TIMESTAMP, datetime_to_java_sql_timestamp, datetime_from_java_sql_timestamp),
+ (common_pb2.INTEGER, None, int),
+ (common_pb2.PRIMITIVE_INT, None, int),
+ (common_pb2.SHORT, None, int),
+ (common_pb2.PRIMITIVE_SHORT, None, int),
+ (common_pb2.LONG, None, _long),
+ (common_pb2.PRIMITIVE_LONG, None, _long),
+ (common_pb2.BYTE, None, int),
+ (common_pb2.JAVA_SQL_TIME, time_to_java_sql_time, time_from_java_sql_time),
+ (common_pb2.JAVA_SQL_DATE, date_to_java_sql_date, date_from_java_sql_date),
+ (common_pb2.JAVA_SQL_TIMESTAMP, datetime_to_java_sql_timestamp, datetime_from_java_sql_timestamp),
],
'bytes_value': [
- ('[B', common_pb2.BYTE_STRING, Binary, None),
+ (common_pb2.BYTE_STRING, Binary, None),
],
'double_value': [
- # if common_pb2.FLOAT is used, incorrect values are sent
- ('java.lang.Float', common_pb2.DOUBLE, float, float),
- ('java.lang.Double', common_pb2.DOUBLE, float, float),
+ (common_pb2.DOUBLE, float, float),
+ (common_pb2.PRIMITIVE_DOUBLE, float, float)
]
}
-"""Groups of Java classes."""
+"""The master map that describes how to handle types, keyed by TypedData field"""
-JAVA_CLASSES_MAP = dict((v[0], (k, v[1], v[2], v[3])) for k in JAVA_CLASSES for v in JAVA_CLASSES[k])
-"""Flips the available types to allow for faster lookup by Java class.
+REP_MAP = dict((v[0], (k, v[0], v[1], v[2])) for k in FIELD_MAP for v in FIELD_MAP[k])
+"""Flips the available types to allow for faster lookup by protobuf Rep
This mapping should be structured as:
{
- 'java.math.BigDecimal': ('string_value', common_pb2.BIG_DECIMAL, str, Decimal),),
+ 'common_pb2.BIG_DECIMAL': ('string_value', common_pb2.BIG_DECIMAL, str, Decimal),),
...
- '<java class>': (<field_name>, <Rep enum>, <mutate_to function>, <cast_from function>),
+ '<Rep enum>': (<field_name>, <mutate_to function>, <cast_from function>),
}
"""
-
+JDBC_TO_REP = dict([
+ # These are the standard types that are used in Phoenix
+ (-6, common_pb2.BYTE), #TINYINT
+ (5, common_pb2.SHORT), #SMALLINT
+ (4, common_pb2.INTEGER), #INTEGER
+ (-5, common_pb2.LONG), #BIGINT
+ (6, common_pb2.DOUBLE), #FLOAT
+ (8, common_pb2.DOUBLE), #DOUBLE
+ (2, common_pb2.BIG_DECIMAL), #NUMERIC
+ (1, common_pb2.STRING), #CHAR
+ (91, common_pb2.JAVA_SQL_DATE), #DATE
+ (93, common_pb2.JAVA_SQL_TIMESTAMP), #TIME
+ (-2, common_pb2.BYTE_STRING), #BINARY
+ (-3, common_pb2.BYTE_STRING), #VARBINARY
+ (16, common_pb2.BOOLEAN), #BOOLEAN
+ # These are the Non-standard types defined by Phoenix
+ (19, common_pb2.JAVA_SQL_DATE), #UNSIGNED_DATE
+ (15, common_pb2.DOUBLE), #UNSIGNED_DOUBLE
+ (14, common_pb2.DOUBLE), #UNSIGNED_FLOAT
+ (9, common_pb2.INTEGER), #UNSIGNED_INT
+ (10, common_pb2.LONG), #UNSIGNED_LONG
+ (13, common_pb2.SHORT), #UNSIGNED_SMALLINT
+ (20, common_pb2.JAVA_SQL_TIMESTAMP), #UNSIGNED_TIMESTAMP
+ (11, common_pb2.BYTE), #UNSIGNED_TINYINT
+ # The following are not used by Phoenix, but some of these are used by Avaticafor
+ # parameter types
+ (-7, common_pb2.BOOLEAN), #BIT
+ (7, common_pb2.DOUBLE), #REAL
+ (3, common_pb2.BIG_DECIMAL), #DECIMAL
+ (12, common_pb2.STRING), #VARCHAR
+ (-1, common_pb2.STRING), #LONGVARCHAR
+ (-4, common_pb2.BYTE_STRING), #LONGVARBINARY
+ (2004, common_pb2.BYTE_STRING), #BLOB
+ (2005, common_pb2.STRING), #CLOB
+ (-15, common_pb2.STRING), #NCHAR
+ (-9, common_pb2.STRING), #NVARCHAR
+ (-16, common_pb2.STRING), #LONGNVARCHAR
+ (2011, common_pb2.STRING), #NCLOB
+ (2009, common_pb2.STRING), #SQLXML
+ # These are defined by JDBC, but cannot be mapped
+ #NULL
+ #OTHER
+ #JAVA_OBJECT
+ #DISTINCT
+ #STRUCT
+ #ARRAY 2003 - We are handling this as a special case
+ #REF
+ #DATALINK
+ #ROWID
+ #REF_CURSOR
+ #TIME WITH TIMEZONE
+ #TIMESTAMP WITH TIMEZONE
+
+ ])
+"""Maps the JDBC Type IDs to Protobuf Reps """
+
+JDBC_MAP = {}
+for k,v in JDBC_TO_REP.items():
+ JDBC_MAP[k & 0xffffffff] = REP_MAP[v]
+"""Flips the available types to allow for faster lookup by JDBC type ID
+
+It has the same format as REP_MAP, but is keyed by JDBC type ID
+"""
class TypeHelper(object):
+
@staticmethod
- def from_class(klass):
- """Retrieves a Rep and functions to cast to/from based on the Java class.
+ def from_param(param):
+ """Retrieves a field name and functions to cast to/from based on an AvaticaParameter object
- :param klass:
- The string of the Java class for the column or parameter.
+ :param param:
+ Protobuf AvaticaParameter object
+
+ :returns: tuple ``(field_name, rep, mutate_to, cast_from, is_array)``
+ WHERE
+ ``field_name`` is the attribute in ``common_pb2.TypedValue``
+ ``rep`` is the common_pb2.Rep enum
+ ``mutate_to`` is the function to cast values into Phoenix values, if any
+ ``cast_from`` is the function to cast from the Phoenix value to the Python value, if any
+ ``is_array`` the param expects an array instead of scalar
+
+ :raises:
+ NotImplementedError
+ """
+ jdbc_code = param.parameter_type
+ if jdbc_code > 2900 and jdbc_code < 3100:
+ return TypeHelper._from_jdbc(jdbc_code-3000) + (True,)
+ else:
+ return TypeHelper._from_jdbc(jdbc_code) + (False,)
+
+ @staticmethod
+ def from_column(column):
+ """Retrieves a field name and functions to cast to/from based on a TypedValue object
+
+ :param column:
+ Protobuf TypedValue object
:returns: tuple ``(field_name, rep, mutate_to, cast_from)``
WHERE
@@ -196,7 +285,15 @@ class TypeHelper(object):
:raises:
NotImplementedError
"""
- if klass not in JAVA_CLASSES_MAP:
- raise NotImplementedError('type {} is not supported'.format(klass))
+ if column.type.id == 2003:
+ return TypeHelper._from_jdbc(column.type.component.id)
+ else:
+ return TypeHelper._from_jdbc(column.type.id)
+
+ @staticmethod
+ def _from_jdbc(jdbc_code):
+ if jdbc_code not in JDBC_MAP:
+ #This should not happen. It's either a bug, or Avatica has added new types
+ raise NotImplementedError('JDBC TYPE CODE {} is not supported'.format(jdbc_code))
- return JAVA_CLASSES_MAP[klass]
+ return JDBC_MAP[jdbc_code]
diff --git a/queryserver-it/src/it/bin/test_phoenixdb.py b/queryserver-it/src/it/bin/test_phoenixdb.py
index 0d5d0c6..2e74b81 100644
--- a/queryserver-it/src/it/bin/test_phoenixdb.py
+++ b/queryserver-it/src/it/bin/test_phoenixdb.py
@@ -21,11 +21,11 @@
import phoenixdb
import phoenixdb.cursor
import sys
+import os
if __name__ == '__main__':
- pqs_port = sys.argv[1]
- database_url = 'http://localhost:' + str(pqs_port) + '/'
+ database_url = os.environ.get('PHOENIXDB_TEST_DB_URL')
print("CREATING PQS CONNECTION")
conn = phoenixdb.connect(database_url, autocommit=True, auth="SPNEGO")
diff --git a/queryserver-it/src/it/bin/test_phoenixdb.sh b/queryserver-it/src/it/bin/test_phoenixdb.sh
index 7309dbe..ff4893e 100755
--- a/queryserver-it/src/it/bin/test_phoenixdb.sh
+++ b/queryserver-it/src/it/bin/test_phoenixdb.sh
@@ -43,7 +43,8 @@ PRINC=$2
KEYTAB_LOC=$3
KRB5_CFG_FILE=$4
PQS_PORT=$5
-PYTHON_SCRIPT=$6
+
+shift 5
PY_ENV_PATH=$( mktemp -d )
@@ -59,7 +60,6 @@ popd
set -u
echo "INSTALLING COMPONENTS"
-pip install -e file:///${LOCAL_PY}/requests-kerberos
pip install -e file:///${LOCAL_PY}/phoenixdb
export KRB5_CONFIG=$KRB5_CFG_FILE
@@ -75,5 +75,22 @@ unset https_proxy
echo "Working Directory is ${PWD}"
-echo "RUN PYTHON TEST on port $PQS_PORT"
-python $PYTHON_SCRIPT $PQS_PORT
+cat > target/krb_setup.sh << EOF
+#!/bin/sh
+export KRB5_CONFIG=$KRB5_CFG_FILE
+export KRB5_TRACE=/dev/stdout
+kinit -kt $KEYTAB_LOC $PRINC
+klist
+
+export PHOENIXDB_TEST_DB_URL="http://localhost:$PQS_PORT"
+export PHOENIXDB_TEST_DB_AUTHENTICATION="SPNEGO"
+
+EOF
+
+
+export PHOENIXDB_TEST_DB_URL="http://localhost:$PQS_PORT"
+export PHOENIXDB_TEST_DB_AUTHENTICATION="SPNEGO"
+
+echo "RUN test: $@"
+"$@" > >(tee -a target/python-stdout.log) 2> >(tee -a target/python-stderr.log >&2)
+
diff --git a/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
index 1e2fb93..76c38ee 100644
--- a/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
+++ b/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
@@ -24,7 +24,6 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -35,6 +34,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.Statement;
+import java.time.LocalDateTime;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
@@ -44,6 +44,7 @@ import org.apache.phoenix.queryserver.QueryServerProperties;
import org.apache.phoenix.util.ThinClientUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
@@ -343,4 +344,46 @@ public class QueryServerBasicsIT extends BaseHBaseManagedTimeIT {
rs = select.executeQuery();
assertFalse(rs.next());
}
+
+ @Ignore("PHOENIX-4664")
+ @Test
+ //This is the java reproducer for PHOENIX-4664
+ //This test only works as intended when the system timezone is NOT GMT
+ public void testTimezoneLess() throws Exception {
+
+ try (Connection conn = DriverManager.getConnection(CONN_STRING);
+ Statement stmt = conn.createStatement();
+ ) {
+ final String tableName = generateUniqueName();
+
+ stmt.execute(
+ "CREATE TABLE " + tableName + " (k VARCHAR NOT NULL PRIMARY KEY, i TIMESTAMP)");
+ conn.commit();
+
+ LocalDateTime now = LocalDateTime.now();
+ try(PreparedStatement upsert = conn.prepareStatement(
+ "UPSERT INTO " + tableName + " VALUES (?, ?)")
+ ) {
+ upsert.setString(1, "1");
+ upsert.setTimestamp(2, java.sql.Timestamp.valueOf(now));
+ upsert.executeUpdate();
+ conn.commit();
+ ResultSet rs = stmt.executeQuery("select * from " + tableName);
+ assertTrue(rs.next());
+ LocalDateTime fromDB = rs.getTimestamp("i").toLocalDateTime();
+ assertTrue("Timestamps do not match. inserted:" + now.toString() + "returned:" + fromDB.toString(), fromDB.compareTo(now) == 0);
+ }
+ }
+ }
+
+ @Ignore
+ @Test
+ //Quick and dirty way start up a local Phoenix+PQS instance for testing against
+ public void startLocalPQS() throws Exception {
+ System.out.println("CONN STRING:" + CONN_STRING);
+ System.out.println("Tests suspended!!!");
+ System.out.println("Kill maven run to stop server");
+ System.out.flush();
+ Thread.sleep(24*60*60*1000);
+ }
}
diff --git a/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
index a9a5d8f..40c5e03 100644
--- a/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
+++ b/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
@@ -25,6 +25,9 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.Reader;
+import java.io.Writer;
import java.lang.reflect.Field;
import java.nio.file.Paths;
import java.security.PrivilegedAction;
@@ -37,11 +40,13 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.client.TestHCM.SleepAndFailFirstTime;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
@@ -74,7 +79,6 @@ import com.google.common.collect.Maps;
* files in phoenix-queryserver/src/it/bin.
*/
@Category(NeedsOwnMiniClusterTest.class)
-@Ignore("Failing since QueryServer moved to its own repository")
public class SecureQueryServerPhoenixDBIT {
private static enum Kdc {
MIT,
@@ -324,6 +328,30 @@ public class SecureQueryServerPhoenixDBIT {
@Test
public void testBasicReadWrite() throws Exception {
+ File file = new File(".");
+ runShellScript("python", Paths.get(file.getAbsolutePath(), "src", "it", "bin", "test_phoenixdb.py").toString());
+ }
+
+ //This takes about 300s, so we are not running this by default
+ @Ignore
+ @Test
+ public void testFullSuite() throws Exception {
+ File file = new File(".");
+ runShellScript("python", "-m", "unittest", "discover", "-v", "-s", Paths.get(file.getAbsolutePath(), "..","python", "phoenixdb").toString());
+ }
+
+ @Ignore
+ @Test
+ //Quick and dirty way start up a local Phoenix+PQS instance for testing against
+ //When started, this will write a setup script into target/krb_setup.sh
+ //If you source that file, it should log you into kerberos with the test principal,
+ //and set the environment so that the python unit tests can run against this instance.
+ //You'll need to kill the test manually
+ public void startLocalPQS() throws Exception {
+ runShellScript("sleep", "86400");
+ }
+
+ public void runShellScript(String ... testCli) throws Exception {
final Entry<String,File> user1 = getUser(1);
String currentDirectory;
File file = new File(".");
@@ -400,20 +428,20 @@ public class SecureQueryServerPhoenixDBIT {
}
cmdList.add(Integer.toString(PQS_PORT));
- cmdList.add(Paths.get(currentDirectory, "src", "it", "bin", "test_phoenixdb.py").toString());
+ cmdList.addAll(Arrays.asList(testCli));
+ //This will intersperse that script's output to the maven output, but that's better than
+ //getting stuck on a full buffer
Process runPythonProcess = new ProcessBuilder(cmdList).start();
- BufferedReader processOutput = new BufferedReader(new InputStreamReader(runPythonProcess.getInputStream()));
- BufferedReader processError = new BufferedReader(new InputStreamReader(runPythonProcess.getErrorStream()));
- int exitCode = runPythonProcess.waitFor();
+ BufferedReader processOutput = new BufferedReader(
+ new InputStreamReader(runPythonProcess.getInputStream()));
+ new Thread(new StreamCopy(processOutput, new PrintWriter(System.out))).start();
- // dump stdout and stderr
- while (processOutput.ready()) {
- LOG.info(processOutput.readLine());
- }
- while (processError.ready()) {
- LOG.error(processError.readLine());
- }
+ BufferedReader processError = new BufferedReader(
+ new InputStreamReader(runPythonProcess.getErrorStream()));
+ new Thread(new StreamCopy(processError, new PrintWriter(System.err))).start();
+
+ int exitCode = runPythonProcess.waitFor();
// Not managed by miniKDC so we have to clean up
if (krb5ConfFile != null)
@@ -422,9 +450,25 @@ public class SecureQueryServerPhoenixDBIT {
assertEquals("Subprocess exited with errors", 0, exitCode);
}
- byte[] copyBytes(byte[] src, int offset, int length) {
- byte[] dest = new byte[length];
- System.arraycopy(src, offset, dest, 0, length);
- return dest;
+ private class StreamCopy implements Runnable {
+ private Reader reader;
+ private Writer writer;
+
+ public StreamCopy(Reader reader, Writer writer) {
+ this.reader = reader;
+ this.writer = writer;
+ }
+
+ @Override
+ public void run() {
+ try {
+ while(true) {
+ IOUtils.copy(reader, writer);
+ Thread.sleep(10);
+ }
+ } catch (Exception e) {
+ //Just exit
+ }
+ }
}
}
diff --git a/queryserver/src/it/bin/test_phoenixdb.py b/queryserver/src/it/bin/test_phoenixdb.py
deleted file mode 100644
index 0d5d0c6..0000000
--- a/queryserver/src/it/bin/test_phoenixdb.py
+++ /dev/null
@@ -1,39 +0,0 @@
-############################################################################
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-############################################################################
-
-import phoenixdb
-import phoenixdb.cursor
-import sys
-
-
-if __name__ == '__main__':
- pqs_port = sys.argv[1]
- database_url = 'http://localhost:' + str(pqs_port) + '/'
-
- print("CREATING PQS CONNECTION")
- conn = phoenixdb.connect(database_url, autocommit=True, auth="SPNEGO")
- cursor = conn.cursor()
-
- cursor.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, username VARCHAR)")
- cursor.execute("UPSERT INTO users VALUES (?, ?)", (1, 'admin'))
- cursor.execute("UPSERT INTO users VALUES (?, ?)", (2, 'user'))
- cursor.execute("SELECT * FROM users")
- print("RESULTS")
- print(cursor.fetchall())
diff --git a/queryserver/src/it/bin/test_phoenixdb.sh b/queryserver/src/it/bin/test_phoenixdb.sh
deleted file mode 100755
index ae9cbd3..0000000
--- a/queryserver/src/it/bin/test_phoenixdb.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env bash
-#
-############################################################################
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-############################################################################
-
-set -u
-set -x
-set -e
-
-function cleanup {
- # Capture last command status
- RCODE=$?
- set +e
- set +u
- kdestroy
- rm -rf $PY_ENV_PATH
- exit $RCODE
-}
-
-trap cleanup EXIT
-
-echo "LAUNCHING SCRIPT"
-
-LOCAL_PY=$1
-PRINC=$2
-KEYTAB_LOC=$3
-KRB5_CFG_FILE=$4
-PQS_PORT=$5
-PYTHON_SCRIPT=$6
-
-PY_ENV_PATH=$( mktemp -d )
-
-virtualenv $PY_ENV_PATH
-
-pushd ${PY_ENV_PATH}/bin
-
-# conda activate does stuff with unbound variables :(
-set +u
-. activate ""
-
-popd
-
-set -u
-echo "INSTALLING COMPONENTS"
-pip install -e file:///${LOCAL_PY}/phoenixdb
-
-export KRB5_CONFIG=$KRB5_CFG_FILE
-cat $KRB5_CONFIG
-export KRB5_TRACE=/dev/stdout
-
-echo "RUNNING KINIT"
-kinit -kt $KEYTAB_LOC $PRINC
-klist
-
-unset http_proxy
-unset https_proxy
-
-echo "Working Directory is ${PWD}"
-
-echo "RUN PYTHON TEST on port $PQS_PORT"
-python $PYTHON_SCRIPT $PQS_PORT