You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by st...@apache.org on 2020/05/04 05:50:50 UTC
[phoenix-queryserver] branch master updated: PHOENIX-5879 Fix
recently introduced python 2.7 incompatibilities and flake8 warning
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix-queryserver.git
The following commit(s) were added to refs/heads/master by this push:
new 1e78243 PHOENIX-5879 Fix recently introduced python 2.7 incompatibilities and flake8 warning
1e78243 is described below
commit 1e78243c4611b4b0ac3f62f90685acf3959afbef
Author: Istvan Toth <st...@apache.org>
AuthorDate: Wed Apr 29 19:02:33 2020 +0200
PHOENIX-5879 Fix recently introduced python 2.7 incompatibilities and flake8 warning
* fix time type bug introduced in previous patch
* fix python27 incompatibilities
* downgrade dependencies to make sure setup works on 2.7
* fix flake8 warnings
additional improvements:
* pare down dependencies
* dockerize tox tests
* update docs
* expose ways to start a local test PQS server from maven
* remove shell example, as it is python2 only
* update author/contact info in setup.py
* update docs
* PHOENIX-5854 Synchronize the python client version to the queryserver version
* set python 3 supported versions to 3.4-3.8 in docs, setup.py and tests
* add simplified date/time/timestamp test
* add unicode test
---
python/README.md | 2 +-
python/phoenixdb/Dockerfile | 9 ++
python/phoenixdb/Dockerfile-pqs | 9 ++
python/phoenixdb/NEWS.rst | 18 ++--
python/phoenixdb/README.rst | 74 ++++++--------
python/phoenixdb/docker-compose.yml | 21 ----
python/phoenixdb/examples/shell.py | 33 -------
python/phoenixdb/phoenixdb/__init__.py | 12 ++-
python/phoenixdb/phoenixdb/avatica/client.py | 16 +--
python/phoenixdb/phoenixdb/connection.py | 5 +-
python/phoenixdb/phoenixdb/cursor.py | 9 +-
python/phoenixdb/phoenixdb/tests/__init__.py | 24 +++--
python/phoenixdb/phoenixdb/tests/test_avatica.py | 1 +
python/phoenixdb/phoenixdb/tests/test_db.py | 8 +-
python/phoenixdb/phoenixdb/tests/test_dbapi20.py | 5 +-
python/phoenixdb/phoenixdb/tests/test_types.py | 34 ++++++-
python/phoenixdb/phoenixdb/types.py | 107 +++++++++++----------
python/phoenixdb/requirements.txt | 4 -
python/phoenixdb/setup.py | 22 ++++-
python/phoenixdb/tox.ini | 7 +-
.../phoenix/end2end/QueryServerBasicsIT.java | 7 +-
.../end2end/SecureQueryServerPhoenixDBIT.java | 4 +-
22 files changed, 223 insertions(+), 208 deletions(-)
diff --git a/python/README.md b/python/README.md
index 45b4fc7..02eb3b9 100644
--- a/python/README.md
+++ b/python/README.md
@@ -23,7 +23,7 @@ This driver implements the Python DB 2.0 API for database drivers as described b
This driver is implemented using the Phoenix Query Server (PQS) and the [Apache Calcite
Avatica](https://calcite.apache.org/avatica) project.
-This driver should be compatible with Python 2.7 and Python 3.3+ and support both unauthenticated access and
+This driver should be compatible with Python 2.7 and Python 3.4+ and support both unauthenticated access and
authenticated access via SPNEGO to PQS.
## Usage
diff --git a/python/phoenixdb/Dockerfile b/python/phoenixdb/Dockerfile
new file mode 100644
index 0000000..7edaced
--- /dev/null
+++ b/python/phoenixdb/Dockerfile
@@ -0,0 +1,9 @@
+from themattrix/tox-base
+
+RUN apt-get update && apt-get install -y krb5-user libkrb5-dev
+
+ENV PHOENIXDB_TEST_DB_URL=http://host.docker.internal:8765
+ENV PHOENIXDB_TEST_DB_TRUSTSTORE $PHOENIXDB_TEST_DB_TRUSTSTORE
+ENV PHOENIXDB_TEST_DB_AUTHENTICATION $PHOENIXDB_TEST_DB_AUTHENTICATION
+ENV PHOENIXDB_TEST_DB_AVATICA_USER $PHOENIXDB_TEST_DB_AVATICA_USER
+ENV PHOENIXDB_TEST_DB_AVATICA_PASSWORD $PHOENIXDB_TEST_DB_AVATICA_PASSWORD
\ No newline at end of file
diff --git a/python/phoenixdb/Dockerfile-pqs b/python/phoenixdb/Dockerfile-pqs
new file mode 100644
index 0000000..7bfb7ca
--- /dev/null
+++ b/python/phoenixdb/Dockerfile-pqs
@@ -0,0 +1,9 @@
+from maven:3-jdk-8
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq krb5-user libkrb5-dev
+
+EXPOSE 8765
+
+# copy all the files to the container
+
+CMD mvn clean verify -am -pl queryserver-it -Dtest=foo -Dit.test=QueryServerBasicsIT#startLocalPQS -Ddo.not.randomize.pqs.port=true -Dstart.unsecure.pqs=true
\ No newline at end of file
diff --git a/python/phoenixdb/NEWS.rst b/python/phoenixdb/NEWS.rst
index 3d4cff1..089e7ed 100644
--- a/python/phoenixdb/NEWS.rst
+++ b/python/phoenixdb/NEWS.rst
@@ -4,14 +4,16 @@ Changelog
Unreleased
----------
- Replaced bundled requests_kerberos with request_gssapi library
-- Refactor authentication code
-- Support for specifying server certificate
-- Support for BASIC and DIGEST authentication
-- Fix HTTP error parsing
-- Add transaction support
-- Add list support
-- Rewrite type handling
-- Refactor test suite
+- Refactored authentication code
+- Added support for specifying server certificate
+- Added support for BASIC and DIGEST authentication
+- Fixed HTTP error parsing
+- Added transaction support
+- Added list support
+- Rewritten type handling
+- Refactored test suite
+- Removed shell example, as it was python2 only
+- Updated documentation
Version 0.7
-----------
diff --git a/python/phoenixdb/README.rst b/python/phoenixdb/README.rst
index 8388517..8f321af 100644
--- a/python/phoenixdb/README.rst
+++ b/python/phoenixdb/README.rst
@@ -1,14 +1,6 @@
Phoenix database adapter for Python
===================================
-.. image:: https://code.oxygene.sk/lukas/python-phoenixdb/badges/master/pipeline.svg
- :target: https://code.oxygene.sk/lukas/python-phoenixdb/commits/master
- :alt: Build Status
-
-.. image:: https://readthedocs.org/projects/python-phoenixdb/badge/?version=latest
- :target: http://python-phoenixdb.readthedocs.io/en/latest/?badge=latest
- :alt: Documentation Status
-
``phoenixdb`` is a Python library for accessing the
`Phoenix SQL database <http://phoenix.apache.org/>`_
using the
@@ -20,14 +12,13 @@ which should be familiar to most Python programmers.
Installation
------------
-The easiest way to install the library is using `pip <https://pip.pypa.io/en/stable/>`_::
-
- pip install phoenixdb
+The source code is part of the phoenix-queryserver source distribution.
+You can download it from <https://phoenix.apache.org/>, or get the latest development version
+from <https://github.com/apache/phoenix-queryserver>
-You can also download the source code from `here <https://phoenix.apache.org/download.html>`_,
-extract the archive and then install it manually::
+Extract the archive and then install it manually::
- cd /path/to/apache-phoenix-x.y.z/phoenix
+ cd /path/to/phoenix-queryserver-x.y.z/python/phoenixdb
python setup.py install
Usage
@@ -65,17 +56,12 @@ necessary requirements::
pip install -r requirements.txt
python setup.py develop
-To create or update the Avatica protobuf classes, change the tag in ``gen-protobuf.sh``
-and run the script.
-
-If you need a Phoenix query server for experimenting, you can get one running
-quickly using `Docker <https://www.docker.com/>`_::
-
- docker-compose up
-
-Or if you need an older version of Phoenix::
+You can start a Phoenix QueryServer instance on http://localhost:8765 for testing by running
+the following command in the phoenix-queryserver directory:
- PHOENIX_VERSION=4.9 docker-compose up
+ mvn clean verify -am -pl queryserver-it -Dtest=foo \
+ -Dit.test=QueryServerBasicsIT\#startLocalPQS \
+ -Ddo.not.randomize.pqs.port=true -Dstart.unsecure.pqs=true
If you want to use the library without installing the phoenixdb library, you can use
the `PYTHONPATH` environment variable to point to the library directly::
@@ -85,24 +71,7 @@ the `PYTHONPATH` environment variable to point to the library directly::
cd ~/my_project
PYTHONPATH=$PHOENIX_HOME/build/lib python my_app.py
-Interactive SQL shell
----------------------
-
-There is a Python-based interactive shell include in the examples folder, which can be
-used to connect to Phoenix and execute queries::
-
- ./examples/shell.py http://localhost:8765/
- db=> CREATE TABLE test (id INTEGER PRIMARY KEY, name VARCHAR);
- no rows affected (1.363 seconds)
- db=> UPSERT INTO test (id, name) VALUES (1, 'Lukas');
- 1 row affected (0.004 seconds)
- db=> SELECT * FROM test;
- +------+-------+
- | ID | NAME |
- +======+=======+
- | 1 | Lukas |
- +------+-------+
- 1 row selected (0.019 seconds)
+Don't forget to run flake8 on your changes.
Running the test suite
----------------------
@@ -114,6 +83,14 @@ working Phoenix database and set the ``PHOENIXDB_TEST_DB_URL`` environment varia
export PHOENIXDB_TEST_DB_URL='http://localhost:8765/'
nosetests
+If you use a secure PQS server, you can set the connection parameters via the following environment
+variables:
+
+- PHOENIXDB_TEST_DB_TRUSTSTORE
+- PHOENIXDB_TEST_DB_AUTHENTICATION
+- PHOENIXDB_TEST_DB_AVATICA_USER
+- PHOENIXDB_TEST_DB_AVATICA_PASSWORD
+
Similarly, tox can be used to run the test suite against multiple Python versions::
pyenv install 3.5.5
@@ -122,6 +99,19 @@ Similarly, tox can be used to run the test suite against multiple Python version
pyenv global 2.7.14 3.5.5 3.6.4
PHOENIXDB_TEST_DB_URL='http://localhost:8765' tox
+You can use tox and docker to run the tests on all supported python versions without installing the
+environments locally::
+
+ docker build -t toxtest .
+ docker run --rm -v `pwd`:/src toxtest
+
+You can also run the test suite from maven as part of the Java build by setting the
+run.full.python.testsuite property. You DO NOT need to set the PHOENIXDB_* enviroment variables,
+maven will set them up for you. The output of the test run will be saved in
+phoenix-queryserver/queryserver-it/target/python-stdout.log and python-stderr.log::
+
+ mvn clean verify -Drun.full.python.testsuite=true
+
Known issues
------------
diff --git a/python/phoenixdb/docker-compose.yml b/python/phoenixdb/docker-compose.yml
deleted file mode 100644
index bf398ec..0000000
--- a/python/phoenixdb/docker-compose.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- phoenix:
- image: docker.oxygene.sk/lukas/python-phoenixdb/phoenix:${PHOENIX_VERSION:-4.11}
- ports:
- - "127.0.0.1:8765:8765"
diff --git a/python/phoenixdb/examples/shell.py b/python/phoenixdb/examples/shell.py
deleted file mode 100755
index 820435e..0000000
--- a/python/phoenixdb/examples/shell.py
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-import argparse
-import sqlline
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--debug', '-d', action='store_true')
-parser.add_argument('url')
-args = parser.parse_args()
-
-if args.debug:
- logging.basicConfig(level=logging.DEBUG)
-
-with sqlline.SqlLine() as sqlline:
- sqlline.connect('phoenixdb', args.url)
- sqlline.connection.autocommit = True
- sqlline.run()
diff --git a/python/phoenixdb/phoenixdb/__init__.py b/python/phoenixdb/phoenixdb/__init__.py
index 0a444e0..a90be99 100644
--- a/python/phoenixdb/phoenixdb/__init__.py
+++ b/python/phoenixdb/phoenixdb/__init__.py
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from gssapi import mechs
from phoenixdb import errors, types
from phoenixdb.avatica import AvaticaClient
@@ -19,10 +20,11 @@ from phoenixdb.connection import Connection
from phoenixdb.errors import * # noqa: F401,F403
from phoenixdb.types import * # noqa: F401,F403
-from requests_gssapi import HTTPSPNEGOAuth, OPTIONAL
-from gssapi import mechs;
from requests.auth import HTTPBasicAuth, HTTPDigestAuth
+from requests_gssapi import HTTPSPNEGOAuth, OPTIONAL
+
+
__all__ = ['connect', 'apilevel', 'threadsafety', 'paramstyle'] + types.__all__ + errors.__all__
@@ -97,11 +99,11 @@ def connect(url, max_retries=None, auth=None, authentication=None, avatica_user=
spnego = mechs.Mechanism.from_sasl_name("SPNEGO")
if auth == "SPNEGO":
- #Special case for backwards compatibility
- auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech = spnego)
+ # Special case for backwards compatibility
+ auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech=spnego)
elif auth is None and authentication is not None:
if authentication == "SPNEGO":
- auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech = spnego, opportunistic_auth=True)
+ auth = HTTPSPNEGOAuth(mutual_authentication=OPTIONAL, mech=spnego, opportunistic_auth=True)
elif authentication == "BASIC" and avatica_user is not None and avatica_password is not None:
auth = HTTPBasicAuth(avatica_user, avatica_password)
elif authentication == "DIGEST" and avatica_user is not None and avatica_password is not None:
diff --git a/python/phoenixdb/phoenixdb/avatica/client.py b/python/phoenixdb/phoenixdb/avatica/client.py
index e392aee..daad12e 100644
--- a/python/phoenixdb/phoenixdb/avatica/client.py
+++ b/python/phoenixdb/phoenixdb/avatica/client.py
@@ -14,14 +14,14 @@
"""Implementation of the JSON-over-HTTP RPC protocol used by Avatica."""
-import re
-import socket
-import pprint
-import math
import logging
+import math
+import pprint
+import re
import time
+
from phoenixdb import errors
-from phoenixdb.avatica.proto import requests_pb2, common_pb2, responses_pb2
+from phoenixdb.avatica.proto import common_pb2, requests_pb2, responses_pb2
import requests
@@ -165,13 +165,13 @@ class AvaticaClient(object):
while True:
logger.debug("POST %s %r %r", self.url.geturl(), body, headers)
- requestArgs = {'data':body, 'stream':True, 'headers':headers}
+ requestArgs = {'data': body, 'stream': True, 'headers': headers}
if self.auth is not None:
- requestArgs.update(auth = self.auth)
+ requestArgs.update(auth=self.auth)
if self.verify is not None:
- requestArgs.update(verify = self.verify)
+ requestArgs.update(verify=self.verify)
try:
response = requests.request('post', self.url.geturl(), **requestArgs)
diff --git a/python/phoenixdb/phoenixdb/connection.py b/python/phoenixdb/phoenixdb/connection.py
index e041425..1cd562b 100644
--- a/python/phoenixdb/phoenixdb/connection.py
+++ b/python/phoenixdb/phoenixdb/connection.py
@@ -16,6 +16,7 @@
import logging
import uuid
import weakref
+
from phoenixdb import errors
from phoenixdb.avatica.client import OPEN_CONNECTION_PROPERTIES
from phoenixdb.cursor import Cursor
@@ -99,12 +100,12 @@ class Connection(object):
def commit(self):
if self._closed:
raise ProgrammingError('the connection is already closed')
- self._client.commit(self._id);
+ self._client.commit(self._id)
def rollback(self):
if self._closed:
raise ProgrammingError('the connection is already closed')
- self._client.rollback(self._id);
+ self._client.rollback(self._id)
def cursor(self, cursor_factory=None):
"""Creates a new cursor.
diff --git a/python/phoenixdb/phoenixdb/cursor.py b/python/phoenixdb/phoenixdb/cursor.py
index e1557c8..1dac835 100644
--- a/python/phoenixdb/phoenixdb/cursor.py
+++ b/python/phoenixdb/phoenixdb/cursor.py
@@ -13,11 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import logging
import collections
-from phoenixdb.types import TypeHelper
-from phoenixdb.errors import ProgrammingError, InternalError
+import logging
+
from phoenixdb.avatica.proto import common_pb2
+from phoenixdb.errors import InternalError, ProgrammingError
+from phoenixdb.types import TypeHelper
__all__ = ['Cursor', 'ColumnDescription', 'DictCursor']
@@ -182,7 +183,7 @@ class Cursor(object):
else:
typed_value.null = False
if is_array:
- if type(value) in [list,tuple]:
+ if type(value) in [list, tuple]:
for element in value:
if mutate_to is not None:
element = mutate_to(element)
diff --git a/python/phoenixdb/phoenixdb/tests/__init__.py b/python/phoenixdb/phoenixdb/tests/__init__.py
index 44b62f7..b541929 100644
--- a/python/phoenixdb/phoenixdb/tests/__init__.py
+++ b/python/phoenixdb/phoenixdb/tests/__init__.py
@@ -15,11 +15,11 @@
import os
import unittest
+
import phoenixdb
-import time
TEST_DB_URL = os.environ.get('PHOENIXDB_TEST_DB_URL')
-#TEST_DB_URL = "http://localhost:8765"
+# TEST_DB_URL = "http://localhost:8765"
TEST_DB_TRUSTSTORE = os.environ.get('PHOENIXDB_TEST_DB_TRUSTSTORE')
TEST_DB_AUTHENTICATION = os.environ.get('PHOENIXDB_TEST_DB_AUTHENTICATION')
TEST_DB_AVATICA_USER = os.environ.get('PHOENIXDB_TEST_DB_AVATICA_USER')
@@ -27,35 +27,39 @@ TEST_DB_AVATICA_PASSWORD = os.environ.get('PHOENIXDB_TEST_DB_AVATICA_PASSWORD')
httpArgs = {}
if TEST_DB_TRUSTSTORE is not None:
- httpArgs.update(verify = TEST_DB_TRUSTSTORE)
+ httpArgs.update(verify=TEST_DB_TRUSTSTORE)
if TEST_DB_AUTHENTICATION is not None:
- httpArgs.update(authentication = TEST_DB_AUTHENTICATION)
+ httpArgs.update(authentication=TEST_DB_AUTHENTICATION)
if TEST_DB_AVATICA_USER is not None:
- httpArgs.update(avatica_user = TEST_DB_AVATICA_USER)
+ httpArgs.update(avatica_user=TEST_DB_AVATICA_USER)
if TEST_DB_AVATICA_PASSWORD is not None:
- httpArgs.update(avatica_password = TEST_DB_AVATICA_PASSWORD)
+ httpArgs.update(avatica_password=TEST_DB_AVATICA_PASSWORD)
+
@unittest.skipIf(TEST_DB_URL is None, "these tests require the PHOENIXDB_TEST_DB_URL environment variable set to a clean database")
class DatabaseTestCase(unittest.TestCase):
def setUp(self):
self.conn = phoenixdb.connect(TEST_DB_URL, autocommit=True, **httpArgs)
+
def closeDb():
self.conn.close()
self.addCleanup(closeDb)
def reopen(self, **avaticaArgs):
self.conn.close()
- self.conn = phoenixdb.connect(TEST_DB_URL, **avaticaArgs, **httpArgs)
+ kwargs = avaticaArgs.copy()
+ kwargs.update(httpArgs)
+ self.conn = phoenixdb.connect(TEST_DB_URL, **kwargs)
def addTableCleanup(self, name):
def dropTable():
with self.conn.cursor() as cursor:
- cursor.execute("DROP TABLE IF EXISTS {table}".format(table = name))
+ cursor.execute("DROP TABLE IF EXISTS {table}".format(table=name))
self.addCleanup(dropTable)
def createTable(self, name, statement):
with self.conn.cursor() as cursor:
- cursor.execute("DROP TABLE IF EXISTS {table}".format(table = name))
- cursor.execute(statement.format(table = name))
+ cursor.execute("DROP TABLE IF EXISTS {table}".format(table=name))
+ cursor.execute(statement.format(table=name))
self.addTableCleanup(name)
diff --git a/python/phoenixdb/phoenixdb/tests/test_avatica.py b/python/phoenixdb/phoenixdb/tests/test_avatica.py
index 6152814..20a7e0b 100644
--- a/python/phoenixdb/phoenixdb/tests/test_avatica.py
+++ b/python/phoenixdb/phoenixdb/tests/test_avatica.py
@@ -14,6 +14,7 @@
# limitations under the License.
import unittest
+
from phoenixdb.avatica.client import parse_url, urlparse
diff --git a/python/phoenixdb/phoenixdb/tests/test_db.py b/python/phoenixdb/phoenixdb/tests/test_db.py
index 485e223..ea81e4a 100644
--- a/python/phoenixdb/phoenixdb/tests/test_db.py
+++ b/python/phoenixdb/phoenixdb/tests/test_db.py
@@ -14,10 +14,10 @@
# limitations under the License.
import unittest
+
import phoenixdb.cursor
from phoenixdb.errors import InternalError
-from phoenixdb.tests import TEST_DB_URL
-from phoenixdb.tests import DatabaseTestCase
+from phoenixdb.tests import DatabaseTestCase, TEST_DB_URL
@unittest.skipIf(TEST_DB_URL is None, "these tests require the PHOENIXDB_TEST_DB_URL environment variable set to a clean database")
@@ -83,5 +83,5 @@ class PhoenixDatabaseTest(DatabaseTestCase):
self.assertEqual(cursor.fetchall(), [[1, 'text 1']])
def test_transaction(self):
- #Todo write some transaction tests
- pass
\ No newline at end of file
+ # Todo write some transaction tests
+ pass
diff --git a/python/phoenixdb/phoenixdb/tests/test_dbapi20.py b/python/phoenixdb/phoenixdb/tests/test_dbapi20.py
index 1be6cb0..c80ecb4 100644
--- a/python/phoenixdb/phoenixdb/tests/test_dbapi20.py
+++ b/python/phoenixdb/phoenixdb/tests/test_dbapi20.py
@@ -14,10 +14,11 @@
# limitations under the License.
import unittest
+
import phoenixdb
+from phoenixdb.tests import TEST_DB_URL, httpArgs
+
from . import dbapi20
-from phoenixdb.tests import TEST_DB_URL
-from phoenixdb.tests import httpArgs
@unittest.skipIf(TEST_DB_URL is None, "these tests require the PHOENIXDB_TEST_DB_URL environment variable set to a clean database")
diff --git a/python/phoenixdb/phoenixdb/tests/test_types.py b/python/phoenixdb/phoenixdb/tests/test_types.py
index 598b5b8..19c730d 100644
--- a/python/phoenixdb/phoenixdb/tests/test_types.py
+++ b/python/phoenixdb/phoenixdb/tests/test_types.py
@@ -13,11 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import datetime
import sys
import unittest
-import datetime
-import phoenixdb
from decimal import Decimal
+
+import phoenixdb
from phoenixdb.tests import DatabaseTestCase
@@ -226,6 +227,16 @@ class TypesTest(DatabaseTestCase):
[5, None],
])
+ # Minimal date/time/timestamp type test that doesn't trigger PHOENIX-4664
+ def test_time_minimal(self):
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val1 date, val2 time, val3 timestamp)")
+ with self.conn.cursor() as cursor:
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, '2015-07-12', '2015-07-12 13:01:02', '2015-07-12 13:01:02.123')")
+ cursor.execute("SELECT * FROM phoenixdb_test_tbl1 ORDER BY id")
+ self.assertEqual(cursor.fetchall(), [
+ [1, datetime.date(2015, 7, 12), datetime.time(13, 1, 2), datetime.datetime(2015, 7, 12, 13, 1, 2, 123000)]
+ ])
+
@unittest.skip("https://issues.apache.org/jira/browse/CALCITE-796")
def test_timestamp_full(self):
self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val timestamp)")
@@ -248,6 +259,25 @@ class TypesTest(DatabaseTestCase):
cursor.execute("SELECT id, val FROM phoenixdb_test_tbl1 ORDER BY id")
self.assertEqual(cursor.fetchall(), [[1, 'abc'], [2, None], [3, 'abc'], [4, None], [5, None], [6, None]])
+ @unittest.skipIf(sys.version_info[0] < 3, "phoenixdb doesn't support unicode strings in Python2")
+ def test_unicode(self):
+ self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val varchar)")
+ with self.conn.cursor() as cursor:
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (1, \
+ '\u00E1rv\u00EDzt\u0171r\u0151 t\u00FCk\u00F6rf\u00FAr\u00F3g\u00E9p')")
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (2, '\u265E')")
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (3, '\U0001F600')")
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (4, ?)",
+ ['\u00E1rv\u00EDzt\u0171r\u0151 t\u00FCk\u00F6rf\u00FAr\u00F3g\u00E9p'])
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (5, ?)", ['\u265E'])
+ cursor.execute("UPSERT INTO phoenixdb_test_tbl1 VALUES (6, ?)", ['\U0001F600'])
+ cursor.execute("SELECT id, val FROM phoenixdb_test_tbl1 ORDER BY id")
+ self.assertEqual(cursor.fetchall(),
+ [[1, '\u00E1rv\u00EDzt\u0171r\u0151 t\u00FCk\u00F6rf\u00FAr\u00F3g\u00E9p'],
+ [2, '\u265E'], [3, '\U0001F600'],
+ [4, '\u00E1rv\u00EDzt\u0171r\u0151 t\u00FCk\u00F6rf\u00FAr\u00F3g\u00E9p'],
+ [5, '\u265E'], [6, '\U0001F600']])
+
def test_varchar_very_long(self):
self.createTable("phoenixdb_test_tbl1", "CREATE TABLE {table} (id integer primary key, val varchar)")
with self.conn.cursor() as cursor:
diff --git a/python/phoenixdb/phoenixdb/types.py b/python/phoenixdb/phoenixdb/types.py
index 20f8572..e446eca 100644
--- a/python/phoenixdb/phoenixdb/types.py
+++ b/python/phoenixdb/phoenixdb/types.py
@@ -13,12 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import datetime
import sys
import time
-import datetime
from decimal import Decimal
+
from phoenixdb.avatica.proto import common_pb2
-from builtins import staticmethod
+
__all__ = [
'Date', 'Time', 'Timestamp', 'DateFromTicks', 'TimeFromTicks', 'TimestampFromTicks',
@@ -90,7 +91,8 @@ def datetime_to_java_sql_timestamp(d):
td = d - datetime.datetime(1970, 1, 1)
return td.microseconds // 1000 + (td.seconds + td.days * 24 * 3600) * 1000
-#FIXME This doesn't seem to be used anywhere in the code
+
+# FIXME This doesn't seem to be used anywhere in the code
class ColumnType(object):
def __init__(self, eq_types):
@@ -181,67 +183,70 @@ This mapping should be structured as:
JDBC_TO_REP = dict([
# These are the standard types that are used in Phoenix
- (-6, common_pb2.BYTE), #TINYINT
- (5, common_pb2.SHORT), #SMALLINT
- (4, common_pb2.INTEGER), #INTEGER
- (-5, common_pb2.LONG), #BIGINT
- (6, common_pb2.DOUBLE), #FLOAT
- (8, common_pb2.DOUBLE), #DOUBLE
- (2, common_pb2.BIG_DECIMAL), #NUMERIC
- (1, common_pb2.STRING), #CHAR
- (91, common_pb2.JAVA_SQL_DATE), #DATE
- (93, common_pb2.JAVA_SQL_TIMESTAMP), #TIME
- (-2, common_pb2.BYTE_STRING), #BINARY
- (-3, common_pb2.BYTE_STRING), #VARBINARY
- (16, common_pb2.BOOLEAN), #BOOLEAN
+ (-6, common_pb2.BYTE), # TINYINT
+ (5, common_pb2.SHORT), # SMALLINT
+ (4, common_pb2.INTEGER), # INTEGER
+ (-5, common_pb2.LONG), # BIGINT
+ (6, common_pb2.DOUBLE), # FLOAT
+ (8, common_pb2.DOUBLE), # DOUBLE
+ (2, common_pb2.BIG_DECIMAL), # NUMERIC
+ (1, common_pb2.STRING), # CHAR
+ (91, common_pb2.JAVA_SQL_DATE), # DATE
+ (92, common_pb2.JAVA_SQL_TIME), # TIME
+ (93, common_pb2.JAVA_SQL_TIMESTAMP), # TIMESTAMP
+ (-2, common_pb2.BYTE_STRING), # BINARY
+ (-3, common_pb2.BYTE_STRING), # VARBINARY
+ (16, common_pb2.BOOLEAN), # BOOLEAN
# These are the Non-standard types defined by Phoenix
- (19, common_pb2.JAVA_SQL_DATE), #UNSIGNED_DATE
- (15, common_pb2.DOUBLE), #UNSIGNED_DOUBLE
- (14, common_pb2.DOUBLE), #UNSIGNED_FLOAT
- (9, common_pb2.INTEGER), #UNSIGNED_INT
- (10, common_pb2.LONG), #UNSIGNED_LONG
- (13, common_pb2.SHORT), #UNSIGNED_SMALLINT
- (20, common_pb2.JAVA_SQL_TIMESTAMP), #UNSIGNED_TIMESTAMP
- (11, common_pb2.BYTE), #UNSIGNED_TINYINT
+ (19, common_pb2.JAVA_SQL_DATE), # UNSIGNED_DATE
+ (15, common_pb2.DOUBLE), # UNSIGNED_DOUBLE
+ (14, common_pb2.DOUBLE), # UNSIGNED_FLOAT
+ (9, common_pb2.INTEGER), # UNSIGNED_INT
+ (10, common_pb2.LONG), # UNSIGNED_LONG
+ (13, common_pb2.SHORT), # UNSIGNED_SMALLINT
+ (20, common_pb2.JAVA_SQL_TIMESTAMP), # UNSIGNED_TIMESTAMP
+ (11, common_pb2.BYTE), # UNSIGNED_TINYINT
# The following are not used by Phoenix, but some of these are used by Avaticafor
# parameter types
- (-7, common_pb2.BOOLEAN), #BIT
- (7, common_pb2.DOUBLE), #REAL
- (3, common_pb2.BIG_DECIMAL), #DECIMAL
- (12, common_pb2.STRING), #VARCHAR
- (-1, common_pb2.STRING), #LONGVARCHAR
- (-4, common_pb2.BYTE_STRING), #LONGVARBINARY
- (2004, common_pb2.BYTE_STRING), #BLOB
- (2005, common_pb2.STRING), #CLOB
- (-15, common_pb2.STRING), #NCHAR
- (-9, common_pb2.STRING), #NVARCHAR
- (-16, common_pb2.STRING), #LONGNVARCHAR
- (2011, common_pb2.STRING), #NCLOB
- (2009, common_pb2.STRING), #SQLXML
+ (-7, common_pb2.BOOLEAN), # BIT
+ (7, common_pb2.DOUBLE), # REAL
+ (3, common_pb2.BIG_DECIMAL), # DECIMAL
+ (12, common_pb2.STRING), # VARCHAR
+ (-1, common_pb2.STRING), # LONGVARCHAR
+ (-4, common_pb2.BYTE_STRING), # LONGVARBINARY
+ (2004, common_pb2.BYTE_STRING), # BLOB
+ (2005, common_pb2.STRING), # CLOB
+ (-15, common_pb2.STRING), # NCHAR
+ (-9, common_pb2.STRING), # NVARCHAR
+ (-16, common_pb2.STRING), # LONGNVARCHAR
+ (2011, common_pb2.STRING), # NCLOB
+ (2009, common_pb2.STRING), # SQLXML
# These are defined by JDBC, but cannot be mapped
- #NULL
- #OTHER
- #JAVA_OBJECT
- #DISTINCT
- #STRUCT
- #ARRAY 2003 - We are handling this as a special case
- #REF
- #DATALINK
- #ROWID
- #REF_CURSOR
- #TIME WITH TIMEZONE
- #TIMESTAMP WITH TIMEZONE
+ # NULL
+ # OTHER
+ # JAVA_OBJECT
+ # DISTINCT
+ # STRUCT
+ # ARRAY 2003 - We are handling this as a special case
+ # REF
+ # DATALINK
+ # ROWID
+ # REF_CURSOR
+ # TIME WITH TIMEZONE
+ # TIMESTAMP WITH TIMEZONE
])
"""Maps the JDBC Type IDs to Protobuf Reps """
JDBC_MAP = {}
-for k,v in JDBC_TO_REP.items():
+for k, v in JDBC_TO_REP.items():
JDBC_MAP[k & 0xffffffff] = REP_MAP[v]
"""Flips the available types to allow for faster lookup by JDBC type ID
It has the same format as REP_MAP, but is keyed by JDBC type ID
"""
+
+
class TypeHelper(object):
@staticmethod
@@ -293,7 +298,7 @@ class TypeHelper(object):
@staticmethod
def _from_jdbc(jdbc_code):
if jdbc_code not in JDBC_MAP:
- #This should not happen. It's either a bug, or Avatica has added new types
+ # This should not happen. It's either a bug, or Avatica has added new types
raise NotImplementedError('JDBC TYPE CODE {} is not supported'.format(jdbc_code))
return JDBC_MAP[jdbc_code]
diff --git a/python/phoenixdb/requirements.txt b/python/phoenixdb/requirements.txt
index 72fd653..b061374 100644
--- a/python/phoenixdb/requirements.txt
+++ b/python/phoenixdb/requirements.txt
@@ -16,10 +16,6 @@
# specific language governing permissions and limitations
# under the License.
#
--e git+https://bitbucket.org/lalinsky/python-sqlline.git#egg=sqlline
-nose
protobuf>=3.0.0
-sphinx
-flake8
requests
requests-gssapi
diff --git a/python/phoenixdb/setup.py b/python/phoenixdb/setup.py
index 3accb45..d1f0ce5 100644
--- a/python/phoenixdb/setup.py
+++ b/python/phoenixdb/setup.py
@@ -32,16 +32,16 @@ def readme():
return f.read()
-version = "0.7"
+version = "1.0.0.dev"
setup(
name="phoenixdb",
version=version,
description="Phoenix database adapter for Python",
long_description=readme(),
- author="Lukas Lalinsky",
- author_email="lukas@oxygene.sk",
- url="https://bitbucket.org/lalinsky/python-phoenixdb",
+ author="Apache Software Foundation",
+ author_email="dev@phoenix.apache.org",
+ url="http://phoenix.apache.org/python.html",
license="Apache 2",
packages=find_packages(),
include_package_data=True,
@@ -60,10 +60,24 @@ setup(
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
+ 'Programming Language :: Python :: 3.7',
+ 'Programming Language :: Python :: 3.8',
],
install_requires=[
'protobuf>=3.0.0',
'requests',
'requests-gssapi'
+ ],
+ tests_require=[
+ 'nose',
+ 'flake8'
+ ],
+ setup_requires=[
+ # Later versions don't work with python2.7
+ 'Sphinx<2.0.0',
+ # These are Sphinx dependencies, included only to be version managed for python2
+ 'MarkupSafe<2.0.0',
+ 'Jinja2<3.0.0',
+ 'pyparsing<3.0.0'
]
)
diff --git a/python/phoenixdb/tox.ini b/python/phoenixdb/tox.ini
index 908696a..49d7c06 100644
--- a/python/phoenixdb/tox.ini
+++ b/python/phoenixdb/tox.ini
@@ -14,11 +14,12 @@
# limitations under the License.
[tox]
-envlist = py27,py35,py36
-
+envlist = py27,py34,py35,py36,py37,py38
[testenv]
passenv = PHOENIXDB_TEST_DB_URL
commands =
- flake8
+ flake8 phoenixdb
nosetests -v
deps = -rrequirements.txt
+ nose
+ flake8
diff --git a/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
index 76c38ee..0540a3d 100644
--- a/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
+++ b/queryserver-it/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
@@ -43,6 +43,7 @@ import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.queryserver.QueryServerProperties;
import org.apache.phoenix.util.ThinClientUtil;
import org.junit.AfterClass;
+import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
@@ -68,7 +69,9 @@ public class QueryServerBasicsIT extends BaseHBaseManagedTimeIT {
@BeforeClass
public static synchronized void beforeClass() throws Exception {
CONF = getTestClusterConfig();
- CONF.setInt(QueryServerProperties.QUERY_SERVER_HTTP_PORT_ATTRIB, 0);
+ if(System.getProperty("do.not.randomize.pqs.port") == null) {
+ CONF.setInt(QueryServerProperties.QUERY_SERVER_HTTP_PORT_ATTRIB, 0);
+ }
String url = getUrl();
AVATICA_SERVER = new QueryServerThread(new String[] { url }, CONF,
QueryServerBasicsIT.class.getName());
@@ -376,10 +379,10 @@ public class QueryServerBasicsIT extends BaseHBaseManagedTimeIT {
}
}
- @Ignore
@Test
//Quick and dirty way start up a local Phoenix+PQS instance for testing against
public void startLocalPQS() throws Exception {
+ Assume.assumeNotNull(System.getProperty("start.unsecure.pqs"));
System.out.println("CONN STRING:" + CONN_STRING);
System.out.println("Tests suspended!!!");
System.out.println("Kill maven run to stop server");
diff --git a/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java b/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
index 40c5e03..95c32f5 100644
--- a/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
+++ b/queryserver-it/src/it/java/org/apache/phoenix/end2end/SecureQueryServerPhoenixDBIT.java
@@ -333,14 +333,13 @@ public class SecureQueryServerPhoenixDBIT {
}
//This takes about 300s, so we are not running this by default
- @Ignore
@Test
public void testFullSuite() throws Exception {
+ Assume.assumeNotNull(System.getProperty("run.full.python.testsuite"));
File file = new File(".");
runShellScript("python", "-m", "unittest", "discover", "-v", "-s", Paths.get(file.getAbsolutePath(), "..","python", "phoenixdb").toString());
}
- @Ignore
@Test
//Quick and dirty way start up a local Phoenix+PQS instance for testing against
//When started, this will write a setup script into target/krb_setup.sh
@@ -348,6 +347,7 @@ public class SecureQueryServerPhoenixDBIT {
//and set the environment so that the python unit tests can run against this instance.
//You'll need to kill the test manually
public void startLocalPQS() throws Exception {
+ Assume.assumeNotNull(System.getProperty("start.secure.pqs"));
runShellScript("sleep", "86400");
}