You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2021/03/31 03:28:03 UTC
[impala] 04/04: IMPALA-10608: Update kudu-python version and remove
some unused packages
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit e7fc18c4ea4907b86c36ffba90a59e98d32c28e0
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Wed Mar 24 20:47:12 2021 -0700
IMPALA-10608: Update kudu-python version and remove some unused packages
This updates kudu-python to version 1.14.0 (from 1.2.0).
As part of this, it disables ccache for bootstrap_virtualenv.py.
ccache wasn't working anyway, because pip install uses random
temporary directories. It also needs to copy a few files to
the build directory for the Kudu install. The advantage to
upgrading is that the new version no longer has a numpy dependency.
Additionally, this modifies a few minor packages:
- virtualenv moves to the latest version prior to the rewrite
that accompanied version 20 (i.e. 16.10.7).
- setuptools moves to the last version that supports python 2.7 (44.1.1)
- remove botos3, ipython, and ordereddict
These changes speed up installing the virtualenv
Before:
real 3m11.956s
user 2m49.620s
sys 0m14.266s
After:
real 1m38.798s
user 1m33.591s
sys 0m8.112s
Testing:
- Hand tests, GVO run
Change-Id: Ib47770df9e46de448fe2bffef7abe2c3aa942fb9
Reviewed-on: http://gerrit.cloudera.org:8080/17231
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/compare_branches.py | 5 +----
bin/impala-ipython | 23 -----------------------
infra/python/bootstrap_virtualenv.py | 15 ++++-----------
infra/python/deps/kudu-requirements.txt | 2 +-
infra/python/deps/requirements.txt | 21 ++++-----------------
infra/python/deps/setuptools-requirements.txt | 7 ++++---
tests/comparison/cluster.py | 2 +-
7 files changed, 15 insertions(+), 60 deletions(-)
diff --git a/bin/compare_branches.py b/bin/compare_branches.py
index 8b54636..a3ae299 100755
--- a/bin/compare_branches.py
+++ b/bin/compare_branches.py
@@ -68,10 +68,7 @@ import subprocess
import sys
from collections import defaultdict
-try:
- from collections import OrderedDict
-except ImportError:
- from ordereddict import OrderedDict
+from collections import OrderedDict
from pprint import pformat
def create_parser():
diff --git a/bin/impala-ipython b/bin/impala-ipython
deleted file mode 100755
index 64b2fe7..0000000
--- a/bin/impala-ipython
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-##############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-##############################################################################
-
-source $(dirname "$0")/impala-python-common.sh
-exec "$PY_ENV_DIR/bin/ipython" "$@"
diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py
index 604177d..7f7e6fc 100644
--- a/infra/python/bootstrap_virtualenv.py
+++ b/infra/python/bootstrap_virtualenv.py
@@ -108,16 +108,6 @@ def exec_cmd(args, **kwargs):
return output
-def use_ccache():
- '''Returns true if ccache is available and should be used'''
- if 'DISABLE_CCACHE' in os.environ: return False
- try:
- exec_cmd(['ccache', '-V'])
- return True
- except:
- return False
-
-
def select_cc():
'''Return the C compiler command that should be used as a string or None if the
compiler is not available '''
@@ -127,7 +117,6 @@ def select_cc():
toolchain_gcc_dir = toolchain_pkg_dir("gcc")
cc = os.path.join(toolchain_gcc_dir, "bin/gcc")
if not os.path.exists(cc): return None
- if use_ccache(): cc = "ccache %s" % cc
return cc
@@ -284,6 +273,10 @@ def install_kudu_client_if_possible():
env = dict(os.environ)
env["KUDU_HOME"] = fake_kudu_build_dir
kudu_client_dir = find_kudu_client_install_dir()
+ # Copy the include directory to the fake build directory
+ kudu_include_dir = os.path.join(kudu_client_dir, "include")
+ shutil.copytree(kudu_include_dir,
+ os.path.join(fake_kudu_build_dir, "build", "latest", "src"))
env["CPLUS_INCLUDE_PATH"] = os.path.join(kudu_client_dir, "include")
env["LIBRARY_PATH"] = os.path.pathsep.join([os.path.join(kudu_client_dir, 'lib'),
os.path.join(kudu_client_dir, 'lib64')])
diff --git a/infra/python/deps/kudu-requirements.txt b/infra/python/deps/kudu-requirements.txt
index 6dd4ada..bb5e2c4 100644
--- a/infra/python/deps/kudu-requirements.txt
+++ b/infra/python/deps/kudu-requirements.txt
@@ -19,4 +19,4 @@
# and also depends on Cython being installed into the virtualenv, so it must be installed
# after the toolchain is bootstrapped and all requirements in requirements.txt and
# compiled-requirements.txt are installed into the virtualenv.
-kudu-python==1.2.0
+kudu-python==1.14.0
diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt
index cc0d40b..42bcf1a 100644
--- a/infra/python/deps/requirements.txt
+++ b/infra/python/deps/requirements.txt
@@ -22,14 +22,6 @@
allpairs == 2.0.1
argparse == 1.4.0
-# TODO: boto3 is now unused, it can be removed.
-boto3 == 1.2.3
- simplejson == 3.3.0 # For python version 2.6
- botocore == 1.3.30
- python_dateutil == 2.5.2
- docutils == 0.12
- jmespath == 0.9.0
- futures == 3.0.5
cm-api == 10.0.0
# Already available as part of python on Linux.
readline == 6.2.4.1; sys_platform == 'darwin'
@@ -41,13 +33,13 @@ gcovr == 3.4
hdfs == 2.0.2
docopt == 0.6.2
execnet == 1.4.0
+ apipkg == 1.4
impyla == 0.17a1
bitarray == 1.2.1
sasl == 0.2.1
# six == 1.14.0 (specified separately)
thrift_sasl == 0.4.2
kazoo == 2.2.1
-ordereddict == 1.1
pexpect == 3.3
pg8000 == 1.10.2
prettytable == 0.7.2
@@ -72,14 +64,9 @@ sh == 1.11
six == 1.14.0
sqlparse == 0.3.1
texttable == 0.8.3
-virtualenv == 13.1.0
+virtualenv == 16.7.10
# Required for Kudu:
- Cython == 0.23.4
- numpy == 1.10.4
- pytz == 2018.3
+ Cython == 0.29.14
+ pytz == 2020.1
-# For dev purposes, not used in scripting. Version 1.2.1 is the latest that supports 2.6.
-# TODO: this seems unused and could be removed
-ipython == 1.2.1
- apipkg == 1.4
diff --git a/infra/python/deps/setuptools-requirements.txt b/infra/python/deps/setuptools-requirements.txt
index 38ef911..071f9fc5 100644
--- a/infra/python/deps/setuptools-requirements.txt
+++ b/infra/python/deps/setuptools-requirements.txt
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-# Newer versions of setuptools don't support Python 2.6
-setuptools == 36.8.0
-setuptools-scm == 1.15.4
+# Newer versions of setuptools don't support Python 2.7
+setuptools == 44.1.1
+ wheel == 0.35.1
+setuptools-scm == 4.1.2
diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py
index 6f850e2..f3a3c50 100644
--- a/tests/comparison/cluster.py
+++ b/tests/comparison/cluster.py
@@ -29,8 +29,8 @@ import subprocess
from abc import ABCMeta, abstractproperty
from cm_api.api_client import ApiResource as CmApiResource
from collections import defaultdict
+from collections import OrderedDict
from contextlib import contextmanager
-from ordereddict import OrderedDict
from getpass import getuser
from itertools import izip
from multiprocessing.pool import ThreadPool