[beam] branch master updated: preinstall various packages for better startup performance and reliability

[al...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 9fa560a  preinstall various packages for better startup performance and reliability
     new 26b0f9d  This closes #4192
9fa560a is described below

commit 9fa560a8e11b49bd3afc46ec4c7301723ca634bb
Author: Ahmet Altay <al...@google.com>
AuthorDate: Tue Nov 28 18:16:56 2017 -0800

    preinstall various packages for better startup performance and reliability
---
 sdks/python/container/Dockerfile | 62 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 60 insertions(+), 2 deletions(-)

diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile
index 826e36c..7cbe6c2 100644
--- a/sdks/python/container/Dockerfile
+++ b/sdks/python/container/Dockerfile
@@ -19,8 +19,66 @@
 FROM python:2
 MAINTAINER "Apache Beam <de...@beam.apache.org>"
 
-# TODO(herohde): preinstall various packages for better startup
-# performance and reliability.
+# Install native bindings required for dependencies.
+RUN apt-get update && \
+    apt-get install -y \
+       # These packages are needed for "pip install python-snappy" below.
+       libsnappy-dev \
+       # This package is needed for "pip install pyyaml" below to have c bindings.
+       libyaml-dev \
+       && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install packages required by the Python SDK.
+#
+# These packages should be kept in sync with the dependencies at
+# sdks/python/setup.py.  If not installed, sdk harness will install these at
+# runtime, but we would like to avoid doing this so that we do not depend on
+# PyPI at runtime whenever possible.
+#
+# Also install cython, numpy, pandas and scipy as well as their dependencies.
+# These are standard Python packages, likely to be used by python SDK customers,
+# and their dependencies.
+#
+RUN \
+    # These are packages needed by the Python SDK.
+    pip install "avro == 1.8.2" && \
+    pip install "crcmod == 1.7" && \
+    pip install "dill == 0.2.6" && \
+    pip install "grpcio == 1.3.0" && \
+    pip install "httplib2 == 0.9.2" && \
+    pip install "mock == 2.0.0" && \
+    pip install "oauth2client == 3.0.0" && \
+    pip install "protobuf == 3.3.0" && \
+    pip install "pyyaml == 3.12" && \
+    pip install "pyvcf == 0.6.8" && \
+    pip install "six == 1.10.0" && \
+    pip install "typing == 3.6.1" && \
+    pip install "futures == 3.1.1" && \
+    # Setup packages
+    pip install "nose == 1.3.7" && \
+    # GCP extra features
+    pip install "google-apitools == 0.5.11" && \
+    pip install "proto-google-cloud-datastore-v1 == 0.90.4" && \
+    pip install "googledatastore == 7.0.1" && \
+    pip install "google-cloud-pubsub == 0.26.0" && \
+    pip install "google-cloud-bigquery == 0.25.0" && \
+    # Optional packages
+    pip install "cython == 0.27.2" && \
+    pip install "guppy == 0.1.10" && \
+    pip install "python-snappy == 0.5.1" && \
+    # These are additional packages likely to be used by customers.
+    pip install "numpy == 1.13.3" --no-binary=:all: && \
+    pip install "pandas == 0.18.1" && \
+    pip install "scipy == 1.0.0" && \
+    pip install "protobuf == 3.3.0" && \
+    pip install "tensorflow == 1.4.0" && \
+    pip install "protorpc == 0.11.1" && \
+    pip install "python-gflags == 3.0.6" && \
+    # Remove pip cache.
+    rm -rf /root/.cache/pip && \
+    # Check that the fast implementation of protobuf is used.
+    python -c "from google.protobuf.internal import api_implementation; assert api_implementation._default_implementation_type == 'cpp'; print 'Verified fast protobuf used.'"
 
 ADD target/linux_amd64/boot /opt/apache/beam/
 

-- 
To stop receiving notification emails like this one, please contact
['"commits@beam.apache.org" <co...@beam.apache.org>'].