You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@climate.apache.org by go...@apache.org on 2016/10/20 22:16:27 UTC
climate git commit: CLIMATE-874 - Remove Easy-OCW
Repository: climate
Updated Branches:
refs/heads/master f9a9517cb -> fc520781d
CLIMATE-874 - Remove Easy-OCW
Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/fc520781
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/fc520781
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/fc520781
Branch: refs/heads/master
Commit: fc520781d218ad2ab6bade79cd3779d1fd6e0dee
Parents: f9a9517
Author: Alex Goodman <ag...@users.noreply.github.com>
Authored: Wed Oct 19 20:49:18 2016 -0700
Committer: Alex Goodman <ag...@users.noreply.github.com>
Committed: Thu Oct 20 13:58:33 2016 -0700
----------------------------------------------------------------------
.travis.yml | 36 ++++-
deps_py2.txt | 14 ++
deps_py3.txt | 11 ++
easy-ocw/install-osx.sh | 242 -------------------------------
easy-ocw/install-ubuntu.sh | 157 --------------------
easy-ocw/ocw-conda-dependencies.txt | 23 ---
easy-ocw/ocw-pip-dependencies.txt | 12 --
ocw/dataset_loader.py | 16 +-
test.sh | 6 +-
test_smoke.py | 14 +-
10 files changed, 80 insertions(+), 451 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 10d9d17..1a50d8b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,23 +17,49 @@
language: python
python:
- "2.7"
- - "3.2"
- - "3.3"
- "3.4"
- "3.5"
notifications:
email: false
install:
- sudo apt-get update
- - source easy-ocw/install-ubuntu.sh
- - pip install python-coveralls
+ # We do this conditionally because it saves us some downloading if the
+ # version is the same.
+ - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+ wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
+ DEPS=deps_py2.txt;
+ else
+ wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+ DEPS=deps_py3.txt;
+ fi
+ - bash miniconda.sh -b -p $HOME/miniconda
+ - export PATH="$HOME/miniconda/bin:$PATH"
+ - hash -r
+ - conda config --set always_yes yes --set changeps1 no
+ - conda update -q conda
+
+ # Useful for debugging any issues with conda
+ - conda info -a
+
+ # Add conda-forge to search so all dependencies can be resolved
+ - conda config --add channels conda-forge
+ - conda create -q -n test-environment nose python=$TRAVIS_PYTHON_VERSION --file $DEPS
+
+ # Need this to exclude pydap test for python 3
+ - pip install nose-exclude
+
+ # Install ocw
+ - source activate test-environment
+ - python setup.py install
+
# Workaround to get travis build working with matplotlib
before_script:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
- sleep 3 # give xvfb some time to start
+
script:
- chmod a+x test.sh
- - ./test.sh
+ - ./test.sh $TRAVIS_PYTHON_VERSION
after_success:
- coveralls
http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/deps_py2.txt
----------------------------------------------------------------------
diff --git a/deps_py2.txt b/deps_py2.txt
new file mode 100644
index 0000000..c5cd8b6
--- /dev/null
+++ b/deps_py2.txt
@@ -0,0 +1,14 @@
+numpy
+scipy
+matplotlib
+basemap
+netcdf4
+h5py
+bottle
+pydap
+python-dateutil
+mock
+webtest
+myproxyclient
+esgf-pyclient
+podaacpy
http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/deps_py3.txt
----------------------------------------------------------------------
diff --git a/deps_py3.txt b/deps_py3.txt
new file mode 100644
index 0000000..b5fbfa8
--- /dev/null
+++ b/deps_py3.txt
@@ -0,0 +1,11 @@
+numpy
+scipy
+matplotlib
+basemap
+netcdf4
+h5py
+bottle
+python-dateutil
+mock
+webtest
+podaacpy
http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/install-osx.sh
----------------------------------------------------------------------
diff --git a/easy-ocw/install-osx.sh b/easy-ocw/install-osx.sh
deleted file mode 100755
index aeaeae6..0000000
--- a/easy-ocw/install-osx.sh
+++ /dev/null
@@ -1,242 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-export ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future
-
-help()
-{
-cat << ENDHELP
-
-Easy OCW assists with the building of the Apache Open Climate Workbench and its dependencies
-
-Flags:
- -h Display this help message.
- -e Install and configure a virtualenv environment before installation.
- -q Quiet install. User prompts are removed (when possible).
-
-It is recommended that you pass -e when running this script. If you don't, parts
-of this installation will pollute your global Python install. If you're unsure,
-pass -e just to be safe!
-
-ENDHELP
-}
-
-header()
-{
- echo
- echo $1
-}
-
-task()
-{
- echo " - " $1
-}
-
-subtask()
-{
- echo " " $1
-}
-
-echo
-echo "---------------------------------------------------------------------------"
-echo " Welcome to Easy OCW"
-echo "---------------------------------------------------------------------------"
-echo
-
-WITH_VIRTUAL_ENV=0
-WITH_HOMEBREW=0
-WITH_INTERACT=1
-INIT_PWD=$PWD
-
-while getopts ":h :e :q" FLAG
-do
- case $FLAG in
- h)
- help
- exit 1
- ;;
- e)
- WITH_VIRTUAL_ENV=1
- ;;
- q)
- WITH_INTERACT=0
- ;;
- ?)
- help
- exit 1
- ;;
- esac
-done
-
-if [ $WITH_INTERACT == 1 ]; then
-cat << ENDINTRO
-A number of dependencies for OCW will now be installed. Please check the wiki
-for a complete list of dependencies. Additionally, please read the wiki for
-useful installation guidelines and information that may be pertinent to your
-situation. All of this can be found at http://s.apache.org/3p2
-
-ENDINTRO
-
-if [ $WITH_VIRTUAL_ENV != 1 ]; then
-cat << VIRTUALENV_WARNING
-$(tput setaf 1)<-----------------------------[WARNING!]----------------------------------->$(tput sgr 0)
-It is highly recommended that you allow Easy OCW to install the dependencies
-into a virtualenv environment to ensure that your global Python install is
-not affected. If you're UNSURE, you should pass the -e flag
-to this script. If you aren't concerned, or you want to create your own
-virtualenv environment, then feel free to ignore this message.
-
-VIRTUALENV_WARNING
-
-read -p "Press [Yy] to begin installation with the flag -e $(tput setaf 2)[RECOMMENDED]$(tput sgr 0)
-[OR]
-Press [Nn] to continue with the normal installation..." yn
-case $yn in
- [Yy]* )
- WITH_VIRTUAL_ENV=1
- ;;
- [Nn]* )
- WITH_VIRTUAL_ENV=0
- ;;
- * ) echo "Please answer yes or no.." ;;
-esac
-fi
-fi
-
-header "Checking for pip ..."
-command -v pip >/dev/null 2>&1 || {
- task "Unable to locate pip."
- task "Installing Distribute"
- curl http://python-distribute.org/distribute_setup.py | python >> install_log
- subtask "done"
-
- task "Installing Pip"
- curl -O http://pypi.python.org/packages/source/p/pip/pip-1.2.1.tar.gz >> install_log
- tar xzf pip-1.2.1.tar.gz >> install_log
- cd pip-1.2.1/
- python setup.py install >> install_log
- subtask "done"
-}
-
-if [ $WITH_VIRTUAL_ENV == 1 ]; then
- header "Setting up a virtualenv ..."
-
- # Check if virtualenv is installed. If it's not, we'll install it for the user.
- command -v virtualenv >/dev/null 2>&1 || {
- task "Installing virtualenv ..."
- pip install virtualenv >> install_log
- subtask "done"
- }
-
- #Read the path for installation from the user.
- read -e -p "Enter the path where you want to install ocw..." ocw_path
- #check if the file path exists.
- while [ ! -d "$ocw_path" ];
- do
- echo "Path not found..."
- read -e -p "Please enter a valid path..." ocw_path
- done
-
- header "Checking for previously installed ocw virtual environment..."
- if [ -e $ocw_path/ocw/bin/python ]; then
- echo "We found an existing 'ocw' virtualenv on your system in $ocw_path/ocw."
- read -n1 -p "Do you want to replace it with a clean install? y/n :" replace
- if [ "$replace" == "y" ]; then
- echo ""
- echo "$(tput setaf 1)[WARNING!] this will delete all file and data in $ocw_path/ocw on your system.$(tput sgr 0)"
- read -n1 -p "To confirm and proceed type y or n to quit:" confirm
- if [ "$confirm" == "y" ]; then
- echo ""
- echo "Deleting contents of $ocw_path/ocw" >> install_log
- rm -rf $ocw_path/ocw
- else
- echo ""
- echo "Stopping Open Climate Workbench Installation"
- exit
- fi
- else
- echo ""
- echo "Stopping Open Climate Workbench Installation because an existing 'ocw'"
- echo "virtual environment already exists."
- exit
- fi
- fi
-
- cd $ocw_path
- # Create a new environment for OCW work
- task "Creating a new environment in ~/ocw..."
- virtualenv ocw >> install_log
- source $ocw_path/ocw/bin/activate >> install_log
- cd $INIT_PWD
- subtask "done"
-fi
-
-header "Installing conda for package management ..."
-
-pip install conda >> install_log
-conda init >> install_log
-
-
-header "Installing dependencies with conda ..."
-echo | conda install --file ocw-conda-dependencies.txt
-
-# We only use conda for the annoying dependencies like numpy,
-# scipy, matplotlib, and basemap. For everything else, we stick
-# with pip.
-header "Installing additional Python packages"
-pip install -r ocw-pip-dependencies.txt >> install_log
-
-if [ $WITH_VIRTUAL_ENV == 1 ]; then
- echo "***POST INSTALLATION NOTE***
-
-If you are familiar with virtualenv you should know that activating
-the new 'ocw' environment is different because we use conda to install
-packages. An example of the command you want to run is listed in the
-alias below.
-
-To make it easier to change into the 'ocw' virtualenv add the
-following alias to your ~/.bash_profile
-
- alias ocw='source $ocw_path/ocw/bin/activate $ocw_path/ocw/'
-
-When you want to use ocw in the future, you just have to type 'ocw'
-in your terminal."
-else
- echo "***POST INSTALLATION NOTE***
-
-If you have run this script within your own virtualenv you need to know
-a couple of caveats/side effects that are caused by using conda to install
-packages within the virtualenv.
-
-- Virtualenv wrapper will throw errors like those outlined here:
-https://issues.apache.org/jira/browse/CLIMATE-451
-
-- You will not be able to 'activate' the environment using the normal
-virtualenv command, you must instead use the conda activate command as follows:
-
-source path/to/your_env/bin/activate path/to/your_env
-
-Example: (assuming your env is in ~/.virtualenv/ocw)
-
-source ~/.virtualenv/ocw/bin/activate ~/.virtualenv/ocw
-
-"
-
-fi
-
http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/install-ubuntu.sh
----------------------------------------------------------------------
diff --git a/easy-ocw/install-ubuntu.sh b/easy-ocw/install-ubuntu.sh
deleted file mode 100755
index c9db70c..0000000
--- a/easy-ocw/install-ubuntu.sh
+++ /dev/null
@@ -1,157 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-help()
-{
-cat << ENDHELP
-
-Easy OCW assists with the building of the Apache Open Climate Workbench and
-its dependencies.
-
-Flags:
- -h Display this help message.
-
-N.B. This install script has been tested against Ubuntu 12.04 and 14.04.
-Please report problems with this script to dev@climate.apache.org
-ENDHELP
-}
-
-header()
-{
- echo
- echo $1
-}
-
-task()
-{
- echo " - " $1
-}
-
-subtask()
-{
- echo " " $1
-}
-
-echo
-echo "---------------------------------------------------------------------------"
-echo " Welcome to Easy OCW"
-echo "---------------------------------------------------------------------------"
-echo
-
-# Find absolute path to the easy-ocw directory
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-cd $DIR
-ocw_path="$DIR/../climate"
-
-while getopts ":h" FLAG
-do
- case $FLAG in
- h)
- help
- exit 1
- ;;
- ?)
- help
- exit 1
- ;;
- esac
-done
-
-cat << ENDINTRO
-A number of dependencies for OCW will now be installed. Please check the wiki
-for a complete list of dependencies. Additionally, please read the wiki for
-useful installation guidelines and information that may be pertinent to your
-situation. All of this can be found at http://s.apache.org/3p2
-ENDINTRO
-
-cat << VIRTUALENV_WARNING
-$(tput setaf 1)<-----------------------------[WARNING!]----------------------------------->$(tput sgr 0)
-Easy OCW will automatically create a conda environment "venv-ocw".
-All the dependencies will be installed in the virtual environment
-to ensure that your global Python install is not affected.
-
-Please activate the virtual environment venv-ocw after the script has completed to use OCW.
-
-To activate the virtual environment execute
-> source active venv-ocw
-
-To deactivate the virtual environment execute
-> source deactivate
-
-VIRTUALENV_WARNING
-
-echo "Easy-OCW script logs" > install_log
-
-# Install Continuum Analytics Miniconda Python distribution. This gives
-# almost all the dependencies that OCW needs in a single, easy to
-# install package.
-header "Installing Miniconda Python distribution ..."
-
-MACHINE_TYPE=`uname -m`
-if [ ${MACHINE_TYPE} == 'x86_64' ]; then
- link="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
-else
- link="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh"
-fi
-
-header "Checking for conda ..."
-command -v conda >/dev/null 2>&1 || {
- task "Unable to locate conda."
- if ! [ -f Miniconda-latest-linux.sh ]; then
- task "Downloading Miniconda ..."
- wget -O Miniconda-latest-linux.sh $link >> install_log 2>&1
- subtask "done"
- fi
- task "Installing ..."
- bash Miniconda-latest-linux.sh -b -p $HOME/miniconda
- export PATH="$HOME/miniconda/bin/:$PATH"
- subtask "done"
-}
-
-header "Creating venv-ocw environment and installing dependencies"
-conda create --name venv-ocw -y --file ocw-conda-dependencies.txt
-task "Activating venv-ocw virtual environment"
-source activate venv-ocw
-
-conda config --set always_yes yes
-conda update -q conda
-
-header "Conda Information"
-echo "------------------------------------------------------------------"
-conda info -a
-
-echo
-task "Installing python-dev"
-sudo apt-get -y install python-dev >> install_log 2>&1
-subtask "done"
-
-cd $DIR
-
-header "Installing remaining dependencies via pip"
-pip install -r ocw-pip-dependencies.txt
-subtask "done"
-
-header "Installing ocw module"
-
-cd ..
-python setup.py install
-subtask "Finished installing OCW module"
-
-header "For any issues with installation please contact dev@climate.apache.org"
http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/ocw-conda-dependencies.txt
----------------------------------------------------------------------
diff --git a/easy-ocw/ocw-conda-dependencies.txt b/easy-ocw/ocw-conda-dependencies.txt
deleted file mode 100644
index 665be4d..0000000
--- a/easy-ocw/ocw-conda-dependencies.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-numpy>=1.10.4
-scipy>=0.17.0
-matplotlib>=1.5.1
-basemap>=1.0.7
-netcdf4>=1.2.2
-h5py>=2.6.0
http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/ocw-pip-dependencies.txt
----------------------------------------------------------------------
diff --git a/easy-ocw/ocw-pip-dependencies.txt b/easy-ocw/ocw-pip-dependencies.txt
deleted file mode 100644
index 867c801..0000000
--- a/easy-ocw/ocw-pip-dependencies.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-requests>=2.10.0
-bottle>=0.12.9
-pydap>=3.1.1
-webtest>=2.0.21
-nose>=1.3.7
-sphinx>=1.4.4
-sphinxcontrib-httpdomain>=1.5.0
-esgf-pyclient>=0.1.6
-python-dateutil>=2.5.3
-mock>=2.0.0
-myproxyclient>=1.4.3
-podaacpy>=1.0.2
http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/ocw/dataset_loader.py
----------------------------------------------------------------------
diff --git a/ocw/dataset_loader.py b/ocw/dataset_loader.py
index ed8fdb0..fdcbb1b 100644
--- a/ocw/dataset_loader.py
+++ b/ocw/dataset_loader.py
@@ -21,11 +21,9 @@ Classes:
'''
import ocw.data_source.local as local
-import ocw.data_source.esgf as esgf
import ocw.data_source.rcmed as rcmed
-import ocw.data_source.dap as dap
import ocw.data_source.podaac_datasource as podaac
-
+import warnings
class DatasetLoader:
'''Generate a list of OCW Dataset objects from a variety of sources.'''
@@ -95,11 +93,19 @@ class DatasetLoader:
self._source_loaders = {
'local': local.load_multiple_files,
'local_split': local.load_dataset_from_multiple_netcdf_files,
- 'esgf': esgf.load_dataset,
'rcmed': rcmed.parameter_dataset,
- 'dap': dap.load,
'podaac': podaac.load_dataset
}
+
+ # Exclude esgf and dap for python 3 until they are compatible
+ try:
+ import ocw.data_source.esgf as esgf
+ import ocw.data_source.dap as dap
+ self._source_loaders['dap'] = dap.load
+ self._source_loaders['esgf'] = esgf.load_dataset
+ except ImportError:
+ warnings.warn('dap and esgf loaders missing. If these are needed, '
+ 'fallback to python 2.7.x.')
def add_source_loader(self, loader_name, loader_func):
'''
http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/test.sh
----------------------------------------------------------------------
diff --git a/test.sh b/test.sh
index 0ea488d..cc73d04 100755
--- a/test.sh
+++ b/test.sh
@@ -25,7 +25,11 @@ echo ""
# nosetests config file should be in home directory
cp .noserc $HOME/.noserc
+# Exclude dap tests if using python 3
+if [[ "$1" != "2.7" ]]; then
+ export NOSE_EXCLUDE_TESTS=ocw.tests.test_dap.TestDap
+fi
+
echo "---------------- Running Unit Tests ---------------"
nosetests
echo "---------------- All Tests successfully completed ---------------"
-
http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/test_smoke.py
----------------------------------------------------------------------
diff --git a/test_smoke.py b/test_smoke.py
index b1599df..f08d72a 100644
--- a/test_smoke.py
+++ b/test_smoke.py
@@ -21,9 +21,13 @@ from ocw.tests.test_local import create_netcdf_object
from ocw.data_source import local
from ocw import dataset_processor as dsp
import os
+import six
+
+if six.PY3:
+ DEPENDENCIES_FILE = 'deps_py3.txt'
+else:
+ DEPENDENCIES_FILE = 'deps_py2.txt'
-PIP_DEPENDENCIES_FILE = 'easy-ocw/ocw-pip-dependencies.txt'
-CONDA_DEPENDENCIES_FILE = 'easy-ocw/ocw-conda-dependencies.txt'
SUCCESS_MARK = '\033[92m' + u'\u2713' + '\033[0m'
FAILURE_MARK = '\033[91m' + u'\u274C' + '\033[0m'
@@ -123,11 +127,9 @@ def end(dataset=None):
def main():
- pip_file = open(PIP_DEPENDENCIES_FILE, 'r')
- conda_file = open(CONDA_DEPENDENCIES_FILE, 'r')
+ dep_file = open(DEPENDENCIES_FILE, 'r')
print("Checking installed dependencies\n")
- check_dependencies(conda_file)
- check_dependencies(pip_file)
+ check_dependencies(dep_file)
success("\nDependencies")
dataset = check_dataset_loading()
check_some_dataset_functions(dataset)